From a62b26dc8ed4917362737e7d9ebc81145b77dac3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:26:58 -0700 Subject: [PATCH 01/37] quest: claim quest/m1/quic/fork/switch Co-Authored-By: Claude Opus 5.5 From 590d323102f2dd9b7630077175b05ae164aa9b6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:32:13 -0700 Subject: [PATCH 02/37] feat(moq-tokio): import quinn's async layer and web-transport-moq verbatim Unmodified copies, not yet compiled, so a reviewer can diff the adaptation that follows against upstream: - quinn-rs/quinn 7616e6b2782722f3ce4a1b181ef08817d7f545e4 quinn/src -> rs/moq-tokio/src/quic/driver (lib.rs -> mod.rs), the commit moq-quic forked from. - moq-dev/noq e27c0ce00741d81c8620b98e345d4f2c88e0768c web-transport-moq/src -> rs/moq-tokio/src/quic/web (lib.rs -> mod.rs). Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1467 +++++++++++++ rs/moq-tokio/src/quic/driver/endpoint.rs | 1020 +++++++++ rs/moq-tokio/src/quic/driver/incoming.rs | 152 ++ rs/moq-tokio/src/quic/driver/mod.rs | 134 ++ rs/moq-tokio/src/quic/driver/mutex.rs | 163 ++ rs/moq-tokio/src/quic/driver/recv_stream.rs | 823 ++++++++ rs/moq-tokio/src/quic/driver/runtime/mod.rs | 330 +++ rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 + rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 111 + rs/moq-tokio/src/quic/driver/send_stream.rs | 454 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 1871 +++++++++++++++++ rs/moq-tokio/src/quic/driver/work_limiter.rs | 224 ++ rs/moq-tokio/src/quic/web/client.rs | 378 ++++ rs/moq-tokio/src/quic/web/connect.rs | 158 ++ rs/moq-tokio/src/quic/web/crypto.rs | 67 + rs/moq-tokio/src/quic/web/error.rs | 405 ++++ rs/moq-tokio/src/quic/web/mod.rs | 71 + rs/moq-tokio/src/quic/web/recv.rs | 159 ++ rs/moq-tokio/src/quic/web/send.rs | 197 ++ rs/moq-tokio/src/quic/web/server.rs | 200 ++ rs/moq-tokio/src/quic/web/session.rs | 1331 ++++++++++++ rs/moq-tokio/src/quic/web/settings.rs | 70 + 22 files changed, 9890 insertions(+) create mode 100644 rs/moq-tokio/src/quic/driver/connection.rs create mode 100644 rs/moq-tokio/src/quic/driver/endpoint.rs create mode 100644 rs/moq-tokio/src/quic/driver/incoming.rs create mode 100644 rs/moq-tokio/src/quic/driver/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/mutex.rs create mode 100644 rs/moq-tokio/src/quic/driver/recv_stream.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/tokio.rs create mode 100644 rs/moq-tokio/src/quic/driver/send_stream.rs create mode 100755 rs/moq-tokio/src/quic/driver/tests.rs create mode 100644 rs/moq-tokio/src/quic/driver/work_limiter.rs create mode 100644 rs/moq-tokio/src/quic/web/client.rs create mode 100644 rs/moq-tokio/src/quic/web/connect.rs create mode 100644 rs/moq-tokio/src/quic/web/crypto.rs create mode 100644 rs/moq-tokio/src/quic/web/error.rs create mode 100644 rs/moq-tokio/src/quic/web/mod.rs create mode 100644 rs/moq-tokio/src/quic/web/recv.rs create mode 100644 rs/moq-tokio/src/quic/web/send.rs create mode 100644 rs/moq-tokio/src/quic/web/server.rs create mode 100644 rs/moq-tokio/src/quic/web/session.rs create mode 100644 rs/moq-tokio/src/quic/web/settings.rs diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs new file mode 100644 index 0000000000..6ce968db47 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -0,0 +1,1467 @@ +use std::{ + any::Any, + fmt, + future::Future, + io, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, Waker, ready}, +}; + +use bytes::Bytes; +use pin_project_lite::pin_project; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tokio::sync::{ + Notify, + futures::{Notified, OwnedNotified}, + mpsc, oneshot, watch, +}; +use tracing::{Instrument, Span, debug_span}; + +use crate::{ + ConnectionEvent, Duration, Instant, VarInt, + mutex::Mutex, + recv_stream::RecvStream, + runtime::{AsyncTimer, Runtime, UdpSender}, + send_stream::SendStream, + udp_transmit, +}; +use proto::{ + ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, + StreamId, TransportError, TransportErrorCode, congestion::Controller, +}; + +/// In-progress connection attempt future +/// +/// Resolves into a [`Connection`] once the TLS handshake completes. In TLS 1.3, peer +/// certificates arrive with handshake completion, so they are only available via +/// [`Connection::peer_identity()`]. [`Connection`] is a lightweight handle, so completing the +/// handshake imposes no extra cost; use [`Connection::close()`] to reject peers that fail +/// custom validation. +#[derive(Debug)] +pub struct Connecting { + conn: Option, + connected: Pin>, + handshake_data_ready: Option>, +} + +impl Connecting { + pub(crate) fn new( + handle: ConnectionHandle, + conn: proto::Connection, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + sender: Pin>, + runtime: Arc, + ) -> Self { + let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); + + let conn = ConnectionRef(Arc::new(ConnectionInner { + state: Mutex::new(State::new( + conn, + handle, + endpoint_events, + conn_events, + on_handshake_data_send, + sender, + runtime.clone(), + )), + shared: Shared::default(), + })); + let connected = Box::pin(conn.shared.connected.clone().notified_owned()); + + let driver = ConnectionDriver::new(conn.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {e}"); + } + } + .instrument(Span::current()), + )); + + Self { + conn: Some(conn), + connected, + handshake_data_ready: Some(on_handshake_data_recv), + } + } + + /// Convert into a 0-RTT or 0.5-RTT connection at the cost of weakened security + /// + /// Returns `Ok` immediately if the local endpoint is able to attempt sending 0/0.5-RTT data. + /// If so, the returned [`Connection`] can be used to send application data without waiting for + /// the rest of the handshake to complete, at the cost of weakened cryptographic security + /// guarantees. The [`Connection::authenticated`] future resolves when the handshake does + /// complete, at which point subsequently opened streams and written data will have full + /// cryptographic protection. + /// + /// ## Outgoing + /// + /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually + /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This + /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may + /// arise when the handshake completes. If 0-RTT was rejected, the existence of streams opened + /// and other application data sent by a client prior to the handshake completing will not be + /// conveyed to the remote application. Clients which need to e.g. re-transmit rejected stream + /// data must take care to detect `ZeroRttRejected` errors even on finished streams opened + /// during 0-RTT, e.g. by awaiting an application-layer response on bidirectional streams, or + /// awaiting [`SendStream::stopped`] on unidirectional streams. + /// + /// A server may reject 0-RTT data at its discretion, but accepting 0-RTT data requires the + /// relevant resumption state to be stored in the server, which servers may limit or lose for + /// various reasons including not persisting resumption state across server restarts. + /// + /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Incoming + /// + /// For incoming connections, conversion to 0.5-RTT will always succeed. + /// + /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Security + /// + /// On outgoing connections, this enables transmission of 0-RTT data, which is vulnerable to + /// replay attacks, and should therefore never invoke non-idempotent operations. + /// + /// On incoming connections, this enables transmission of 0.5-RTT data, which may be sent + /// before TLS client authentication has occurred, and should therefore not be used to send + /// data for which client authentication is being used. + pub fn into_0rtt(mut self) -> Result { + // This lock borrows `self` and would normally be dropped at the end of this scope, so we'll + // have to release it explicitly before returning `self` by value. + let conn = (self.conn.as_mut().unwrap()).state.lock("into_0rtt"); + + let is_ok = conn.inner.has_0rtt() || conn.inner.side().is_server(); + drop(conn); + + if is_ok { + let conn = self.conn.take().unwrap(); + Ok(Connection(conn)) + } else { + Err(self) + } + } + + /// Parameters negotiated during the handshake + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a + /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// + /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. + /// + /// This operation is cancel-safe. + pub async fn handshake_data(&mut self) -> Result, ConnectionError> { + // Taking &mut self allows us to use a single oneshot channel rather than dealing with + // potentially many tasks waiting on the same event. It's a bit of a hack, but keeps things + // simple. + if let Some(x) = self.handshake_data_ready.as_mut() { + let _ = x.await; + // Once the data is ready, apply state changes. This prevents a panic due to + // inconsistent state when retrying a call to `handshake_data`. + self.handshake_data_ready = None; + } + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("handshake"); + inner + .inner + .crypto_session() + .handshake_data() + .ok_or_else(|| { + inner + .error + .clone() + .expect("spurious handshake data ready notification") + }) + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn local_ip(&self) -> Option { + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("local_ip"); + + inner.inner.local_ip() + } + + /// The peer's UDP address + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn remote_address(&self) -> SocketAddr { + let conn_ref: &ConnectionRef = self.conn.as_ref().expect("used after yielding Ready"); + conn_ref.state.lock("remote_address").inner.remote_address() + } +} + +impl Future for Connecting { + type Output = Result; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.connected).poll(cx).map(|_| { + let conn = self.conn.take().unwrap(); + let inner = conn.state.lock("connecting"); + if inner.connected { + drop(inner); + Ok(Connection(conn)) + } else { + Err(inner + .error + .clone() + .expect("connected signaled without connection success or error")) + } + }) + } +} + +/// A future that drives protocol logic for a connection +/// +/// This future handles the protocol logic for a single connection, routing events from the +/// `Connection` API object to the `Endpoint` task and the related stream-related interfaces. +/// It also keeps track of outstanding timeouts for the `Connection`. +/// +/// If the connection encounters an error condition, this future will yield an error. It will +/// terminate (yielding `Ok(())`) if the connection was closed without error. Unlike other +/// connection-related futures, this waits for the draining period to complete to ensure that +/// packets still in flight from the peer are handled gracefully. +#[must_use = "connection drivers must be spawned for their connections to function"] +#[derive(Debug)] +struct ConnectionDriver { + conn: ConnectionRef, + span: Span, +} + +impl ConnectionDriver { + fn new(conn: ConnectionRef) -> Self { + let span = { + let conn = &mut conn.state.lock("poll"); + debug_span!("drive", id = conn.handle.0) + }; + + Self { conn, span } + } +} + +impl Future for ConnectionDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let conn = &mut *self.conn.state.lock("poll"); + let _guard = self.span.enter(); + + if let Err(e) = conn.process_conn_events(&self.conn.shared, cx) { + conn.terminate(e, &self.conn.shared); + return Poll::Ready(Ok(())); + } + let mut keep_going = conn.drive_transmit(cx).inspect_err(|_| { + // Transmit failed, so close the connection and clean state before returning the error. + if !conn.inner.is_closed() { + conn.implicit_close(&self.conn.shared); + } + })?; + // If a timer expires, there might be more to transmit. When we transmit something, we + // might need to reset a timer. Hence, we must loop until neither happens. + keep_going |= conn.drive_timer(cx); + conn.forward_endpoint_events(); + conn.forward_app_events(&self.conn.shared); + + if !conn.inner.is_drained() { + if keep_going { + // If the connection hasn't processed all tasks, schedule it again + cx.waker().wake_by_ref(); + } else { + match conn.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => conn.driver = Some(cx.waker().clone()), + }; + } + return Poll::Pending; + } + if conn.error.is_none() { + unreachable!("drained connections always have an error"); + } + Poll::Ready(Ok(())) + } +} + +/// A QUIC connection. +/// +/// If all references to a connection (including every clone of the `Connection` handle, streams of +/// incoming streams, and the various stream types) have been dropped, then the connection will be +/// automatically closed with an `error_code` of 0 and an empty `reason`. You can also close the +/// connection explicitly by calling [`Connection::close()`]. +/// +/// Closing the connection immediately abandons efforts to deliver data to the peer. Upon +/// receiving CONNECTION_CLOSE the peer *may* drop any stream data not yet delivered to the +/// application. [`Connection::close()`] describes in more detail how to gracefully close a +/// connection without losing application data. +/// +/// May be cloned to obtain another handle to the same connection. +/// +/// [`Connection::close()`]: Connection::close +#[derive(Debug, Clone)] +pub struct Connection(ConnectionRef); + +impl Connection { + /// Initiate a new outgoing unidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. + pub fn open_uni(&self) -> OpenUni<'_> { + OpenUni { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Uni as usize].notified(), + } + } + + /// Initiate a new outgoing bidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing + /// anything to [`SendStream`] will never succeed. + /// + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + pub fn open_bi(&self) -> OpenBi<'_> { + OpenBi { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Bi as usize].notified(), + } + } + + /// Accept the next incoming uni-directional stream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_uni(&self) -> AcceptUni<'_> { + AcceptUni { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Uni as usize].notified(), + } + } + + /// Accept the next incoming bidirectional stream + /// + /// **Important Note**: The `Connection` that calls [`open_bi()`] must write to its [`SendStream`] + /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then + /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. + /// + /// [`accept_bi()`]: crate::Connection::accept_bi + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_bi(&self) -> AcceptBi<'_> { + AcceptBi { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Bi as usize].notified(), + } + } + + /// Receive an application datagram + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no datagrams were consumed. + pub fn read_datagram(&self) -> ReadDatagram<'_> { + ReadDatagram { + conn: &self.0, + notify: self.0.shared.datagram_received.notified(), + } + } + + /// Wait for the connection to be closed for any reason + /// + /// Despite the return type's name, closed connections are often not an error condition at the + /// application layer. Cases that might be routine include [`ConnectionError::LocallyClosed`] + /// and [`ConnectionError::ApplicationClosed`]. + pub async fn closed(&self) -> ConnectionError { + { + let conn = self.0.state.lock("closed"); + if let Some(error) = conn.error.as_ref() { + return error.clone(); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.closed.notified() + } + .await; + self.0 + .state + .lock("closed") + .error + .as_ref() + .expect("closed without an error") + .clone() + } + + /// Whether the connection is closed, and why. + /// + /// The close_reason is always set to `Some(ConnectionError)` when a socket is + /// closed; whether it was closed manually by calling [`Connection::close()`] or due to + /// an internal error (such as an idle timeout or the peer closing the + /// connection). + /// + /// Note: when the connection is closed, `connection.close_reason().is_some()` will always be true. + pub fn close_reason(&self) -> Option { + self.0.state.lock("close_reason").error.clone() + } + + /// Close the connection immediately. + /// + /// Pending operations will fail immediately with [`ConnectionError::LocallyClosed`]. No + /// more data is sent to the peer and the peer may drop buffered data upon receiving + /// the CONNECTION_CLOSE frame. + /// + /// `error_code` and `reason` are not interpreted, and are provided directly to the peer. + /// + /// `reason` will be truncated to fit in a single packet with overhead; to improve odds that it + /// is preserved in full, it should be kept under 1KiB. + /// + /// # Gracefully closing a connection + /// + /// Only the peer last receiving application data can be certain that all data is + /// delivered. The only reliable action it can then take is to close the connection, + /// potentially with a custom error code. The delivery of the final CONNECTION_CLOSE + /// frame is very likely if both endpoints stay online long enough, and + /// [`Endpoint::wait_idle()`] can be used to provide sufficient time. Otherwise, the + /// remote peer will time out the connection, provided that the idle timeout is not + /// disabled. + /// + /// The sending side can not guarantee all stream data is delivered to the remote + /// application. It only knows the data is delivered to the QUIC stack of the remote + /// endpoint. Once the local side sends a CONNECTION_CLOSE frame in response to calling + /// [`close()`] the remote endpoint may drop any data it received but is as yet + /// undelivered to the application, including data that was acknowledged as received to + /// the local endpoint. + /// + /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`close()`]: Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let conn = &mut *self.0.state.lock("close"); + conn.close(error_code, Bytes::copy_from_slice(reason), &self.0.shared); + } + + /// Wait for the handshake to be confirmed. + /// + /// As a server, who must be authenticated by clients, + /// this happens when the handshake completes + /// upon receiving a TLS Finished message from the client. + /// In return, the server send a HANDSHAKE_DONE frame. + /// + /// As a client, this happens when receiving a HANDSHAKE_DONE frame. + /// At this point, the server has either accepted our authentication, + /// or, if client authentication is not required, accepted our lack of authentication. + pub async fn handshake_confirmed(&self) -> Result<(), ConnectionError> { + { + let conn = self.0.state.lock("handshake_confirmed"); + if let Some(error) = conn.error.as_ref() { + return Err(error.clone()); + } + if conn.handshake_confirmed { + return Ok(()); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.handshake_confirmed.notified() + } + .await; + if let Some(error) = self.0.state.lock("handshake_confirmed").error.as_ref() { + Err(error.clone()) + } else { + Ok(()) + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Application datagrams are a low-level primitive. They may be lost or delivered out of order, + /// and `data` must both fit inside a single QUIC packet and be smaller than the maximum + /// dictated by the peer. + /// + /// Previously queued datagrams which are still unsent may be discarded to make space for this + /// datagram, in order of oldest to newest. + pub fn send_datagram(&self, data: Bytes) -> Result<(), SendDatagramError> { + let conn = &mut *self.0.state.lock("send_datagram"); + if let Some(x) = &conn.error { + return Err(SendDatagramError::ConnectionLost(x.clone())); + } + use proto::SendDatagramError::*; + match conn.inner.datagrams().send(data, true) { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(e) => Err(match e { + Blocked(..) => unreachable!(), + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + }), + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Unlike [`send_datagram()`], this method will wait for buffer space during congestion + /// conditions, which effectively prioritizes old datagrams over new datagrams. + /// + /// See [`send_datagram()`] for details. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn send_datagram_wait(&self, data: Bytes) -> SendDatagram<'_> { + SendDatagram { + conn: &self.0, + data: Some(data), + notify: self.0.shared.datagrams_unblocked.notified(), + } + } + + /// Compute the maximum size of datagrams that may be passed to [`send_datagram()`]. + /// + /// Returns `None` if datagrams are unsupported by the peer or disabled locally. + /// + /// This may change over the lifetime of a connection according to variation in the path MTU + /// estimate. The peer can also enforce an arbitrarily small fixed limit, but if the peer's + /// limit is large this is guaranteed to be a little over a kilobyte at minimum. + /// + /// Not necessarily the maximum size of received datagrams. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn max_datagram_size(&self) -> Option { + self.0 + .state + .lock("max_datagram_size") + .inner + .datagrams() + .max_size() + } + + /// Bytes available in the outgoing datagram buffer + /// + /// When greater than zero, calling [`send_datagram()`](Self::send_datagram) with a datagram of + /// at most this size is guaranteed not to cause older datagrams to be dropped. + pub fn datagram_send_buffer_space(&self) -> usize { + self.0 + .state + .lock("datagram_send_buffer_space") + .inner + .datagrams() + .send_buffer_space() + } + + /// The side of the connection (client or server) + pub fn side(&self) -> Side { + self.0.state.lock("side").inner.side() + } + + /// The peer's UDP address + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn remote_address(&self) -> SocketAddr { + self.0.state.lock("remote_address").inner.remote_address() + } + + /// Watch the peer's UDP address. + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn watch_remote_address(&self) -> RemoteAddressWatcher { + let receiver = self + .0 + .state + .lock("watch_remote_address") + .remote_address + .subscribe(); + + RemoteAddressWatcher { receiver } + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + pub fn local_ip(&self) -> Option { + self.0.state.lock("local_ip").inner.local_ip() + } + + /// Current best estimate of this connection's latency (round-trip-time) + pub fn rtt(&self) -> Duration { + self.0.state.lock("rtt").inner.rtt() + } + + /// Minimum RTT seen on this path, ignoring ack delay + pub fn min_rtt(&self) -> Duration { + self.0.state.lock("min_rtt").inner.min_rtt() + } + + /// Returns connection statistics + pub fn stats(&self) -> ConnectionStats { + self.0.state.lock("stats").inner.stats() + } + + /// Current state of the congestion control algorithm, for debugging purposes + pub fn congestion_state(&self) -> Box { + self.0 + .state + .lock("congestion_state") + .inner + .congestion_state() + .clone_box() + } + + /// Succeeds when an incoming connection is proven not to be a replay attack. + /// + /// Only interesting for `Connection`s obtained from [`Connecting::into_0rtt`]. On 1-RTT + /// connections, always completes immediately. Contrast + /// [`handshake_confirmed`](Self::handshake_confirmed), which waits longer on clients e.g. to + /// confirm client authentication. + /// + /// For incoming connections, reads from [`RecvStream`]s are guaranteed not to arise from replay + /// attacks after this succeeds, even for streams accepted or read during 0-RTT. For outgoing + /// connections, streams opened after this succeeds will never be discarded by the server due to + /// 0-RTT rejection. + pub async fn authenticated(&self) -> Result<(), ConnectionError> { + let notified = { + let conn = self.0.state.lock("connected"); + if let Some(e) = &conn.error { + return Err(e.clone()); + } + if conn.connected { + return Ok(()); + } + self.0.shared.connected.notified() + }; + notified.await; + let conn = self.0.state.lock("connected"); + conn.error.clone().map_or(Ok(()), Err) + } + + /// Parameters negotiated during the handshake + /// + /// Guaranteed to return `Some` on fully established connections or after + /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on + /// the returned value. + /// + /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + pub fn handshake_data(&self) -> Option> { + self.0 + .state + .lock("handshake_data") + .inner + .crypto_session() + .handshake_data() + } + + /// Cryptographic identity of the peer + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. + /// + /// For custom asynchronous validation (e.g. checking a certificate against an external + /// database), call this after awaiting [`Connecting`] and use [`close()`][Self::close] to + /// reject untrusted peers. + pub fn peer_identity(&self) -> Option> { + self.0 + .state + .lock("peer_identity") + .inner + .crypto_session() + .peer_identity() + } + + /// A stable identifier for this connection + /// + /// Peer addresses and connection IDs can change, but this value will remain + /// fixed for the lifetime of the connection. + pub fn stable_id(&self) -> usize { + self.0.stable_id() + } + + /// Update traffic keys spontaneously + /// + /// This primarily exists for testing purposes. + pub fn force_key_update(&self) { + self.0 + .state + .lock("force_key_update") + .inner + .force_key_update() + } + + /// Derive keying material from this connection's TLS session secrets. + /// + /// When both peers call this method with the same `label` and `context` + /// arguments and `output` buffers of equal length, they will get the + /// same sequence of bytes in `output`. These bytes are cryptographically + /// strong and pseudorandom, and are suitable for use as keying material. + /// + /// See [RFC5705](https://tools.ietf.org/html/rfc5705) for more information. + pub fn export_keying_material( + &self, + output: &mut [u8], + label: &[u8], + context: &[u8], + ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + self.0 + .state + .lock("export_keying_material") + .inner + .crypto_session() + .export_keying_material(output, label, context) + } + + /// Modify the number of remotely initiated unidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_uni_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_uni_streams"); + conn.inner.set_max_concurrent_streams(Dir::Uni, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } + + /// See [`proto::TransportConfig::send_window()`] + pub fn set_send_window(&self, send_window: u64) { + let mut conn = self.0.state.lock("set_send_window"); + conn.inner.set_send_window(send_window); + conn.wake(); + } + + /// See [`proto::TransportConfig::receive_window()`] + pub fn set_receive_window(&self, receive_window: VarInt) { + let mut conn = self.0.state.lock("set_receive_window"); + conn.inner.set_receive_window(receive_window); + conn.wake(); + } + + /// Modify the number of remotely initiated bidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_bi_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_bi_streams"); + conn.inner.set_max_concurrent_streams(Dir::Bi, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } +} + +pin_project! { + /// Future produced by [`Connection::open_uni`] + pub struct OpenUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenUni<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(SendStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::open_bi`] + pub struct OpenBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Bi))?; + + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_open<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_open"); + if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } else if let Some(id) = state.inner.streams().open(dir) { + let is_0rtt = state.inner.side().is_client() && state.inner.is_handshaking(); + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => { + notify.set(conn.shared.stream_budget_available[dir as usize].notified()) + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::accept_uni`] + pub struct AcceptUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptUni<'_> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(RecvStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::accept_bi`] + pub struct AcceptBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Bi))?; + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_accept<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_accept"); + // Check for incoming streams before checking `state.error` so that already-received streams, + // which are necessarily finite, can be drained from a closed connection. + if let Some(id) = state.inner.streams().accept(dir) { + let is_0rtt = state.inner.is_handshaking(); + state.wake(); // To send additional stream ID credit + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => notify.set(conn.shared.stream_incoming[dir as usize].notified()), + } + } +} + +pin_project! { + /// Future produced by [`Connection::read_datagram`] + pub struct ReadDatagram<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for ReadDatagram<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("ReadDatagram::poll"); + // Check for buffered datagrams before checking `state.error` so that already-received + // datagrams, which are necessarily finite, can be drained from a closed connection. + if let Some(x) = state.inner.datagrams().recv() { + return Poll::Ready(Ok(x)); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagram_received.notified()), + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::send_datagram_wait`] + pub struct SendDatagram<'a> { + conn: &'a ConnectionRef, + data: Option, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for SendDatagram<'_> { + type Output = Result<(), SendDatagramError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("SendDatagram::poll"); + if let Some(e) = &state.error { + return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); + } + use proto::SendDatagramError::*; + match state + .inner + .datagrams() + .send(this.data.take().unwrap(), false) + { + Ok(()) => { + state.wake(); + Poll::Ready(Ok(())) + } + Err(e) => Poll::Ready(Err(match e { + Blocked(data) => { + this.data.replace(data); + loop { + match this.notify.as_mut().poll(ctx) { + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagrams_unblocked.notified()), + } + } + } + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + })), + } + } +} + +/// Watcher produced by [`Connection::watch_remote_address`] +pub struct RemoteAddressWatcher { + receiver: watch::Receiver, +} + +impl RemoteAddressWatcher { + /// Wait for an address change, mark it as seen and return the address. + /// + /// If multiple migrations occur between calls to `wait` or polls of the resulting future, + /// the last observed address will be returned, skipping intermediate updates. + pub async fn wait(&mut self) -> Option { + if self.receiver.changed().await.is_err() { + return None; + } + + Some(*self.receiver.borrow()) + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionRef(Arc); + +impl ConnectionRef { + fn stable_id(&self) -> usize { + &*self.0 as *const _ as usize + } +} + +impl Clone for ConnectionRef { + fn clone(&self) -> Self { + self.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for ConnectionRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let conn = &mut *self.state.lock("drop"); + + if !conn.inner.is_closed() { + // If the driver is alive, it's just it and us, so we'd better shut it down. If it's + // not, we can't do any harm. If there were any streams being opened, then either + // the connection will be closed for an unrelated reason or a fresh reference will + // be constructed for the newly opened stream. + conn.implicit_close(&self.shared); + } + } +} + +impl std::ops::Deref for ConnectionRef { + type Target = ConnectionInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +#[derive(Debug, Default)] +pub(crate) struct Shared { + handshake_confirmed: Notify, + /// Notified when new streams may be locally initiated due to an increase in stream ID flow + /// control budget + stream_budget_available: [Notify; 2], + /// Notified when the peer has initiated a new stream + stream_incoming: [Notify; 2], + datagram_received: Notify, + datagrams_unblocked: Notify, + closed: Notify, + connected: Arc, + /// Number of live handles that can used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +pub(crate) struct State { + pub(crate) inner: proto::Connection, + driver: Option, + handle: ConnectionHandle, + on_handshake_data: Option>, + connected: bool, + handshake_confirmed: bool, + timer: Option>>, + timer_deadline: Option, + conn_events: mpsc::UnboundedReceiver, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + pub(crate) blocked_writers: FxHashMap, + pub(crate) blocked_readers: FxHashMap, + pub(crate) stopped: FxHashMap>, + /// Always set to Some before the connection becomes drained + pub(crate) error: Option, + sender: Pin>, + runtime: Arc, + send_buffer: Vec, + /// We buffer a transmit when the underlying I/O would block + buffered_transmit: Option, + remote_address: watch::Sender, +} + +impl State { + #[allow(clippy::too_many_arguments)] + fn new( + inner: proto::Connection, + handle: ConnectionHandle, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + on_handshake_data: oneshot::Sender<()>, + sender: Pin>, + runtime: Arc, + ) -> Self { + let remote_address = inner.remote_address(); + + Self { + inner, + driver: None, + handle, + on_handshake_data: Some(on_handshake_data), + connected: false, + handshake_confirmed: false, + timer: None, + timer_deadline: None, + conn_events, + endpoint_events, + blocked_writers: FxHashMap::default(), + blocked_readers: FxHashMap::default(), + stopped: FxHashMap::default(), + error: None, + sender, + runtime, + send_buffer: Vec::new(), + buffered_transmit: None, + remote_address: watch::Sender::new(remote_address), + } + } + + fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { + let now = self.runtime.now(); + let mut transmits = 0; + + let max_datagrams = self + .sender + .max_transmit_segments() + .min(MAX_TRANSMIT_SEGMENTS); + + loop { + // Retry the last transmit, or get a new one. + let t = match self.buffered_transmit.take() { + Some(t) => t, + None => { + self.send_buffer.clear(); + self.send_buffer.reserve(self.inner.current_mtu() as usize); + match self + .inner + .poll_transmit(now, max_datagrams, &mut self.send_buffer) + { + Some(t) => { + transmits += match t.segment_size { + None => 1, + Some(s) => t.size.div_ceil(s), // round up + }; + t + } + None => break, + } + } + }; + + let len = t.size; + match self + .sender + .as_mut() + .poll_send(&udp_transmit(&t, &self.send_buffer[..len]), cx) + { + Poll::Pending => { + self.buffered_transmit = Some(t); + return Ok(false); + } + Poll::Ready(Err(e)) => return Err(e), + Poll::Ready(Ok(())) => {} + } + + if transmits >= MAX_TRANSMIT_DATAGRAMS { + // TODO: What isn't ideal here yet is that if we don't poll all + // datagrams that could be sent we don't go into the `app_limited` + // state and CWND continues to grow until we get here the next time. + // See https://github.com/quinn-rs/quinn/issues/1126 + return Ok(true); + } + } + + Ok(false) + } + + fn forward_endpoint_events(&mut self) { + while let Some(event) = self.inner.poll_endpoint_events() { + // If the endpoint driver is gone, noop. + let _ = self.endpoint_events.send((self.handle, event)); + } + } + + /// If this returns `Err`, the endpoint is dead, so the driver should exit immediately. + fn process_conn_events( + &mut self, + shared: &Shared, + cx: &mut Context<'_>, + ) -> Result<(), ConnectionError> { + loop { + match self.conn_events.poll_recv(cx) { + Poll::Ready(Some(ConnectionEvent::Rebind(sender))) => { + self.sender = sender; + self.inner.local_address_changed(); + } + Poll::Ready(Some(ConnectionEvent::Proto(event))) => { + self.inner.handle_event(event); + } + Poll::Ready(Some(ConnectionEvent::Close { reason, error_code })) => { + self.close(error_code, reason, shared); + } + Poll::Ready(None) => { + return Err(ConnectionError::TransportError(TransportError::new( + TransportErrorCode::INTERNAL_ERROR, + "endpoint driver future was dropped".to_string(), + ))); + } + Poll::Pending => { + return Ok(()); + } + } + } + } + + fn forward_app_events(&mut self, shared: &Shared) { + while let Some(event) = self.inner.poll() { + use proto::Event::*; + match event { + HandshakeDataReady => { + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + } + Connected => { + self.connected = true; + shared.connected.notify_waiters(); + if self.inner.side().is_client() && !self.inner.accepted_0rtt() { + // Wake up rejected 0-RTT streams so they can fail immediately with + // `ZeroRttRejected` errors. + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + wake_all_notify(&mut self.stopped); + } + } + HandshakeConfirmed => { + self.handshake_confirmed = true; + shared.handshake_confirmed.notify_waiters(); + } + ConnectionLost { reason } => { + self.terminate(reason, shared); + } + Stream(StreamEvent::Writable { id }) => wake_stream(id, &mut self.blocked_writers), + Stream(StreamEvent::Opened { dir: Dir::Uni }) => { + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + } + Stream(StreamEvent::Opened { dir: Dir::Bi }) => { + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + } + DatagramReceived => { + shared.datagram_received.notify_waiters(); + } + DatagramsUnblocked => { + shared.datagrams_unblocked.notify_waiters(); + } + Stream(StreamEvent::Readable { id }) => wake_stream(id, &mut self.blocked_readers), + Stream(StreamEvent::Available { dir }) => { + // Might mean any number of streams are ready, so we wake up everyone + shared.stream_budget_available[dir as usize].notify_waiters(); + } + Stream(StreamEvent::Finished { id }) => wake_stream_notify(id, &mut self.stopped), + Stream(StreamEvent::Stopped { id, .. }) => { + wake_stream_notify(id, &mut self.stopped); + wake_stream(id, &mut self.blocked_writers); + } + PathUpdated => { + // Using `send_replace` since `send` does not update the value + // when there are no subscribed receivers. + let _ = self + .remote_address + .send_replace(self.inner.remote_address()); + } + _ => {} + } + } + } + + fn drive_timer(&mut self, cx: &mut Context<'_>) -> bool { + let Some(deadline) = self.inner.poll_timeout() else { + self.timer_deadline = None; + return false; + }; + + // Use the clock rather than the async timer to detect expiry: Sleep::poll + // respects Tokio's cooperative budget and can return Pending for elapsed + // deadlines. + let now = self.runtime.now(); + if now >= deadline { + self.inner.handle_timeout(now); + self.timer_deadline = None; + return true; + } + + match &mut self.timer { + // Avoid resetting the timer when the deadline is unchanged. + Some(delay) if self.timer_deadline != Some(deadline) => { + delay.as_mut().reset(deadline); + } + None => { + self.timer = Some(self.runtime.new_timer(deadline)); + } + _ => {} + } + self.timer_deadline = Some(deadline); + + let delay = self + .timer + .as_mut() + .expect("timer must exist in this state") + .as_mut(); + if delay.poll(cx).is_pending() { + return false; + } + + // The deadline elapsed in the window between the clock check and poll. + self.inner.handle_timeout(self.runtime.now()); + self.timer_deadline = None; + true + } + + /// Wake up a blocked `Driver` task to process I/O + pub(crate) fn wake(&mut self) { + if let Some(x) = self.driver.take() { + x.wake(); + } + } + + /// Used to wake up all blocked futures when the connection becomes closed for any reason + fn terminate(&mut self, reason: ConnectionError, shared: &Shared) { + self.error = Some(reason.clone()); + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + shared.stream_budget_available[Dir::Uni as usize].notify_waiters(); + shared.stream_budget_available[Dir::Bi as usize].notify_waiters(); + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + shared.datagram_received.notify_waiters(); + shared.datagrams_unblocked.notify_waiters(); + shared.handshake_confirmed.notify_waiters(); + wake_all_notify(&mut self.stopped); + shared.closed.notify_waiters(); + shared.connected.notify_waiters(); + } + + fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { + self.inner.close(self.runtime.now(), error_code, reason); + self.terminate(ConnectionError::LocallyClosed, shared); + self.wake(); + } + + /// Close for a reason other than the application's explicit request + pub(crate) fn implicit_close(&mut self, shared: &Shared) { + self.close(0u32.into(), Bytes::new(), shared); + } + + pub(crate) fn check_0rtt(&self) -> Result<(), ()> { + if self.inner.is_handshaking() + || self.inner.accepted_0rtt() + || self.inner.side().is_server() + { + Ok(()) + } else { + Err(()) + } + } +} + +impl Drop for State { + fn drop(&mut self) { + if !self.inner.is_drained() { + // Ensure the endpoint can tidy up + let _ = self + .endpoint_events + .send((self.handle, EndpointEvent::drained())); + } + } +} + +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("State").field("inner", &self.inner).finish() + } +} + +fn wake_stream(stream_id: StreamId, wakers: &mut FxHashMap) { + if let Some(waker) = wakers.remove(&stream_id) { + waker.wake(); + } +} + +fn wake_all(wakers: &mut FxHashMap) { + wakers.drain().for_each(|(_, waker)| waker.wake()) +} + +fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { + if let Some(notify) = wakers.remove(&stream_id) { + notify.notify_waiters() + } +} + +fn wake_all_notify(wakers: &mut FxHashMap>) { + wakers + .drain() + .for_each(|(_, notify)| notify.notify_waiters()) +} + +/// Errors that can arise when sending a datagram +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub enum SendDatagramError { + /// The peer does not support receiving datagram frames + #[error("datagrams not supported by peer")] + UnsupportedByPeer, + /// Datagram support is disabled locally + #[error("datagram support disabled")] + Disabled, + /// The datagram is larger than the connection can currently accommodate + /// + /// Indicates that the path MTU minus overhead or the limit advertised by the peer has been + /// exceeded. + #[error("datagram too large")] + TooLarge, + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), +} + +/// The maximum amount of datagrams which will be produced in a single `drive_transmit` call +/// +/// This limits the amount of CPU resources consumed by datagram generation, +/// and allows other tasks (like receiving ACKs) to run in between. +const MAX_TRANSMIT_DATAGRAMS: usize = 20; + +/// The maximum amount of datagrams that are sent in a single transmit +/// +/// This can be lower than the maximum platform capabilities, to avoid excessive +/// memory allocations when calling `poll_transmit()`. Benchmarks have shown +/// that numbers around 10 are a good compromise. +const MAX_TRANSMIT_SEGMENTS: usize = 10; diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs new file mode 100644 index 0000000000..2012e415ff --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -0,0 +1,1020 @@ +use std::{ + collections::VecDeque, + fmt, + future::Future, + io::{self, IoSliceMut}, + mem, + net::{SocketAddr, SocketAddrV6}, + pin::Pin, + str, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use crate::runtime::default_runtime; +use crate::{ + Instant, + runtime::{AsyncUdpSocket, Runtime, UdpSender}, + udp_transmit, +}; +use bytes::{Bytes, BytesMut}; +use pin_project_lite::pin_project; +use proto::{ + self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, + EndpointEvent, ServerConfig, +}; +use rustc_hash::FxHashMap; +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use socket2::{Domain, Protocol, Socket, Type}; +use tokio::sync::{Notify, futures::Notified, mpsc}; +use tracing::{Instrument, Span}; +use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; + +use crate::{ + ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, + connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, +}; + +/// A QUIC endpoint. +/// +/// An endpoint corresponds to a single UDP socket, may host many connections, and may act as both +/// client and server for different connections. +/// +/// May be cloned to obtain another handle to the same endpoint. +#[derive(Debug, Clone)] +pub struct Endpoint { + pub(crate) inner: EndpointRef, + runtime: Arc, +} + +impl Endpoint { + /// Helper to construct an endpoint for use with outgoing connections only + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 addresses. As such, calling `Endpoint::client` with + /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other + /// address. For example: + /// + /// ``` + /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ``` + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn client(addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + None, + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Returns relevant stats from this Endpoint + pub fn stats(&self) -> EndpointStats { + self.inner.state.lock().unwrap().stats + } + + /// Helper to construct an endpoint for use with both incoming and outgoing connections + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 clients. As such, calling `Endpoint::server` with + /// the address `[::]:0` is a reasonable default to maximize the ability to accept connections + /// from any address. + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never + /// dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + Some(config), + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Construct an endpoint with arbitrary configuration and socket + #[cfg(not(wasm_browser))] + pub fn new( + config: EndpointConfig, + server_config: Option, + socket: std::net::UdpSocket, + runtime: Arc, + ) -> io::Result { + let socket = runtime.wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket, runtime) + } + + /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket + /// + /// Useful when `socket` has additional state (e.g. sidechannels) attached for which shared + /// ownership is needed. + pub fn new_with_abstract_socket( + config: EndpointConfig, + server_config: Option, + socket: Box, + runtime: Arc, + ) -> io::Result { + let addr = socket.local_addr()?; + let allow_mtud = !socket.may_fragment(); + let rc = EndpointRef::new( + socket, + proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + addr.is_ipv6(), + runtime.clone(), + ); + let driver = EndpointDriver(rc.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {}", e); + } + } + .instrument(Span::current()), + )); + Ok(Self { inner: rc, runtime }) + } + + /// Get the next incoming connection attempt from a client + /// + /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] + /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// filter connection attempts or force address validation, or converted into an intermediate + /// `Connecting` future which can be used to e.g. send 0.5-RTT data. + pub fn accept(&self) -> Accept<'_> { + Accept { + endpoint: self, + notify: self.inner.shared.incoming.notified(), + } + } + + /// Set the client configuration used by `connect` + pub fn set_default_client_config(&self, config: ClientConfig) { + self.inner.0.state.lock().unwrap().default_client_config = Some(config); + } + + /// Connect to a remote endpoint + /// + /// `server_name` must be covered by the certificate presented by the server. This prevents a + /// connection from being intercepted by an attacker with a valid certificate for some other + /// server. + /// + /// May fail immediately due to configuration errors, or in the future if the connection could + /// not be established. + pub fn connect(&self, addr: SocketAddr, server_name: &str) -> Result { + let Some(config) = self + .inner + .0 + .state + .lock() + .unwrap() + .default_client_config + .clone() + else { + return Err(ConnectError::NoDefaultClientConfig); + }; + + self.connect_with(config, addr, server_name) + } + + /// Connect to a remote endpoint using a custom configuration. + /// + /// See [`connect()`] for details. + /// + /// [`connect()`]: Endpoint::connect + pub fn connect_with( + &self, + config: ClientConfig, + addr: SocketAddr, + server_name: &str, + ) -> Result { + let mut endpoint = self.inner.state.lock().unwrap(); + if endpoint.driver_lost || endpoint.recv_state.connections.close.is_some() { + return Err(ConnectError::EndpointStopping); + } + if addr.is_ipv6() && !endpoint.ipv6 { + return Err(ConnectError::InvalidRemoteAddress(addr)); + } + let addr = if endpoint.ipv6 { + SocketAddr::V6(ensure_ipv6(addr)) + } else { + addr + }; + + let (ch, conn) = endpoint + .inner + .connect(self.runtime.now(), config, addr, server_name)?; + + let sender = endpoint.socket.create_sender(); + endpoint.stats.outgoing_handshakes += 1; + Ok(endpoint + .recv_state + .connections + .insert(ch, conn, sender, false, self.runtime.clone())) + } + + /// Switch to a new UDP socket + /// + /// See [`Endpoint::rebind_abstract()`] for details. + #[cfg(not(wasm_browser))] + pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { + self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + } + + /// Switch to a new UDP socket + /// + /// Allows the endpoint's address to be updated live, affecting all active connections. Incoming + /// connections and connections to servers unreachable from the new address will be lost. + /// + /// On error, the old UDP socket is retained. + pub fn rebind_abstract(&self, socket: Box) -> io::Result<()> { + let addr = socket.local_addr()?; + let mut inner = self.inner.state.lock().unwrap(); + inner.prev_socket = Some(mem::replace(&mut inner.socket, socket)); + inner.sender = inner.socket.create_sender(); + inner.ipv6 = addr.is_ipv6(); + + // Update connection socket references + for sender in inner.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Rebind(inner.socket.create_sender())); + } + if let Some(driver) = inner.driver.take() { + // Ensure the driver can register for wake-ups from the new socket + driver.wake(); + } + + Ok(()) + } + + /// Replace the server configuration, affecting new incoming connections only + /// + /// Useful for e.g. refreshing TLS certificates without disrupting existing connections. + pub fn set_server_config(&self, server_config: Option) { + self.inner + .state + .lock() + .unwrap() + .inner + .set_server_config(server_config.map(Arc::new)) + } + + /// Get the local `SocketAddr` the underlying socket is bound to + pub fn local_addr(&self) -> io::Result { + self.inner.state.lock().unwrap().socket.local_addr() + } + + /// Get the number of connections that are currently open + pub fn open_connections(&self) -> usize { + self.inner.state.lock().unwrap().inner.open_connections() + } + + /// Close all of this endpoint's connections immediately and cease accepting new connections. + /// + /// See [`Connection::close()`] for details. + /// + /// [`Connection::close()`]: crate::Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let reason = Bytes::copy_from_slice(reason); + let mut endpoint = self.inner.state.lock().unwrap(); + endpoint.recv_state.connections.close = Some((error_code, reason.clone())); + for sender in endpoint.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Close { + error_code, + reason: reason.clone(), + }); + } + self.inner.shared.incoming.notify_waiters(); + } + + /// Wait for all connections on the endpoint to be cleanly shut down + /// + /// Waiting for this condition before exiting ensures that a good-faith effort is made to notify + /// peers of recent connection closes, whereas exiting immediately could force them to wait out + /// the idle timeout period. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + pub async fn wait_idle(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.is_idle() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.idle.notified() + } + .await; + } + } +} + +/// Statistics on [Endpoint] activity +#[non_exhaustive] +#[derive(Debug, Default, Copy, Clone)] +pub struct EndpointStats { + /// Cumulative number of Quic handshakes accepted by this [Endpoint] + pub accepted_handshakes: u64, + /// Cumulative number of Quic handshakes sent from this [Endpoint] + pub outgoing_handshakes: u64, + /// Cumulative number of Quic handshakes refused on this [Endpoint] + pub refused_handshakes: u64, + /// Cumulative number of Quic handshakes ignored on this [Endpoint] + pub ignored_handshakes: u64, +} + +/// A future that drives IO on an endpoint +/// +/// This task functions as the switch point between the UDP socket object and the +/// `Endpoint` responsible for routing datagrams to their owning `Connection`. +/// In order to do so, it also facilitates the exchange of different types of events +/// flowing between the `Endpoint` and the tasks managing `Connection`s. As such, +/// running this task is necessary to keep the endpoint's connections running. +/// +/// `EndpointDriver` futures terminate when all clones of the `Endpoint` have been dropped, or when +/// an I/O error occurs. +#[must_use = "endpoint drivers must be spawned for I/O to occur"] +#[derive(Debug)] +pub(crate) struct EndpointDriver(pub(crate) EndpointRef); + +impl Future for EndpointDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut endpoint = self.0.state.lock().unwrap(); + match endpoint.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => endpoint.driver = Some(cx.waker().clone()), + } + + let now = endpoint.runtime.now(); + let mut keep_going = false; + keep_going |= endpoint.drive_recv(cx, now)?; + keep_going |= endpoint.handle_events(cx, &self.0.shared); + + if !endpoint.recv_state.incoming.is_empty() { + self.0.shared.incoming.notify_waiters(); + } + + if self.0.shared.ref_count.load(Ordering::Relaxed) == 0 && endpoint.is_idle() { + Poll::Ready(Ok(())) + } else { + drop(endpoint); + // If there is more work to do schedule the endpoint task again. + // `wake_by_ref()` is called outside the lock to minimize + // lock contention on a multithreaded runtime. + if keep_going { + cx.waker().wake_by_ref(); + } + Poll::Pending + } + } +} + +impl Drop for EndpointDriver { + fn drop(&mut self) { + let mut endpoint = self.0.state.lock().unwrap(); + endpoint.driver_lost = true; + self.0.shared.incoming.notify_waiters(); + // Drop all outgoing channels, signaling the termination of the endpoint to the associated + // connections. + endpoint.recv_state.connections.senders.clear(); + // Clearing the senders is what makes the endpoint idle, but a waiter already parked on + // `idle` only re-reads that after being notified, and with the driver gone nothing else + // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. + if endpoint.is_idle() { + self.0.shared.idle.notify_waiters(); + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +impl EndpointInner { + pub(crate) fn accept( + &self, + incoming: proto::Incoming, + server_config: Option>, + ) -> Result { + let mut response_buffer = Vec::new(); + + // Phase 1: reserve endpoint state for the connection under the lock. + let accepting = { + let mut state = self.state.lock().unwrap(); + let now = state.runtime.now(); + match state + .inner + .start_accept(incoming, now, &mut response_buffer, server_config) + { + Ok(accepting) => { + state.pending_accepts += 1; + accepting + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + return Err(error.cause); + } + } + }; + + // Phase 2: TLS session setup, connection construction, and first-packet handling, + // without holding the lock. + let accepted = accepting.accept(); + + // Phase 3: register the connection, or release the reservation, under the lock. + let mut state = self.state.lock().unwrap(); + state.pending_accepts -= 1; + let result = match state.inner.finish_accept(accepted, &mut response_buffer) { + Ok((handle, conn)) => { + state.stats.accepted_handshakes += 1; + let sender = state.socket.create_sender(); + let runtime = state.runtime.clone(); + let driver_lost = state.driver_lost; + Ok(state + .recv_state + .connections + .insert(handle, conn, sender, driver_lost, runtime)) + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + Err(error.cause) + } + }; + // Failed accepts and accepts completed after driver loss cannot rely on a Drained event + // being processed to wake idle waiters. + if state.is_idle() { + self.shared.idle.notify_waiters(); + } + result + } + + pub(crate) fn refuse(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.refused_handshakes += 1; + let mut response_buffer = Vec::new(); + let transmit = state.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut state.sender); + } + + pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + let mut state = self.state.lock().unwrap(); + let mut response_buffer = Vec::new(); + let transmit = state.inner.retry(incoming, &mut response_buffer)?; + respond(transmit, &response_buffer, &mut state.sender); + Ok(()) + } + + pub(crate) fn ignore(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.ignored_handshakes += 1; + state.inner.ignore(incoming); + } +} + +#[derive(Debug)] +pub(crate) struct State { + socket: Box, + sender: Pin>, + /// During an active migration, abandoned_socket receives traffic + /// until the first packet arrives on the new socket. + prev_socket: Option>, + inner: proto::Endpoint, + recv_state: RecvState, + driver: Option, + ipv6: bool, + events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, + driver_lost: bool, + runtime: Arc, + stats: EndpointStats, + default_client_config: Option, + /// Connections in the process of being accepted + pending_accepts: usize, +} + +#[derive(Debug)] +pub(crate) struct Shared { + incoming: Notify, + idle: Notify, + /// Number of live handles that can be used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +impl State { + fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { + let get_time = || self.runtime.now(); + self.recv_state.recv_limiter.start_cycle(get_time); + if let Some(socket) = &mut self.prev_socket { + // We don't care about the `PollProgress` from old sockets. + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut **socket, + &mut self.sender, + &*self.runtime, + now, + ); + if poll_res.is_err() { + self.prev_socket = None; + } + }; + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut *self.socket, + &mut self.sender, + &*self.runtime, + now, + ); + self.recv_state.recv_limiter.finish_cycle(get_time); + let poll_res = poll_res?; + if poll_res.received_connection_packet { + // Traffic has arrived on self.socket, therefore there is no need for the abandoned + // one anymore. TODO: Account for multiple outgoing connections. + self.prev_socket = None; + } + Ok(poll_res.keep_going) + } + + fn handle_events(&mut self, cx: &mut Context<'_>, shared: &Shared) -> bool { + for _ in 0..IO_LOOP_BOUND { + let (ch, event) = match self.events.poll_recv(cx) { + Poll::Ready(Some(x)) => x, + Poll::Ready(None) => unreachable!("EndpointInner owns one sender"), + Poll::Pending => { + return false; + } + }; + + if event.is_drained() { + self.recv_state.connections.senders.remove(&ch); + if self.is_idle() { + shared.idle.notify_waiters(); + } + } + let Some(event) = self.inner.handle_event(ch, event) else { + continue; + }; + // Ignoring errors from dropped connections that haven't yet been cleaned up + let _ = self + .recv_state + .connections + .senders + .get_mut(&ch) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + + true + } + + fn is_idle(&self) -> bool { + self.recv_state.connections.is_empty() && self.pending_accepts == 0 + } +} + +impl Drop for State { + fn drop(&mut self) { + for incoming in self.recv_state.incoming.drain(..) { + self.inner.ignore(incoming); + } + } +} + +fn respond( + transmit: proto::Transmit, + response_buffer: &[u8], + sender: &mut Pin>, +) { + // Send if there's kernel buffer space; otherwise, drop it + // + // As an endpoint-generated packet, we know this is an + // immediate, stateless response to an unconnected peer, + // one of: + // + // - A version negotiation response due to an unknown version + // - A `CLOSE` due to a malformed or unwanted connection attempt + // - A stateless reset due to an unrecognized connection + // - A `Retry` packet due to a connection attempt when + // `use_retry` is set + // + // In each case, a well-behaved peer can be trusted to retry a + // few times, which is guaranteed to produce the same response + // from us. Repeated failures might at worst cause a peer's new + // connection attempt to time out, which is acceptable if we're + // under such heavy load that there's never room for this code + // to transmit. This is morally equivalent to the packet getting + // lost due to congestion further along the link, which + // similarly relies on peer retries for recovery. + + // Copied from rust 1.85's std::task::Waker::noop() implementation for backwards compatibility + const NOOP: RawWaker = { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| NOOP, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, + ); + RawWaker::new(std::ptr::null(), &VTABLE) + }; + // SAFETY: Copied from rust stdlib, the NOOP waker is thread-safe and doesn't violate the RawWakerVTable contract, + // it doesn't access the data pointer at all. + let waker = unsafe { Waker::from_raw(NOOP) }; + let mut cx = Context::from_waker(&waker); + _ = sender.as_mut().poll_send( + &udp_transmit(&transmit, &response_buffer[..transmit.size]), + &mut cx, + ); +} + +#[inline] +fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { + match ecn { + udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, + udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, + udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + } +} + +#[derive(Debug)] +struct ConnectionSet { + /// Senders for communicating with the endpoint's connections + senders: FxHashMap>, + /// Stored to give out clones to new ConnectionInners + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + /// Set if the endpoint has been manually closed + close: Option<(VarInt, Bytes)>, +} + +impl ConnectionSet { + fn insert( + &mut self, + handle: ConnectionHandle, + conn: proto::Connection, + sender: Pin>, + driver_lost: bool, + runtime: Arc, + ) -> Connecting { + let (send, recv) = mpsc::unbounded_channel(); + if let Some((error_code, reason)) = &self.close { + send.send(ConnectionEvent::Close { + error_code: *error_code, + reason: reason.clone(), + }) + .unwrap(); + } + match driver_lost { + // Close the event channel before spawning the connection driver, so it cannot + // transmit a handshake after the endpoint driver has stopped. + true => drop(send), + false => { + self.senders.insert(handle, send); + } + } + Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + } + + fn is_empty(&self) -> bool { + self.senders.is_empty() + } +} + +fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { + match x { + SocketAddr::V6(x) => x, + SocketAddr::V4(x) => SocketAddrV6::new(x.ip().to_ipv6_mapped(), x.port(), 0, 0), + } +} + +pin_project! { + /// Future produced by [`Endpoint::accept`] + pub struct Accept<'a> { + endpoint: &'a Endpoint, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for Accept<'_> { + type Output = Option; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut endpoint = this.endpoint.inner.state.lock().unwrap(); + if endpoint.driver_lost { + return Poll::Ready(None); + } + if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // Release the mutex lock on endpoint so cloning it doesn't deadlock + drop(endpoint); + let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); + return Poll::Ready(Some(incoming)); + } + if endpoint.recv_state.connections.close.is_some() { + return Poll::Ready(None); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.endpoint.inner.shared.incoming.notified()), + } + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointRef(Arc); + +impl EndpointRef { + pub(crate) fn new( + socket: Box, + inner: proto::Endpoint, + ipv6: bool, + runtime: Arc, + ) -> Self { + let (sender, events) = mpsc::unbounded_channel(); + let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); + let sender = socket.create_sender(); + Self(Arc::new(EndpointInner { + shared: Shared { + incoming: Notify::new(), + idle: Notify::new(), + ref_count: AtomicUsize::new(0), + }, + state: Mutex::new(State { + socket, + sender, + prev_socket: None, + inner, + ipv6, + events, + driver: None, + driver_lost: false, + recv_state, + runtime, + stats: EndpointStats::default(), + default_client_config: None, + pending_accepts: 0, + }), + })) + } +} + +impl Clone for EndpointRef { + fn clone(&self) -> Self { + self.0.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for EndpointRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let endpoint = &mut *self.0.state.lock().unwrap(); + // If the driver is about to be on its own, ensure it can shut down if the last + // connection is gone. + if let Some(task) = endpoint.driver.take() { + task.wake(); + } + } +} + +impl std::ops::Deref for EndpointRef { + type Target = EndpointInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// State directly involved in handling incoming packets +struct RecvState { + incoming: VecDeque, + connections: ConnectionSet, + recv_buf: Box<[u8]>, + /// Reused buffer that received datagrams are copied into and split into owned slices. + datagrams: BytesMut, + recv_limiter: WorkLimiter, +} + +impl RecvState { + fn new( + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + max_receive_segments: usize, + endpoint: &proto::Endpoint, + ) -> Self { + let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) + as usize + * max_receive_segments; + let recv_buf = vec![0; datagram_capacity * BATCH_SIZE]; + Self { + connections: ConnectionSet { + senders: FxHashMap::default(), + sender, + close: None, + }, + incoming: VecDeque::new(), + recv_buf: recv_buf.into(), + datagrams: BytesMut::with_capacity(datagram_capacity), + recv_limiter: WorkLimiter::new(RECV_TIME_BOUND), + } + } + + fn poll_socket( + &mut self, + cx: &mut Context<'_>, + endpoint: &mut proto::Endpoint, + socket: &mut dyn AsyncUdpSocket, + sender: &mut Pin>, + runtime: &dyn Runtime, + now: Instant, + ) -> Result { + let mut received_connection_packet = false; + let mut metas = [RecvMeta::default(); BATCH_SIZE]; + let mut iovs: [IoSliceMut<'_>; BATCH_SIZE] = { + let mut bufs = self + .recv_buf + .chunks_mut(self.recv_buf.len() / BATCH_SIZE) + .map(IoSliceMut::new); + + // expect() safe as self.recv_buf is chunked into BATCH_SIZE items + // and iovs will be of size BATCH_SIZE, thus from_fn is called + // exactly BATCH_SIZE times. + std::array::from_fn(|_| bufs.next().expect("BATCH_SIZE elements")) + }; + loop { + match socket.poll_recv(cx, &mut iovs, &mut metas) { + Poll::Ready(Ok(msgs)) => { + self.recv_limiter.record_work(msgs); + // Copy the received batch into `datagrams` and split each datagram off as + // an owned slice. We can't read directly into `datagrams` and drop + // `recv_buf`: once a slice has been handed to a connection the allocation + // is shared, so the next `reserve` must allocate fresh backing memory. + // Reading in place would therefore mean reserving the worst-case iovec + // envelope before every poll and reallocating it whenever a prior datagram + // is still in flight. `recv_buf` stays an allocate-once scratch buffer, and + // we reserve only the bytes actually received. + let batch_len = metas.iter().take(msgs).map(|meta| meta.len).sum(); + self.datagrams.reserve(batch_len); + for (meta, buf) in metas.iter().zip(iovs.iter()).take(msgs) { + self.datagrams.extend_from_slice(&buf[..meta.len]); + } + for meta in metas.iter().take(msgs) { + let mut data = self.datagrams.split_to(meta.len); + while !data.is_empty() { + let buf = data.split_to(meta.stride.min(data.len())); + let mut response_buffer = Vec::new(); + match endpoint.handle( + now, + meta.addr, + meta.dst_ip, + meta.ecn.map(proto_ecn), + buf, + &mut response_buffer, + ) { + Some(DatagramEvent::NewConnection(incoming)) => { + if self.connections.close.is_none() { + self.incoming.push_back(incoming); + } else { + let transmit = + endpoint.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, sender); + } + } + Some(DatagramEvent::ConnectionEvent(handle, event)) => { + // Ignoring errors from dropped connections that haven't yet been cleaned up + received_connection_packet = true; + let _ = self + .connections + .senders + .get_mut(&handle) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + Some(DatagramEvent::Response(transmit)) => { + respond(transmit, &response_buffer, sender); + } + None => {} + } + } + } + } + Poll::Pending => { + return Ok(PollProgress { + received_connection_packet, + keep_going: false, + }); + } + // Ignore ECONNRESET as it's undefined in QUIC and may be injected by an + // attacker + Poll::Ready(Err(e)) if e.kind() == io::ErrorKind::ConnectionReset => { + continue; + } + // Ignore EMSGSIZE as we're currently not handling ICMPv4 Fragmentation Needed + // and ICMPv6 Packet Too Big (PTB) messages since they cannot be authenticated, + // and Datagram Packetization Layer Path MTU Discovery (DPLPMTUD) works without + // it anyways. + Poll::Ready(Err(e)) if is_msg_size_err(&e) => { + continue; + } + Poll::Ready(Err(e)) => { + return Err(e); + } + } + if !self.recv_limiter.allow_work(|| runtime.now()) { + return Ok(PollProgress { + received_connection_packet, + keep_going: true, + }); + } + } + } +} + +impl fmt::Debug for RecvState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RecvState") + .field("incoming", &self.incoming) + .field("connections", &self.connections) + // recv_buf too large + .field("recv_limiter", &self.recv_limiter) + .finish_non_exhaustive() + } +} + +#[derive(Default)] +struct PollProgress { + /// Whether a datagram was routed to an existing connection + received_connection_packet: bool, + /// Whether datagram handling was interrupted early by the work limiter for fairness + keep_going: bool, +} diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs new file mode 100644 index 0000000000..b5d60a08ec --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -0,0 +1,152 @@ +use std::{ + future::{Future, IntoFuture}, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use proto::{ConnectionError, ConnectionId, ServerConfig}; +use thiserror::Error; + +use crate::{ + connection::{Connecting, Connection}, + endpoint::EndpointRef, +}; + +/// An incoming connection for which the server has not yet begun its part of the handshake +#[derive(Debug)] +pub struct Incoming(Option); + +impl Incoming { + pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + Self(Some(State { inner, endpoint })) + } + + /// Attempt to accept this incoming connection (an error may still occur) + pub fn accept(mut self) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, None) + } + + /// Accept this incoming connection using a custom configuration + /// + /// See [`accept()`][Incoming::accept] for more details. + pub fn accept_with( + mut self, + server_config: Arc, + ) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, Some(server_config)) + } + + /// Reject this incoming connection attempt + pub fn refuse(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.refuse(state.inner); + } + + /// Respond with a retry packet, requiring the client to retry with address validation + /// + /// Errors if `may_retry()` is false. + pub fn retry(mut self) -> Result<(), RetryError> { + let state = self.0.take().unwrap(); + state.endpoint.retry(state.inner).map_err(|e| { + RetryError(Box::new(Self(Some(State { + inner: e.into_incoming(), + endpoint: state.endpoint, + })))) + }) + } + + /// Ignore this incoming connection attempt, not sending any packet in response + pub fn ignore(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.ignore(state.inner); + } + + /// The local IP address which was used when the peer established the connection + pub fn local_ip(&self) -> Option { + self.0.as_ref().unwrap().inner.local_ip() + } + + /// The peer's UDP address + pub fn remote_address(&self) -> SocketAddr { + self.0.as_ref().unwrap().inner.remote_address() + } + + /// Whether the socket address that is initiating this connection has been validated + /// + /// This means that the sender of the initial packet has proved that they can receive traffic + /// sent to `self.remote_address()`. + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn remote_address_validated(&self) -> bool { + self.0.as_ref().unwrap().inner.remote_address_validated() + } + + /// Whether it is legal to respond with a retry packet + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn may_retry(&self) -> bool { + self.0.as_ref().unwrap().inner.may_retry() + } + + /// The original destination CID when initiating the connection + pub fn orig_dst_cid(&self) -> ConnectionId { + self.0.as_ref().unwrap().inner.orig_dst_cid() + } +} + +impl Drop for Incoming { + fn drop(&mut self) { + // Implicit reject, similar to Connection's implicit close + if let Some(state) = self.0.take() { + state.endpoint.refuse(state.inner); + } + } +} + +#[derive(Debug)] +struct State { + inner: proto::Incoming, + endpoint: EndpointRef, +} + +/// Error for attempting to retry an [`Incoming`] which already bears a token from a previous retry +#[derive(Debug, Error)] +#[error("retry() with validated Incoming")] +pub struct RetryError(Box); + +impl RetryError { + /// Get the [`Incoming`] + pub fn into_incoming(self) -> Incoming { + *self.0 + } +} + +/// Basic adapter to let [`Incoming`] be `await`-ed like a [`Connecting`] +#[derive(Debug)] +pub struct IncomingFuture(Result); + +impl Future for IncomingFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match &mut self.0 { + Ok(connecting) => Pin::new(connecting).poll(cx), + Err(e) => Poll::Ready(Err(e.clone())), + } + } +} + +impl IntoFuture for Incoming { + type Output = Result; + type IntoFuture = IncomingFuture; + + fn into_future(self) -> Self::IntoFuture { + IncomingFuture(self.accept()) + } +} diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs new file mode 100644 index 0000000000..649801fff8 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -0,0 +1,134 @@ +//! QUIC transport protocol implementation +//! +//! [QUIC](https://en.wikipedia.org/wiki/QUIC) is a modern transport protocol addressing +//! shortcomings of TCP, such as head-of-line blocking, poor security, slow handshakes, and +//! inefficient congestion control. This crate provides a portable userspace implementation. It +//! builds on top of quinn-proto, which implements protocol logic independent of any particular +//! runtime. +//! +//! The entry point of this crate is the [`Endpoint`]. +//! +//! # About QUIC +//! +//! A QUIC connection is an association between two endpoints. The endpoint which initiates the +//! connection is termed the client, and the endpoint which accepts it is termed the server. A +//! single endpoint may function as both client and server for different connections, for example +//! in a peer-to-peer application. To communicate application data, each endpoint may open streams +//! up to a limit dictated by its peer. Typically, that limit is increased as old streams are +//! finished. +//! +//! Streams may be unidirectional or bidirectional, and are cheap to create and disposable. For +//! example, a traditionally datagram-oriented application could use a new stream for every +//! message it wants to send, no longer needing to worry about MTUs. Bidirectional streams behave +//! much like a traditional TCP connection, and are useful for sending messages that have an +//! immediate response, such as an HTTP request. Stream data is delivered reliably, and there is no +//! ordering enforced between data on different streams. +//! +//! By avoiding head-of-line blocking and providing unified congestion control across all streams +//! of a connection, QUIC is able to provide higher throughput and lower latency than one or +//! multiple TCP connections between the same two hosts, while providing more useful behavior than +//! raw UDP sockets. +//! +//! Quinn also exposes unreliable datagrams, which are a low-level primitive preferred when +//! automatic fragmentation and retransmission of certain data is not desired. +//! +//! QUIC uses encryption and identity verification built directly on TLS 1.3. Just as with a TLS +//! server, it is useful for a QUIC server to be identified by a certificate signed by a trusted +//! authority. If this is infeasible--for example, if servers are short-lived or not associated +//! with a domain name--then as with TLS, self-signed certificates can be used to provide +//! encryption alone. +#![warn(missing_docs)] + +use std::pin::Pin; + +mod connection; +mod endpoint; +mod incoming; +mod mutex; +mod recv_stream; +mod runtime; +mod send_stream; +mod work_limiter; + +#[cfg(not(wasm_browser))] +pub(crate) use std::time::{Duration, Instant}; +#[cfg(wasm_browser)] +pub(crate) use web_time::{Duration, Instant}; + +#[cfg(feature = "bloom")] +pub use proto::BloomTokenLog; +pub use proto::{ + AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, + ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, + ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, + InvalidCid, MtuDiscoveryConfig, NoneTokenLog, NoneTokenStore, PathStats, ServerConfig, Side, + StdSystemTime, StreamId, TimeSource, TokenLog, TokenMemoryCache, TokenReuseError, TokenStore, + Transmit, TransportConfig, TransportErrorCode, UdpStats, ValidationTokenConfig, VarInt, + VarIntBoundsExceeded, Written, congestion, crypto, +}; +#[cfg(feature = "qlog")] +pub use proto::{QlogConfig, QlogStream}; +#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +pub use rustls; +pub use udp; + +pub use crate::connection::{ + AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, + RemoteAddressWatcher, SendDatagram, SendDatagramError, +}; +pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; +#[cfg(feature = "runtime-smol")] +pub use crate::runtime::SmolRuntime; +#[cfg(feature = "runtime-tokio")] +pub use crate::runtime::TokioRuntime; +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +pub use crate::runtime::default_runtime; +pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; +pub use crate::send_stream::{SendStream, StoppedError, WriteError}; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +enum ConnectionEvent { + Close { + error_code: VarInt, + reason: bytes::Bytes, + }, + Proto(proto::ConnectionEvent), + Rebind(Pin>), +} + +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { + udp::Transmit { + destination: t.destination, + ecn: t.ecn.map(udp_ecn), + contents: buffer, + segment_size: t.segment_size, + src_ip: t.src_ip, + } +} + +fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { + match ecn { + EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + } +} + +/// Maximum number of datagrams processed in send/recv calls to make before moving on to other processing +/// +/// This helps ensure we don't starve anything when the CPU is slower than the link. +/// Value is selected by picking a low number which didn't degrade throughput in benchmarks. +const IO_LOOP_BOUND: usize = 160; + +/// The maximum amount of time that should be spent in `recvmsg()` calls per endpoint iteration +/// +/// 50us are chosen so that an endpoint iteration with a 50us sendmsg limit blocks +/// the runtime for a maximum of about 100us. +/// Going much lower does not yield any noticeable difference, since a single `recvmmsg` +/// batch of size 32 was observed to take 30us on some systems. +const RECV_TIME_BOUND: Duration = Duration::from_micros(50); diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs new file mode 100644 index 0000000000..91962cb072 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -0,0 +1,163 @@ +use std::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +#[cfg(feature = "lock_tracking")] +mod tracking { + use super::*; + use crate::{Duration, Instant}; + use std::collections::VecDeque; + use tracing::warn; + + #[derive(Debug)] + struct Inner { + last_lock_owner: VecDeque<(&'static str, Duration)>, + value: T, + } + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + pub(crate) struct Mutex { + inner: std::sync::Mutex>, + } + + impl Debug for Mutex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.inner, f) + } + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(Inner { + last_lock_owner: VecDeque::new(), + value, + }), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { + // We don't bother dispatching through Runtime::now because they're pure performance + // diagnostics. + let now = Instant::now(); + let guard = self.inner.lock().unwrap(); + + let lock_time = Instant::now(); + let elapsed = lock_time.duration_since(now); + + if elapsed > Duration::from_millis(1) { + warn!( + "Locking the connection for {} took {:?}. Last owners: {:?}", + purpose, elapsed, guard.last_lock_owner + ); + } + + MutexGuard { + guard, + start_time: lock_time, + purpose, + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, Inner>, + start_time: Instant, + purpose: &'static str, + } + + impl Drop for MutexGuard<'_, T> { + fn drop(&mut self) { + if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { + self.guard.last_lock_owner.pop_back(); + } + + let duration = self.start_time.elapsed(); + + if duration > Duration::from_millis(1) { + warn!( + "Utilizing the connection for {} took {:?}", + self.purpose, duration + ); + } + + self.guard + .last_lock_owner + .push_front((self.purpose, duration)); + } + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard.value + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard.value + } + } + + const MAX_LOCK_OWNERS: usize = 20; +} + +#[cfg(feature = "lock_tracking")] +pub(crate) use tracking::Mutex; + +#[cfg(not(feature = "lock_tracking"))] +mod non_tracking { + use super::*; + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + #[derive(Debug)] + pub(crate) struct Mutex { + inner: std::sync::Mutex, + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(value), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, _purpose: &'static str) -> MutexGuard<'_, T> { + MutexGuard { + guard: self.inner.lock().unwrap(), + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, T>, + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.guard.deref() + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.deref_mut() + } + } +} + +#[cfg(not(feature = "lock_tracking"))] +pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs new file mode 100644 index 0000000000..d3548a1a33 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -0,0 +1,823 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use bytes::Bytes; +use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use thiserror::Error; +use tokio::io::ReadBuf; + +use crate::{VarInt, connection::ConnectionRef}; + +/// A stream that can only be used to receive data +/// +/// `stop(0)` is implicitly called on drop unless: +/// - A variant of [`ReadError`] has been yielded by a read call +/// - [`stop()`] was called explicitly +/// +/// # Cancellation +/// +/// A `read` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready cannot lead to loss of stream data. This is true of methods which succeed immediately when +/// any progress is made, and is not true of methods which might need to perform multiple reads +/// internally before succeeding. Each `read` method documents whether it is cancel-safe. +/// +/// # Common issues +/// +/// ## Data never received on a locally-opened stream +/// +/// Peers are not notified of streams until they or a later-numbered stream are used to send +/// data. If a bidirectional stream is locally opened but never used to send, then the peer may +/// never see it. Application protocols should always arrange for the endpoint which will first +/// transmit on a stream to be the endpoint responsible for opening it. +/// +/// ## Data never received on a remotely-opened stream +/// +/// Verify that the stream you are receiving is the same one that the server is sending on, e.g. by +/// logging the [`id`] of each. Streams are always accepted in the same order as they are created, +/// i.e. ascending order by [`StreamId`]. For example, even if a sender first transmits on +/// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver +/// will be bidirectional stream 0. +/// +/// [`ReadError`]: crate::ReadError +/// [`stop()`]: RecvStream::stop +/// [`SendStream::finish`]: crate::SendStream::finish +/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`id`]: RecvStream::id +/// [`Connection::accept_bi`]: crate::Connection::accept_bi +#[derive(Debug)] +pub struct RecvStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, + all_data_read: bool, + reset: Option, +} + +impl RecvStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + all_data_read: false, + reset: None, + } + } + + /// Read data contiguously from the stream. + /// + /// Yields the number of bytes read into `buf` on success, or `None` if the stream was finished. + /// + /// This operation is cancel-safe. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + Read { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Read an exact number of bytes contiguously from the stream. + /// + /// See [`read()`] for details. This operation is *not* cancel-safe. + /// + /// [`read()`]: RecvStream::read + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + ReadExact { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Attempts to read from the stream into the provided buffer + /// + /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this + /// returns zero bytes read (and `buf` has a non-zero length), that indicates that the remote + /// side has [`finish`]ed the stream and the local side has already read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read( + &mut self, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(self.poll_read_buf(cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } + + /// Attempts to read from the stream into the provided buffer, which may be uninitialized + /// + /// On success, returns `Poll::Ready(Ok(()))` and places data into the unfilled portion of + /// `buf`. If this does not write any bytes to `buf` (and `buf.remaining()` is non-zero), that + /// indicates that the remote side has [`finish`]ed the stream and the local side has already + /// read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read_buf( + &mut self, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = false; + loop { + if buf.remaining() == 0 { + // We know `read` is `true` because `buf.remaining()` was not 0 before + return ReadStatus::Readable(()); + } + + match chunks.next(buf.remaining()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read = true; + } + res => return (if read { Some(()) } else { None }, res.err()).into(), + } + } + }) + .map(|res| res.map(|_| ())) + } + + /// Read the next segment of data + /// + /// Yields `None` if the stream was finished. Otherwise, yields a segment of data and its + /// offset in the stream. If `ordered` is `true`, the chunk's offset will be immediately after + /// the last data yielded by `read()` or `read_chunk()`. If `ordered` is `false`, segments may + /// be received in any order, and the `Chunk`'s `offset` field can be used to determine + /// ordering in the caller. Unordered reads are less prone to head-of-line blocking within a + /// stream, but require the application to manage reassembling the original data. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries do not correspond + /// to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunk( + &mut self, + max_length: usize, + ordered: bool, + ) -> Result, ReadError> { + ReadChunk { + stream: self, + max_length, + ordered, + } + .await + } + + /// Attempts to read a chunk from the stream. + /// + /// On success, returns `Poll::Ready(Ok(Some(chunk)))`. If `Poll::Ready(Ok(None))` + /// is returned, it implies that EOF has been reached. + /// + /// If no data is available for reading, the method returns `Poll::Pending` + /// and arranges for the current task (via cx.waker()) to receive a notification + /// when the stream becomes readable or is closed. + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max_length: usize, + ordered: bool, + ) -> Poll, ReadError>> { + self.poll_read_generic(cx, ordered, |chunks| match chunks.next(max_length) { + Ok(Some(chunk)) => ReadStatus::Readable(chunk), + res => (None, res.err()).into(), + }) + } + + /// Read the next segments of data + /// + /// Fills `bufs` with the segments of data beginning immediately after the + /// last data yielded by `read` or `read_chunk`, or `None` if the stream was + /// finished. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries + /// do not correspond to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunks(&mut self, bufs: &mut [Bytes]) -> Result, ReadError> { + ReadChunks { stream: self, bufs }.await + } + + /// Foundation of [`Self::read_chunks`] + fn poll_read_chunks( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [Bytes], + ) -> Poll, ReadError>> { + if bufs.is_empty() { + return Poll::Ready(Ok(Some(0))); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = 0; + loop { + if read >= bufs.len() { + // We know `read > 0` because `bufs` cannot be empty here + return ReadStatus::Readable(read); + } + + match chunks.next(usize::MAX) { + Ok(Some(chunk)) => { + bufs[read] = chunk.bytes; + read += 1; + } + res => return (if read == 0 { None } else { Some(read) }, res.err()).into(), + } + } + }) + } + + /// Convenience method to read all remaining data into a buffer + /// + /// Fails with [`ReadToEndError::TooLong`] on reading more than `size_limit` bytes, discarding + /// all data read. Uses unordered reads to be more efficient than using `AsyncRead` would + /// allow. `size_limit` should be set to limit worst-case memory use. + /// + /// If unordered reads have already been made, the resulting buffer may have gaps containing + /// arbitrary data. + /// + /// This operation is *not* cancel-safe. + /// + /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + ReadToEnd { + stream: self, + buffer: ReadToEndBuffer::new(size_limit), + } + .await + } + + /// Stop accepting data + /// + /// Discards unread data and notifies the peer to stop transmitting. Once stopped, further + /// attempts to operate on a stream will yield `ClosedStream` errors. + pub fn stop(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("RecvStream::stop"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.recv_stream(self.stream).stop(error_code)?; + conn.wake(); + self.all_data_read = true; + // Clean up shared state that might be left over from a cancalled read + // operation, so `drop` doesn't have to + conn.blocked_readers.remove(&self.stream); + Ok(()) + } + + /// Check if this stream predates completion of the handshake on an incoming connection. + /// + /// True only if the stream was accepted before the handshake completed, which is only possible + /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// client chose to send 0-RTT data. + /// + /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application + /// data may be a replay attack. To guard against this, applications should not execute + /// non-idempotent operations until + /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + pub fn is_0rtt(&self) -> bool { + self.is_0rtt + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Completes when the stream has been reset by the peer or otherwise closed + /// + /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` + /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was + /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// which it is no longer meaningful for the stream to be reset. + /// + /// This operation is cancel-safe. + pub async fn received_reset(&mut self) -> Result, ResetError> { + poll_fn(|cx| { + let mut conn = self.conn.state.lock("RecvStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Poll::Ready(Err(ResetError::ZeroRttRejected)); + } + + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + + match conn.inner.recv_stream(self.stream).received_reset() { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(error_code)) => { + // Stream state has just now been freed, so the connection may need to issue new + // stream ID flow control credit + conn.wake(); + Poll::Ready(Ok(Some(error_code))) + } + Ok(None) => { + if let Some(e) = &conn.error { + return Poll::Ready(Err(e.clone().into())); + } + // Resets always notify readers, since a reset is an immediate read error. We + // could introduce a dedicated channel to reduce the risk of spurious wakeups, + // but that increased complexity is probably not justified, as an application + // that is expecting a reset is not likely to receive large amounts of data. + conn.blocked_readers.insert(self.stream, cx.waker().clone()); + Poll::Pending + } + } + }) + .await + } + + /// Handle common logic related to reading out of a receive stream + /// + /// This takes an `FnMut` closure that takes care of the actual reading process, matching + /// the detailed read semantics for the calling function with a particular return type. + /// The closure can read from the passed `&mut Chunks` and has to return the status after + /// reading: the amount of data read, and the status after the final read call. + fn poll_read_generic( + &mut self, + cx: &mut Context<'_>, + ordered: bool, + mut read_fn: T, + ) -> Poll, ReadError>> + where + T: FnMut(&mut Chunks<'_>) -> ReadStatus, + { + use proto::ReadError::*; + use std::collections::hash_map::Entry; + + if self.all_data_read { + return Poll::Ready(Ok(None)); + } + + let mut conn = self.conn.state.lock("RecvStream::poll_read"); + if self.is_0rtt { + conn.check_0rtt().map_err(|()| ReadError::ZeroRttRejected)?; + } + + // If we stored an error during a previous call, return it now. This can happen if a + // `read_fn` both wants to return data and also returns an error in its final stream status. + let status = match self.reset { + Some(code) => ReadStatus::Failed(None, Reset(code)), + None => { + let mut recv = conn.inner.recv_stream(self.stream); + let mut chunks = recv.read(ordered)?; + let status = read_fn(&mut chunks); + if chunks.finalize().should_transmit() { + conn.wake(); + } + status + } + }; + + match status { + ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), + ReadStatus::Finished(read) => { + self.all_data_read = true; + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Ok(read)) + } + ReadStatus::Failed(read, Blocked) => match read { + Some(val) => Poll::Ready(Ok(Some(val))), + None => { + if let Some(x) = &conn.error { + return Poll::Ready(Err(ReadError::ConnectionLost(x.clone()))); + } + match conn.blocked_readers.entry(self.stream) { + Entry::Occupied(mut entry) => { + entry.get_mut().clone_from(cx.waker()); + } + Entry::Vacant(entry) => { + entry.insert(cx.waker().clone()); + } + } + Poll::Pending + } + }, + ReadStatus::Failed(read, Reset(error_code)) => match read { + None => { + self.all_data_read = true; + self.reset = Some(error_code); + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Err(ReadError::Reset(error_code))) + } + done => { + self.reset = Some(error_code); + Poll::Ready(Ok(done)) + } + }, + } + } +} + +enum ReadStatus { + Readable(T), + Finished(Option), + Failed(Option, proto::ReadError), +} + +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { + match status { + (read, None) => Self::Finished(read), + (read, Some(e)) => Self::Failed(read, e), + } + } +} + +/// Future produced by [`RecvStream::read_to_end()`]. +/// +/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +struct ReadToEnd<'a> { + stream: &'a mut RecvStream, + buffer: ReadToEndBuffer, +} + +impl Future for ReadToEnd<'_> { + type Output = Result, ReadToEndError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + loop { + match ready!(self.stream.poll_read_chunk(cx, usize::MAX, false))? { + Some(chunk) => self.buffer.push(chunk)?, + None => return Poll::Ready(Ok(self.buffer.finish())), + } + } + } +} + +struct ReadToEndBuffer { + read: Vec<(Bytes, u64)>, + start: u64, + end: u64, + size_limit: usize, +} + +impl ReadToEndBuffer { + fn new(size_limit: usize) -> Self { + Self { + read: Vec::new(), + start: u64::MAX, + end: 0, + size_limit, + } + } + + fn push(&mut self, chunk: Chunk) -> Result<(), ReadToEndError> { + self.start = self.start.min(chunk.offset); + self.end = self.end.max(chunk.bytes.len() as u64 + chunk.offset); + if (self.end - self.start) > self.size_limit as u64 { + return Err(ReadToEndError::TooLong); + } + self.read.push((chunk.bytes, chunk.offset)); + Ok(()) + } + + fn finish(&mut self) -> Vec { + if self.end == 0 { + return Vec::new(); + } + let mut buffer = vec![0; (self.end - self.start) as usize]; + for (data, offset) in self.read.drain(..) { + let offset = (offset - self.start) as usize; + buffer[offset..offset + data.len()].copy_from_slice(&data); + } + buffer + } +} + +/// Errors from [`RecvStream::read_to_end`] +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadToEndError { + /// An error occurred during reading + #[error("read error: {0}")] + Read(#[from] ReadError), + /// The stream is larger than the user-supplied limit + #[error("stream too long")] + TooLong, +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + ready!(Self::poll_read_buf(self.get_mut(), cx, buf))?; + Poll::Ready(Ok(())) + } +} + +impl Drop for RecvStream { + fn drop(&mut self) { + if self.all_data_read { + debug_assert!( + !self + .conn + .state + .lock("RecvStream:drop") + .blocked_readers + .contains_key(&self.stream), + "Stream {} should not have a blocked reader when all data read is true", + self.stream + ); + return; + } + let mut conn = self.conn.state.lock("RecvStream::drop"); + + // clean up any previously registered wakers + conn.blocked_readers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + + // Ignore ClosedStream errors + let _ = conn.inner.recv_stream(self.stream).stop(0u32.into()); + conn.wake(); + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadError { + /// The peer abandoned transmitting data on this stream + /// + /// Carries an application-defined error code. + #[error("stream reset by peer: error {0}")] + Reset(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been stopped, finished, or reset + #[error("closed stream")] + ClosedStream, + /// Attempted an ordered read following an unordered read + /// + /// Performing an unordered read allows discontinuities to arise in the receive buffer of a + /// stream which cannot be recovered, making further ordered reads impossible. + #[error("ordered read after unordered read")] + IllegalOrderedRead, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for ReadError { + fn from(e: ReadableError) -> Self { + match e { + ReadableError::ClosedStream => Self::ClosedStream, + ReadableError::IllegalOrderedRead => Self::IllegalOrderedRead, + } + } +} + +impl From for ReadError { + fn from(e: ResetError) -> Self { + match e { + ResetError::ConnectionLost(e) => Self::ConnectionLost(e), + ResetError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: ReadError) -> Self { + use ReadError::*; + let kind = match x { + Reset { .. } | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + IllegalOrderedRead => io::ErrorKind::InvalidInput, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while waiting for a stream to be reset +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ResetError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: ResetError) -> Self { + use ResetError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Future produced by [`RecvStream::read()`]. +/// +/// [`RecvStream::read()`]: crate::RecvStream::read +struct Read<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for Read<'_> { + type Output = Result, ReadError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + match this.buf.filled().len() { + 0 if this.buf.capacity() != 0 => Poll::Ready(Ok(None)), + n => Poll::Ready(Ok(Some(n))), + } + } +} + +/// Future produced by [`RecvStream::read_exact()`]. +/// +/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +struct ReadExact<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for ReadExact<'_> { + type Output = Result<(), ReadExactError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let mut remaining = this.buf.remaining(); + while remaining > 0 { + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + let new = this.buf.remaining(); + if new == remaining { + return Poll::Ready(Err(ReadExactError::FinishedEarly(this.buf.filled().len()))); + } + remaining = new; + } + Poll::Ready(Ok(())) + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadExactError { + /// The stream finished before all bytes were read + #[error("stream finished early ({0} bytes read)")] + FinishedEarly(usize), + /// A read error occurred + #[error(transparent)] + ReadError(#[from] ReadError), +} + +/// Future produced by [`RecvStream::read_chunk()`]. +/// +/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +struct ReadChunk<'a> { + stream: &'a mut RecvStream, + max_length: usize, + ordered: bool, +} + +impl Future for ReadChunk<'_> { + type Output = Result, ReadError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (max_length, ordered) = (self.max_length, self.ordered); + self.stream.poll_read_chunk(cx, max_length, ordered) + } +} + +/// Future produced by [`RecvStream::read_chunks()`]. +/// +/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +struct ReadChunks<'a> { + stream: &'a mut RecvStream, + bufs: &'a mut [Bytes], +} + +impl Future for ReadChunks<'_> { + type Output = Result, ReadError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + this.stream.poll_read_chunks(cx, this.bufs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_to_end_reordered_limit() { + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 4, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 0, + bytes: Bytes::from_static(b"abcd") + }), + Err(ReadToEndError::TooLong) + ); + assert_eq!(buffer.read.len(), 1); + } + + #[test] + fn read_to_end_reordered_exact_limit_after_prefix() { + let mut buffer = ReadToEndBuffer::new(8); + // A previously consumed prefix must not count against the remaining-data limit. + buffer + .push(Chunk { + offset: 104, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 100, + bytes: Bytes::from_static(b"abcd"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"abcdefgh"); + } + + #[test] + fn read_to_end_gaps_and_empty() { + assert!(ReadToEndBuffer::new(0).finish().is_empty()); + let mut buffer = ReadToEndBuffer::new(5); + buffer + .push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"a\0\0\0e"); + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e") + }), + Err(ReadToEndError::TooLong) + ); + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs new file mode 100644 index 0000000000..9898a0e879 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -0,0 +1,330 @@ +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +use std::sync::Arc; +use std::{ + fmt::{self, Debug}, + future::Future, + io::{self, IoSliceMut}, + net::SocketAddr, + pin::Pin, + task::{self, Context, Poll}, + time::Duration, +}; + +use udp::{RecvMeta, Transmit}; + +use crate::Instant; + +/// Abstracts I/O and timer operations for runtime independence +pub trait Runtime: Send + Sync + Debug + 'static { + /// Construct a timer that will expire at `i` + fn new_timer(&self, i: Instant) -> Pin>; + /// Drive `future` to completion in the background + #[track_caller] + fn spawn(&self, future: Pin + Send>>); + /// Convert `t` into the socket type used by this runtime + #[cfg(not(wasm_browser))] + fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; + /// Look up the current time + /// + /// Allows simulating the flow of time for testing. + fn now(&self) -> Instant { + Instant::now() + } +} + +/// Abstract implementation of an async timer for runtime independence +pub trait AsyncTimer: Send + Debug + 'static { + /// Update the timer to expire at `i` + fn reset(self: Pin<&mut Self>, i: Instant); + /// Check whether the timer has expired, and register to be woken if not + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; +} + +/// Abstract implementation of a UDP socket for runtime independence +pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { + /// Create a [`UdpSender`] that can register a single task for write-readiness notifications + /// and send a transmit, if ready. + /// + /// A `poll_send` method on a single object can usually store only one [`Waker`] at a time, + /// i.e. allow at most one caller to wait for an event. This method allows any number of + /// interested tasks to construct their own [`UdpSender`] object. They can all then wait for the + /// same event and be notified concurrently, because each [`UdpSender`] can store a separate + /// [`Waker`]. + /// + /// [`Waker`]: std::task::Waker + fn create_sender(&self) -> Pin>; + + /// Receive UDP datagrams, or register to be woken if receiving may succeed in the future + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll>; + + /// Look up the local IP address and port used by this socket + fn local_addr(&self) -> io::Result; + + /// Maximum number of datagrams that might be described by a single [`RecvMeta`] + fn max_receive_segments(&self) -> usize { + 1 + } + + /// Whether datagrams might get fragmented into multiple parts + /// + /// Sockets should prevent this for best performance. See e.g. the `IPV6_DONTFRAG` socket + /// option. + fn may_fragment(&self) -> bool { + true + } +} + +/// An object for asynchronously writing to an associated [`AsyncUdpSocket`]. +/// +/// Any number of [`UdpSender`]s may exist for a single [`AsyncUdpSocket`]. Each [`UdpSender`] is +/// responsible for notifying at most one task for send readiness. +pub trait UdpSender: Send + Sync + Debug + 'static { + /// Send a UDP datagram, or register to be woken if sending may succeed in the future. + /// + /// Usually implementations of this will poll the socket for writability before trying to + /// write to them, and retry both if writing fails. + /// + /// Quinn will create multiple [`UdpSender`]s, one for each task it's using it from. Thus it's + /// important to poll the underlying socket in a way that doesn't overwrite wakers. + /// + /// A single [`UdpSender`] will be re-used, even if `poll_send` returns `Poll::Ready` once, + /// unlike [`Future::poll`], so calling it again after readiness should not panic. + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll>; + + /// Maximum number of datagrams that a [`Transmit`] may encode. + fn max_transmit_segments(&self) -> usize { + 1 + } +} + +pin_project_lite::pin_project! { + /// A helper for constructing [`UdpSender`]s from an underlying `Socket` type. + /// + /// This struct implements [`UdpSender`] if `MakeWritableFn` produces a `WritableFut`. + /// + /// Also serves as a trick, since `WritableFut` doesn't need to be a named future, + /// it can be an anonymous async block, as long as `MakeWritableFn` produces that + /// anonymous async block type. + /// + /// The `UdpSenderHelper` generic type parameters don't need to named, as it will be + /// used in its dyn-compatible form as a `Pin>`. + struct UdpSenderHelper { + socket: Socket, + last_send_error: Option, + make_writable_fut_fn: MakeWritableFutFn, + #[pin] + writable_fut: Option, + } +} + +impl Debug + for UdpSenderHelper +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("UdpSender") + } +} + +impl + UdpSenderHelper +{ + /// Create helper that implements [`UdpSender`] from a socket. + /// + /// Additionally you need to provide what is essentially an async function + /// that resolves once the socket is write-ready. + /// + /// See also the bounds on this struct's [`UdpSender`] implementation. + #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] + fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { + Self { + socket: inner, + last_send_error: None, + make_writable_fut_fn: make_fut, + writable_fut: None, + } + } +} + +impl UdpSender + for UdpSenderHelper +where + Socket: UdpSenderHelperSocket, + MakeWritableFutFn: Fn(&Socket) -> WritableFut + Send + Sync + 'static, + WritableFut: Future> + Send + Sync + 'static, +{ + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let mut this = self.project(); + loop { + if this.writable_fut.is_none() { + this.writable_fut + .set(Some((this.make_writable_fut_fn)(this.socket))); + } + // We're forced to `unwrap` here because `Fut` may be `!Unpin`, which means we can't safely + // obtain an `&mut WritableFut` after storing it in `self.writable_fut` when `self` is already behind `Pin`, + // and if we didn't store it then we wouldn't be able to keep it alive between + // `poll_send` calls. + let result = task::ready!(this.writable_fut.as_mut().as_pin_mut().unwrap().poll(cx)); + + // Polling an arbitrary `Future` after it becomes ready is a logic error, so arrange for + // a new `Future` to be created on the next call. + this.writable_fut.set(None); + + // If .writable() fails, propagate the error + result?; + + match this.socket.try_send(transmit) { + // We thought the socket was writable, but it wasn't, then retry so that either another + // `writable().await` call determines that the socket is indeed not writable and + // registers us for a wakeup, or the send succeeds if this really was just a + // transient failure. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => { + log_sendmsg_error(this.last_send_error, &e, transmit); + return Poll::Ready(Ok(())); + } + Ok(()) => return Poll::Ready(Ok(())), + } + } + } + + fn max_transmit_segments(&self) -> usize { + self.socket.max_transmit_segments() + } +} + +/// Limits I/O error logging to one message per minute. +const IO_ERROR_LOG_INTERVAL: Duration = Duration::from_secs(60); + +fn log_sendmsg_error( + last_send_error: &mut Option, + error: &io::Error, + transmit: &Transmit<'_>, +) { + #[cfg(unix)] + // Unix `EMSGSIZE` is expected for MTU probes. + if udp::is_msg_size_err(error) { + return; + } + + let now = Instant::now(); + if last_send_error + .is_some_and(|last| now.saturating_duration_since(last) <= IO_ERROR_LOG_INTERVAL) + { + return; + } + *last_send_error = Some(now); + + tracing::warn!( + "sendmsg error: {:?}, Transmit: {{ destination: {:?}, src_ip: {:?}, ecn: {:?}, len: {:?}, segment_size: {:?} }}", + error, + transmit.destination, + transmit.src_ip, + transmit.ecn, + transmit.contents.len(), + transmit.segment_size + ); +} + +/// Parts of the [`UdpSender`] trait that aren't asynchronous or require storing wakers. +/// +/// This trait is used by [`UdpSenderHelper`] to help construct [`UdpSender`]s. +trait UdpSenderHelperSocket: Send + Sync + 'static { + /// Try to send a transmit, if the socket happens to be write-ready. + /// + /// If not write-ready, this is allowed to return [`std::io::ErrorKind::WouldBlock`]. + /// + /// The [`UdpSenderHelper`] will use this to implement [`UdpSender::poll_send`]. + fn try_send(&self, transmit: &Transmit<'_>) -> io::Result<()>; + + /// See [`UdpSender::max_transmit_segments`]. + fn max_transmit_segments(&self) -> usize; +} + +/// Automatically select an appropriate runtime from those enabled at compile time +/// +/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, +/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is +/// returned. Otherwise, `None` is returned. +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +#[allow(clippy::needless_return)] // Be sure we return the right thing +pub fn default_runtime() -> Option> { + #[cfg(feature = "runtime-tokio")] + { + if ::tokio::runtime::Handle::try_current().is_ok() { + return Some(Arc::new(TokioRuntime)); + } + } + + #[cfg(feature = "runtime-smol")] + { + return Some(Arc::new(SmolRuntime)); + } + + #[cfg(not(feature = "runtime-smol"))] + None +} + +#[cfg(feature = "runtime-tokio")] +mod tokio; +#[cfg(feature = "runtime-tokio")] +pub use tokio::TokioRuntime; + +#[cfg(feature = "runtime-smol")] +mod smol; +#[cfg(feature = "runtime-smol")] +pub use smol::*; + +#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +mod tests { + use std::{future, task::Waker}; + + use super::*; + + #[test] + fn non_would_block_send_errors_are_logged_and_ignored() { + let mut sender = Box::pin(UdpSenderHelper::new(TestSocket, writable)); + let transmit = Transmit { + destination: SocketAddr::from(([127, 0, 0, 1], 4433)), + ecn: None, + contents: &[], + segment_size: None, + src_ip: None, + }; + let mut cx = Context::from_waker(Waker::noop()); + + let result = sender.as_mut().poll_send(&transmit, &mut cx); + + assert!(matches!(result, Poll::Ready(Ok(())))); + assert!(sender.last_send_error.is_some()); + } + + #[derive(Debug)] + struct TestSocket; + + impl UdpSenderHelperSocket for TestSocket { + fn try_send(&self, _transmit: &Transmit<'_>) -> io::Result<()> { + Err(io::ErrorKind::PermissionDenied.into()) + } + + fn max_transmit_segments(&self) -> usize { + 1 + } + } + + fn writable(_: &TestSocket) -> future::Ready> { + future::ready(Ok(())) + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs new file mode 100644 index 0000000000..b5ae3d0cf2 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/smol.rs @@ -0,0 +1,105 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, + time::Instant, +}; +use std::{io, sync::Arc, task::ready}; + +use async_io::Async; +use async_io::Timer; + +use super::AsyncTimer; +use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for smol +#[derive(Debug)] +pub struct SmolRuntime; + +impl Runtime for SmolRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(Timer::at(t)) + } + + fn spawn(&self, future: Pin + Send>>) { + ::smol::spawn(future).detach(); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket::new(sock)?)) + } +} + +impl AsyncTimer for Timer { + fn reset(mut self: Pin<&mut Self>, t: Instant) { + self.set_at(t) + } + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx).map(|_| ()) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc>, + inner: Arc, +} + +impl UdpSocket { + fn new(sock: std::net::UdpSocket) -> io::Result { + Ok(Self { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(Async::new_nonblocking(sock)?), + }) + } +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.inner.try_send((&self.io).into(), transmit)?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_readable(cx))?; + match self.inner.recv((&self.io).into(), bufs, meta) { + Ok(res) => return Poll::Ready(Ok(res)), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.as_ref().as_ref().local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs new file mode 100644 index 0000000000..ac5dfd6f20 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -0,0 +1,111 @@ +use std::{ + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll, ready}, + time::Instant, +}; + +use tokio::{ + io::Interest, + time::{Sleep, sleep_until}, +}; + +use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for Tokio +#[derive(Debug)] +pub struct TokioRuntime; + +impl Runtime for TokioRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(sleep_until(t.into())) + } + + fn spawn(&self, future: Pin + Send>>) { + tokio::spawn(future); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) + } + + fn now(&self) -> Instant { + tokio::time::Instant::now().into_std() + } +} + +impl AsyncTimer for Sleep { + fn reset(self: Pin<&mut Self>, t: Instant) { + Self::reset(self, t.into()) + } + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc, + inner: Arc, +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.io.try_io(Interest::WRITABLE, || { + self.inner.try_send((&self.io).into(), transmit) + })?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_recv_ready(cx))?; + match self.io.try_io(Interest::READABLE, || { + self.inner.recv((&self.io).into(), bufs, meta) + }) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`. Looping on any other + // error would spin: readiness stays asserted, so `poll_recv_ready` + // completes at once and the recv fails again, without ever yielding. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs new file mode 100644 index 0000000000..570db9ac81 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -0,0 +1,454 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::{Pin, pin}, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use thiserror::Error; + +use crate::{ + VarInt, + connection::{ConnectionRef, State}, +}; + +/// A stream that can only be used to send data +/// +/// If dropped, streams that haven't been explicitly [`reset()`] will be implicitly [`finish()`]ed, +/// continuing to (re)transmit previously written data until it has been fully acknowledged or the +/// connection is closed. +/// +/// # Cancellation +/// +/// A `write` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready will always result in no data being written to the stream. This is true of methods which +/// succeed immediately when any progress is made, and is not true of methods which might need to +/// perform multiple writes internally before succeeding. Each `write` method documents whether it is +/// cancel-safe. +/// +/// [`reset()`]: SendStream::reset +/// [`finish()`]: SendStream::finish +#[derive(Debug)] +pub struct SendStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, +} + +impl SendStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + } + } + + /// Write a buffer into this stream, returning how many bytes were written + /// + /// Unless this method errors, it waits until some amount of `buf` can be written into this + /// stream, and then writes as much as it can without waiting again. Due to congestion and flow + /// control, this may be shorter than `buf.len()`. On success this yields the length of the + /// prefix that was written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write(&mut self, buf: &[u8]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write(buf))).await + } + + /// Write a buffer into this stream in its entirety + /// + /// This method repeatedly calls [`write`](Self::write) until all bytes are written, or an + /// error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some prefix of `buf` + /// may have been written when previously polled. + pub async fn write_all(&mut self, mut buf: &[u8]) -> Result<(), WriteError> { + while !buf.is_empty() { + let written = self.write(buf).await?; + buf = &buf[written..]; + } + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream, returning how much was written + /// + /// Bytes to try to write are provided to this method as an array of cheaply cloneable chunks. + /// Unless this method errors, it waits until some amount of those bytes can be written into + /// this stream, and then writes as much as it can without waiting again. Due to congestion and + /// flow control, this may be less than the total number of bytes. + /// + /// On success, this method both mutates `bufs` and yields an informative [`Written`] struct + /// indicating how much was written: + /// + /// - [`Bytes`] chunks that were fully written are mutated to be [empty](Bytes::is_empty). + /// - If a [`Bytes`] chunk was partially written, it is [split to](Bytes::split_to) contain + /// only the suffix of bytes that were not written. + /// - The yielded [`Written`] struct indicates how many chunks were fully written as well as + /// how many bytes were written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write_chunks(&mut self, bufs: &mut [Bytes]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write_chunks(bufs))).await + } + + /// Write a single [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an single cheaply cloneable chunk. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.write_all_chunks(&mut [buf]).await?; + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an array of cheaply cloneable chunks. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. This method mutates `bufs` by mutating all chunks to be + /// [empty](Bytes::is_empty). + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_all_chunks(&mut self, mut bufs: &mut [Bytes]) -> Result<(), WriteError> { + while !bufs.is_empty() { + let written = self.write_chunks(bufs).await?; + bufs = &mut bufs[written.chunks..]; + } + Ok(()) + } + + fn execute_poll( + &mut self, + cx: &mut Context<'_>, + write_fn: F, + ) -> Poll> + where + F: FnOnce(&mut proto::SendStream<'_>) -> Result, + { + use proto::WriteError::*; + let mut conn = self.conn.state.lock("SendStream::poll_write"); + if self.is_0rtt { + conn.check_0rtt() + .map_err(|()| WriteError::ZeroRttRejected)?; + } + if let Some(x) = &conn.error { + return Poll::Ready(Err(WriteError::ConnectionLost(x.clone()))); + } + + let result = match write_fn(&mut conn.inner.send_stream(self.stream)) { + Ok(result) => result, + Err(Blocked) => { + conn.blocked_writers.insert(self.stream, cx.waker().clone()); + return Poll::Pending; + } + Err(Stopped(error_code)) => { + return Poll::Ready(Err(WriteError::Stopped(error_code))); + } + Err(ClosedStream) => { + return Poll::Ready(Err(WriteError::ClosedStream)); + } + }; + + conn.wake(); + Poll::Ready(Ok(result)) + } + + /// Notify the peer that no more data will ever be written to this stream + /// + /// It is an error to write to a [`SendStream`] after `finish()`ing it. [`reset()`](Self::reset) + /// may still be called after `finish` to abandon transmission of any stream data that might + /// still be buffered. + /// + /// To wait for the peer to receive all buffered stream data, see [`stopped()`](Self::stopped). + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("finish"); + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(FinishError::ClosedStream) => Err(ClosedStream::default()), + // Harmless. If the application needs to know about stopped streams at this point, it + // should call `stopped`. + Err(FinishError::Stopped(_)) => Ok(()), + } + } + + /// Close the send stream immediately. + /// + /// No new data can be written after calling this method. Locally buffered data is dropped, and + /// previously transmitted data will no longer be retransmitted if lost. If an attempt has + /// already been made to finish the stream, the peer may still receive all written data. + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn reset(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.send_stream(self.stream).reset(error_code)?; + conn.wake(); + Ok(()) + } + + /// Set the priority of the send stream + /// + /// Every send stream has an initial priority of 0. Locally buffered data from streams with + /// higher priority will be transmitted before data from streams with lower priority. Changing + /// the priority of a stream with pending data may only take effect after that data has been + /// transmitted. Using many different priority levels per connection may have a negative + /// impact on performance. + pub fn set_priority(&self, priority: i32) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::set_priority"); + conn.inner.send_stream(self.stream).set_priority(priority)?; + Ok(()) + } + + /// Get the priority of the send stream + pub fn priority(&self) -> Result { + let mut conn = self.conn.state.lock("SendStream::priority"); + conn.inner.send_stream(self.stream).priority() + } + + /// Completes when the peer stops the stream or reads the stream to completion + /// + /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the + /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt + /// of all stream data (although not necessarily the processing of it), after which the peer + /// closing the stream is no longer meaningful. + /// + /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving + /// data. As such, relying on `stopped` to know when the peer has read a stream to completion + /// may introduce more latency than using an application-level response of some sort. + /// + /// Clients may wish to await this after finishing a unidirectional 0-RTT stream to reliably + /// determine whether the stream was rejected. + pub fn stopped( + &self, + ) -> impl Future, StoppedError>> + Send + Sync + 'static { + let conn = self.conn.clone(); + let stream = self.stream; + let is_0rtt = self.is_0rtt; + async move { + loop { + // The `Notify::notified` future needs to be created while the lock is being held, + // otherwise a wakeup could be missed if triggered inbetween releasing the lock + // and creating the future. + // The lock may only be held in a block without `await`s, otherwise the future + // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore + // we need to declare `notify` outside of the block, and initialize it inside. + let notify; + { + let mut conn = conn.state.lock("SendStream::stopped"); + if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { + return output; + } + + notify = conn.stopped.entry(stream).or_default().clone(); + notify.notified() + } + .await + } + } + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Attempt to write bytes from buf into the stream. + /// + /// On success, returns Poll::Ready(Ok(num_bytes_written)). + /// + /// If the stream is not ready for writing, the method returns Poll::Pending and arranges + /// for the current task (via cx.waker().wake_by_ref()) to receive a notification when the + /// stream becomes writable or is closed. + pub fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + pin!(self.get_mut().write(buf)).as_mut().poll(cx) + } +} + +/// Check if a send stream is stopped. +/// +/// Returns `Some` if the stream is stopped or the connection is closed. +/// Returns `None` if the stream is not stopped. +fn send_stream_stopped( + conn: &mut State, + stream: StreamId, + is_0rtt: bool, +) -> Option, StoppedError>> { + if is_0rtt && conn.check_0rtt().is_err() { + return Some(Err(StoppedError::ZeroRttRejected)); + } + match conn.inner.send_stream(stream).stopped() { + Err(ClosedStream { .. }) => Some(Ok(None)), + Ok(Some(error_code)) => Some(Ok(Some(error_code))), + Ok(None) => conn.error.clone().map(|error| Err(error.into())), + } +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl Drop for SendStream { + fn drop(&mut self) { + let mut conn = self.conn.state.lock("SendStream::drop"); + + // clean up any previously registered wakers + conn.blocked_writers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => conn.wake(), + Err(FinishError::Stopped(reason)) => { + if conn.inner.send_stream(self.stream).reset(reason).is_ok() { + conn.wake(); + } + } + // Already finished or reset, which is fine. + Err(FinishError::ClosedStream) => {} + } + } +} + +/// Errors that arise from writing to a stream +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum WriteError { + /// The peer is no longer accepting data on this stream + /// + /// Carries an application-defined error code. + #[error("sending stopped by peer: error {0}")] + Stopped(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been finished or reset + #[error("closed stream")] + ClosedStream, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for WriteError { + #[inline] + fn from(_: ClosedStream) -> Self { + Self::ClosedStream + } +} + +impl From for WriteError { + fn from(x: StoppedError) -> Self { + match x { + StoppedError::ConnectionLost(e) => Self::ConnectionLost(e), + StoppedError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: WriteError) -> Self { + use WriteError::*; + let kind = match x { + Stopped(_) | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while monitoring for a send stream stop from the peer +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum StoppedError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: StoppedError) -> Self { + use StoppedError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs new file mode 100755 index 0000000000..e0c6f3ef9e --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -0,0 +1,1871 @@ +#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] + +#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +use rustls::crypto::aws_lc_rs::default_provider; +#[cfg(feature = "rustls-ring")] +use rustls::crypto::ring::default_provider; + +use std::{ + convert::TryInto, + future::Future, + io::{self, IoSliceMut}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}, + pin::{Pin, pin}, + str, + sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; +use crate::{Duration, Instant}; +use bytes::Bytes; +use proto::{ + ConnectionId, RandomConnectionIdGenerator, + crypto::rustls::{QuicClientConfig, QuicServerConfig}, + crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, + transport_parameters::TransportParameters, +}; +use rand::{Rng, SeedableRng, rngs::StdRng}; +use rustls::{ + RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}, +}; +use tokio::time::{sleep, timeout}; +use tokio::{ + join, + runtime::{Builder, Runtime}, +}; +use tracing::instrument::Instrument as _; +use tracing::{error_span, info}; +use tracing_subscriber::EnvFilter; +use udp::RecvMeta; + +use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; + +#[test] +fn handshake_timeout() { + let _guard = subscribe(); + let runtime = rt_threaded(); + let client = { + let _guard = runtime.enter(); + Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap() + }; + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + const IDLE_TIMEOUT: Duration = Duration::from_millis(500); + let mut transport_config = TransportConfig::default(); + transport_config + .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .initial_rtt(Duration::from_millis(10)); + client_config.transport_config(Arc::new(transport_config)); + + let start = Instant::now(); + runtime.block_on(async move { + match client + .connect_with( + client_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1), + "localhost", + ) + .unwrap() + .await + { + Err(crate::ConnectionError::TimedOut) => {} + Err(e) => panic!("unexpected error: {e:?}"), + Ok(_) => panic!("unexpected success"), + } + }); + let dt = start.elapsed(); + assert!(dt > IDLE_TIMEOUT && dt < 2 * IDLE_TIMEOUT); +} + +#[tokio::test] +async fn close_endpoint() { + let _guard = subscribe(); + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let endpoint = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + endpoint + .set_default_client_config(ClientConfig::with_root_certificates(Arc::new(roots)).unwrap()); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + + tokio::spawn(async move { + let _ = conn.await; + }); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + endpoint.close(0u32.into(), &[]); + match conn.await { + Err(crate::ConnectionError::LocallyClosed) => (), + Err(e) => panic!("unexpected error: {e}"), + Ok(_) => { + panic!("unexpected success"); + } + } +} + +#[test] +fn local_addr() { + let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).unwrap(); + let addr = socket.local_addr().unwrap(); + let runtime = rt_basic(); + let ep = { + let _guard = runtime.enter(); + Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + }; + assert_eq!( + addr, + ep.local_addr() + .expect("Could not obtain our local endpoint") + ); +} + +#[test] +fn read_after_close() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + const MSG: &[u8] = b"goodbye!"; + let endpoint2 = endpoint.clone(); + runtime.spawn(async move { + let new_conn = endpoint2 + .accept() + .await + .expect("endpoint") + .await + .expect("connection"); + let mut s = new_conn.open_uni().await.unwrap(); + s.write_all(MSG).await.unwrap(); + s.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = s.stopped().await; + }); + runtime.block_on(async move { + let new_conn = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect"); + sleep(Duration::from_millis(100)).await; + let mut stream = new_conn.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG); + }); +} + +#[test] +fn export_keying_material() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + runtime.block_on(async move { + let outgoing_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect") + } + }); + let incoming_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .accept() + .await + .expect("endpoint") + .await + .expect("connection") + } + }); + let outgoing_conn = outgoing_conn_fut.await.unwrap(); + let incoming_conn = incoming_conn_fut.await.unwrap(); + let mut i_buf = [0u8; 64]; + incoming_conn + .export_keying_material(&mut i_buf, b"asdf", b"qwer") + .unwrap(); + let mut o_buf = [0u8; 64]; + outgoing_conn + .export_keying_material(&mut o_buf, b"asdf", b"qwer") + .unwrap(); + assert_eq!(&i_buf[..], &o_buf[..]); + }); +} + +#[tokio::test] +async fn ip_blocking() { + let _guard = subscribe(); + let endpoint_factory = EndpointFactory::new(); + let client_1 = endpoint_factory.endpoint(); + let client_1_addr = client_1.local_addr().unwrap(); + let client_2 = endpoint_factory.endpoint(); + let server = endpoint_factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let server_task = tokio::spawn(async move { + loop { + let accepting = server.accept().await.unwrap(); + if accepting.remote_address() == client_1_addr { + accepting.refuse(); + } else if accepting.remote_address_validated() { + accepting.await.expect("connection"); + } else { + accepting.retry().unwrap(); + } + } + }); + tokio::join!( + async move { + let e = client_1 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect_err("server should have blocked this"); + assert!( + matches!(e, crate::ConnectionError::ConnectionClosed(_)), + "wrong error" + ); + }, + async move { + client_2 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + } + ); + server_task.abort(); +} + +/// Construct an endpoint suitable for connecting to itself +fn endpoint() -> Endpoint { + EndpointFactory::new().endpoint() +} + +fn endpoint_with_config(transport_config: TransportConfig) -> Endpoint { + EndpointFactory::new().endpoint_with_config(transport_config) +} + +/// Constructs endpoints suitable for connecting to themselves and each other +struct EndpointFactory { + cert: rcgen::CertifiedKey, + endpoint_config: EndpointConfig, +} + +impl EndpointFactory { + fn new() -> Self { + Self { + cert: rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(), + endpoint_config: EndpointConfig::default(), + } + } + + fn endpoint(&self) -> Endpoint { + self.endpoint_with_config(TransportConfig::default()) + } + + fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { + let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); + let transport_config = Arc::new(transport_config); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + server_config.transport_config(transport_config.clone()); + + let mut roots = RootCertStore::empty(); + roots.add(self.cert.cert.der().clone()).unwrap(); + let endpoint = Endpoint::new( + self.endpoint_config.clone(), + Some(server_config), + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + Arc::new(TokioRuntime), + ) + .unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(transport_config); + endpoint.set_default_client_config(client_config); + + endpoint + } +} + +#[derive(Default)] +struct HandshakeBlocker { + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct HandshakeBlockerState { + started: bool, + released: bool, +} + +impl HandshakeBlocker { + fn block(&self) { + let mut state = self.state.lock().unwrap(); + state.started = true; + self.changed.notify_all(); + while !state.released { + state = self.changed.wait(state).unwrap(); + } + } + + fn wait_until_started(&self) { + let state = self.state.lock().unwrap(); + let (state, _) = self + .changed + .wait_timeout_while(state, Duration::from_secs(5), |state| !state.started) + .unwrap(); + assert!(state.started, "timed out waiting for handshake to start"); + } + + fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.released = true; + self.changed.notify_all(); + } +} + +struct HandshakeReleaseGuard(Arc); + +impl Drop for HandshakeReleaseGuard { + fn drop(&mut self) { + self.0.release(); + } +} + +struct BlockingServerConfig { + inner: Arc, + blocker: Arc, +} + +impl ProtoServerConfig for BlockingServerConfig { + fn initial_keys( + &self, + version: u32, + dst_cid: ConnectionId, + ) -> Result { + self.inner.initial_keys(version, dst_cid) + } + + fn retry_tag(&self, version: u32, orig_dst_cid: ConnectionId, packet: &[u8]) -> [u8; 16] { + self.inner.retry_tag(version, orig_dst_cid, packet) + } + + fn start_session( + self: Arc, + version: u32, + params: &TransportParameters, + ) -> Box { + self.blocker.block(); + self.inner.clone().start_session(version, params) + } +} + +/// Wraps a socket so that a test can make its next receive fail, which ends the endpoint driver +#[derive(Debug)] +struct FailingSocket { + inner: Box, + failure: Arc, +} + +#[derive(Debug, Default)] +struct SocketFailure { + armed: AtomicBool, + recv_waker: Mutex>, +} + +impl SocketFailure { + /// Make the next receive fail, waking the endpoint driver so that it observes the failure + fn trigger(&self) { + self.armed.store(true, Ordering::SeqCst); + if let Some(waker) = self.recv_waker.lock().unwrap().as_ref() { + waker.wake_by_ref(); + } + } +} + +impl AsyncUdpSocket for FailingSocket { + fn create_sender(&self) -> Pin> { + self.inner.create_sender() + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + if self.failure.armed.load(Ordering::SeqCst) { + return Poll::Ready(Err(io::Error::other("injected socket failure"))); + } + *self.failure.recv_waker.lock().unwrap() = Some(cx.waker().clone()); + self.inner.poll_recv(cx, bufs, meta) + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } +} + +/// Poll a connection driver before its registration returns, once the endpoint has failed. +#[derive(Debug)] +struct DriverLossRuntime { + failure: Arc, + stopped_on_first_poll: AtomicBool, +} + +impl crate::runtime::Runtime for DriverLossRuntime { + fn new_timer(&self, deadline: Instant) -> Pin> { + TokioRuntime.new_timer(deadline) + } + + fn spawn(&self, mut future: Pin + Send>>) { + if self.failure.armed.load(Ordering::SeqCst) { + let mut cx = Context::from_waker(Waker::noop()); + if future.as_mut().poll(&mut cx).is_ready() { + self.stopped_on_first_poll.store(true, Ordering::SeqCst); + return; + } + } + TokioRuntime.spawn(future); + } + + fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { + TokioRuntime.wrap_udp_socket(socket) + } + + fn now(&self) -> Instant { + TokioRuntime.now() + } +} + +fn localhost_socket() -> Box { + TokioRuntime + .wrap_udp_socket( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + ) + .unwrap() +} + +fn blocking_server_pair( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> (Endpoint, Endpoint, Arc) { + let factory = EndpointFactory::new(); + let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); + let mut server_crypto = rustls::ServerConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![factory.cert.cert.der().clone()], key) + .unwrap(); + server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; + let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + QuicServerConfig::try_from(server_crypto).unwrap(), + )); + let blocker = Arc::new(HandshakeBlocker::default()); + server_config.crypto = Arc::new(BlockingServerConfig { + inner: server_config.crypto.clone(), + blocker: blocker.clone(), + }); + + let server = Endpoint::new_with_abstract_socket( + factory.endpoint_config.clone(), + Some(server_config), + server_socket, + server_runtime, + ) + .unwrap(); + + let mut roots = RootCertStore::empty(); + roots.add(factory.cert.cert.der().clone()).unwrap(); + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.alpn_protocols = vec![client_alpn.to_vec()]; + client.set_default_client_config(ClientConfig::new(Arc::new( + QuicClientConfig::try_from(client_crypto).unwrap(), + ))); + + (client, server, blocker) +} + +async fn wait_for_blocked_handshake(blocker: Arc) { + tokio::task::spawn_blocking(move || blocker.wait_until_started()) + .await + .unwrap(); +} + +async fn accept_with_blocked_start_session( + server: Endpoint, +) -> Result { + let incoming = server.accept().await.unwrap(); + // The test crypto provider intentionally blocks in `start_session`, so run + // accept on the blocking pool rather than parking a Tokio worker thread. + let connecting = tokio::task::spawn_blocking(move || incoming.accept()) + .await + .unwrap()?; + connecting.await +} + +struct BlockedHandshake { + client: Endpoint, + server: Endpoint, + blocker: Arc, + release_guard: HandshakeReleaseGuard, + accept: tokio::task::JoinHandle>, + connect: tokio::task::JoinHandle>, +} + +fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { + blocked_handshake_with_socket(server_alpn, client_alpn, localhost_socket()) +} + +fn blocked_handshake_with_socket( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, +) -> BlockedHandshake { + blocked_handshake_with_runtime( + server_alpn, + client_alpn, + server_socket, + Arc::new(TokioRuntime), + ) +} + +fn blocked_handshake_with_runtime( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> BlockedHandshake { + let (client, server, blocker) = + blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let server_addr = server.local_addr().unwrap(); + + let accept = tokio::spawn({ + let server = server.clone(); + async move { accept_with_blocked_start_session(server).await } + }); + let connect = tokio::spawn({ + let client = client.clone(); + async move { client.connect(server_addr, "localhost").unwrap().await } + }); + + BlockedHandshake { + client, + server, + blocker: blocker.clone(), + release_guard: HandshakeReleaseGuard(blocker), + accept, + connect, + } +} + +async fn await_connection( + task: tokio::task::JoinHandle>, +) -> Result { + timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() +} + +async fn assert_wait_idle_pending(endpoint: &Endpoint) { + assert!( + timeout(Duration::from_millis(50), endpoint.wait_idle()) + .await + .is_err() + ); +} + +async fn wait_idle(endpoint: &Endpoint) { + timeout(Duration::from_secs(5), endpoint.wait_idle()) + .await + .unwrap(); +} + +/// A parked `wait_idle` must not hang when the endpoint driver is dropped. +/// +/// `EndpointDriver::drop` clears the connection senders, which is what makes +/// the endpoint idle, but it wakes only `incoming`. A caller already parked on +/// `idle` is therefore never woken, and with the driver gone nothing else can +/// wake it. `EndpointInner::accept` already guards this same hazard for accepts +/// that complete after driver loss. +#[test] +fn wait_idle_returns_when_the_driver_is_dropped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + + // The endpoint under test owns its runtime, so shutting that runtime down + // drops its driver. + let endpoint_runtime = rt_threaded(); + let endpoint_handle = endpoint_runtime.handle().clone(); + let endpoint = { + let _guard = endpoint_runtime.enter(); + factory.endpoint() + }; + let endpoint_addr = endpoint.local_addr().unwrap(); + + // A peer on a second runtime, so a live connection exists to wait for. + let peer_runtime = rt_threaded(); + let peer = { + let _guard = peer_runtime.enter(); + factory.endpoint() + }; + let peer_handle = peer_runtime.handle().clone(); + // Accept concurrently: the client's handshake cannot complete until the + // server drives its half, and the accepted handle must stay alive or the + // connection would close and the endpoint become idle before the assertion. + let accepted = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(async { + let incoming = endpoint.accept().await.expect("incoming"); + incoming.await.expect("accept") + }) + }) + }; + let connection = peer_handle.block_on(async { + peer.connect(endpoint_addr, "localhost") + .expect("connect") + .await + .expect("connection") + }); + let _server_connection = accepted.join().expect("accept thread"); + + // Park `wait_idle`; it blocks while that connection is live. + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let waiter = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(endpoint.wait_idle()); + let _ = done_tx.send(()); + }) + }; + + // Otherwise a pass below would be vacuous. + assert!( + done_rx.recv_timeout(Duration::from_millis(200)).is_err(), + "wait_idle returned before the driver was dropped" + ); + + // Dropping the runtime drops the endpoint driver. + endpoint_runtime.shutdown_background(); + + let woke = done_rx.recv_timeout(Duration::from_secs(10)).is_ok(); + drop(connection); + drop(peer); + drop(peer_runtime); + assert!( + woke, + "wait_idle never returned after the endpoint driver was dropped" + ); + let _ = waiter.join(); +} + +async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { + timeout(Duration::from_secs(5), conn.closed()) + .await + .unwrap() +} + +#[tokio::test] +async fn zero_rtt() { + let _guard = subscribe(); + let endpoint = endpoint(); + + const MSG0: &[u8] = b"zero"; + const MSG1: &[u8] = b"one"; + let endpoint2 = endpoint.clone(); + let accept_connection = move |expect_0rtt| { + let endpoint2 = endpoint2.clone(); + async move { + let incoming = endpoint2.accept().await.unwrap().accept().unwrap(); + let connection = incoming.into_0rtt().unwrap_or_else(|_| unreachable!()); + let c = connection.clone(); + tokio::spawn(async move { + while let Ok(mut x) = c.accept_uni().await { + assert_eq!(x.is_0rtt(), expect_0rtt); + let msg = x.read_to_end(usize::MAX).await.unwrap(); + assert_eq!(msg, MSG0); + } + }); + info!("sending 0.5-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG0).await.expect("write"); + s.finish().unwrap(); + connection.authenticated().await.expect("connected"); + info!("sending 1-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG1).await.expect("write"); + // The peer might close the connection before ACKing + let _ = s.finish(); + } + }; + + tokio::spawn(accept_connection(false)); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .expect_err("0-RTT succeeded without keys") + .await + .expect("connect"); + + { + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + // Read a 1-RTT message to ensure the handshake completes fully, allowing the server's + // NewSessionTicket frame to be received. + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG1); + drop(connection); + } + + info!("initial connection complete"); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .unwrap_or_else(|_| panic!("missing 0-RTT keys")); + + // Send something before the handshake can make progress, thereby forcing 0-RTT + let mut stream_0rtt = connection.open_uni().await.expect("0-RTT open uni"); + info!("sending 0-RTT"); + stream_0rtt.write_all(MSG0).await.expect("0-RTT write"); + stream_0rtt.finish().unwrap(); + + tokio::spawn(accept_connection(true)); + + // Receive 0.5-RTT + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + connection.authenticated().await.expect("connected"); + + // Ensure 0-RTT was accepted + stream_0rtt.stopped().await.expect("0-RTT stopped"); + + // Allow the connection to close + drop((stream_0rtt, stream, connection)); + + endpoint.wait_idle().await; +} + +#[test] +#[cfg_attr( + any(target_os = "solaris", target_os = "illumos"), + ignore = "Fails on Solaris and Illumos" +)] +fn echo_v6() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Sometimes hangs in poll() on Solaris")] +fn echo_v4() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn echo_dualstack() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_receive_window() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(100 * 1024 * 1024), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_stream_receive_window() { + // Note that there is no point in running this with too many streams, + // since the window is only active within a stream. + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 2, + stream_size: 250 * 1024 + 11, + receive_window: Some(100 * 1024 * 1024), + stream_receive_window: Some(37), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_both_windows() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(37), + }); +} + +fn run_echo(args: EchoArgs) { + let _guard = subscribe(); + let runtime = rt_basic(); + let handle = { + // Use small receive windows + let mut transport_config = TransportConfig::default(); + if let Some(receive_window) = args.receive_window { + transport_config.receive_window(receive_window.try_into().unwrap()); + } + if let Some(stream_receive_window) = args.stream_receive_window { + transport_config.stream_receive_window(stream_receive_window.try_into().unwrap()); + } + transport_config.max_concurrent_bidi_streams(1_u8.into()); + transport_config.max_concurrent_uni_streams(1_u8.into()); + let transport_config = Arc::new(transport_config); + + // We don't use the `endpoint` helper here because we want two different endpoints with + // different addresses. + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + + server_config.transport = transport_config.clone(); + let server_sock = UdpSocket::bind(args.server_addr).unwrap(); + let server_addr = server_sock.local_addr().unwrap(); + let server = { + let _guard = runtime.enter(); + let _guard = error_span!("server").entered(); + Endpoint::new( + Default::default(), + Some(server_config), + server_sock, + Arc::new(TokioRuntime), + ) + .unwrap() + }; + + let mut roots = RootCertStore::empty(); + roots.add(cert).unwrap(); + let mut client_crypto = + rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.key_log = Arc::new(rustls::KeyLogFile::new()); + + let client = { + let _guard = runtime.enter(); + let _guard = error_span!("client").entered(); + Endpoint::client(args.client_addr).unwrap() + }; + let mut client_config = + ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())); + client_config.transport_config(transport_config); + client.set_default_client_config(client_config); + + let handle = runtime.spawn(async move { + let incoming = server.accept().await.unwrap(); + + // Note for anyone modifying the platform support in this test: + // If `local_ip` gets available on additional platforms - which + // requires modifying this test - please update the list of supported + // platforms in the doc comment of `quinn_udp::RecvMeta::dst_ip`. + if cfg!(target_os = "linux") + || cfg!(target_os = "android") + || cfg!(target_os = "freebsd") + || cfg!(target_os = "openbsd") + || cfg!(target_os = "netbsd") + || cfg!(target_os = "macos") + || cfg!(target_os = "windows") + { + let local_ip = incoming.local_ip().expect("Local IP must be available"); + assert!(local_ip.is_loopback()); + } else { + assert_eq!(None, incoming.local_ip()); + } + + let new_conn = incoming.await.unwrap(); + tokio::spawn(async move { + while let Ok(stream) = new_conn.accept_bi().await { + tokio::spawn(echo(stream)); + } + }); + server.wait_idle().await; + }); + + info!("connecting from {} to {}", args.client_addr, server_addr); + runtime.block_on( + async move { + let new_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + + /// This is just an arbitrary number to generate deterministic test data + const SEED: u64 = 0x12345678; + + for i in 0..args.nr_streams { + println!("Opening stream {i}"); + let (mut send, mut recv) = new_conn.open_bi().await.expect("stream open"); + let msg = gen_data(args.stream_size, SEED); + + let send_task = async { + send.write_all(&msg).await.expect("write"); + send.finish().unwrap(); + }; + let recv_task = async { recv.read_to_end(usize::MAX).await.expect("read") }; + + let (_, data) = tokio::join!(send_task, recv_task); + + assert_eq!(data[..], msg[..], "Data mismatch"); + } + new_conn.close(0u32.into(), b"done"); + client.wait_idle().await; + } + .instrument(error_span!("client")), + ); + handle + }; + runtime.block_on(handle).unwrap(); +} + +struct EchoArgs { + client_addr: SocketAddr, + server_addr: SocketAddr, + nr_streams: usize, + stream_size: usize, + receive_window: Option, + stream_receive_window: Option, +} + +async fn echo((mut send, mut recv): (SendStream, RecvStream)) { + loop { + // These are 32 buffers, for reading approximately 32kB at once + #[rustfmt::skip] + let mut bufs = [ + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + ]; + + match recv.read_chunks(&mut bufs).await.expect("read chunks") { + Some(n) => { + send.write_all_chunks(&mut bufs[..n]) + .await + .expect("write chunks"); + } + None => break, + } + } + + let _ = send.finish(); +} + +fn gen_data(size: usize, seed: u64) -> Vec { + let mut rng: StdRng = SeedableRng::seed_from_u64(seed); + let mut buf = vec![0; size]; + rng.fill_bytes(&mut buf); + buf +} + +fn subscribe() -> tracing::subscriber::DefaultGuard { + let sub = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(|| TestWriter) + .finish(); + tracing::subscriber::set_default(sub) +} + +struct TestWriter; + +impl io::Write for TestWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + print!( + "{}", + str::from_utf8(buf).expect("tried to log invalid UTF-8") + ); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + io::stdout().flush() + } +} + +fn rt_basic() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +fn rt_threaded() -> Runtime { + Builder::new_multi_thread().enable_all().build().unwrap() +} + +#[tokio::test] +async fn retry_after_rebind() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + server + .rebind(UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap()) + .unwrap(); + + timeout(Duration::from_secs(5), async { + let connecting = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + let incoming = server.accept().await.unwrap(); + assert!(!incoming.remote_address_validated()); + incoming.retry().unwrap(); + + let incoming = server.accept().await.unwrap(); + assert!(incoming.remote_address_validated()); + let (accepted, connected) = join!(incoming, connecting); + let _server_connection = accepted.unwrap(); + let _client_connection = connected.unwrap(); + server.close(0u32.into(), b"done"); + client.close(0u32.into(), b"done"); + }) + .await + .expect("retried handshake must complete using the rebound socket"); +} + +#[tokio::test] +async fn rebind_recv() { + let _guard = subscribe(); + + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + + let mut roots = RootCertStore::empty(); + roots.add(cert.clone()).unwrap(); + + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(Arc::new({ + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + cfg + })); + client.set_default_client_config(client_config); + + let server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + let server = { + let _guard = tracing::error_span!("server").entered(); + Endpoint::server( + server_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + ) + .unwrap() + }; + let server_addr = server.local_addr().unwrap(); + + const MSG: &[u8; 5] = b"hello"; + + let write_send = Arc::new(tokio::sync::Notify::new()); + let write_recv = write_send.clone(); + let connected_send = Arc::new(tokio::sync::Notify::new()); + let connected_recv = connected_send.clone(); + let server = tokio::spawn(async move { + let connection = server.accept().await.unwrap().await.unwrap(); + info!("got conn"); + connected_send.notify_one(); + write_recv.notified().await; + let mut stream = connection.open_uni().await.unwrap(); + stream.write_all(MSG).await.unwrap(); + stream.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = stream.stopped().await; + }); + + let connection = { + let _guard = tracing::error_span!("client").entered(); + client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap() + }; + info!("connected"); + connected_recv.notified().await; + client + .rebind(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) + .unwrap(); + info!("rebound"); + write_send.notify_one(); + let mut stream = connection.accept_uni().await.unwrap(); + assert_eq!(stream.read_to_end(MSG.len()).await.unwrap(), MSG); + server.await.unwrap(); +} + +/// Verify endpoint behavior while a connection is between `start_accept` and `finish_accept`: +/// `open_connections` must be 0, and `wait_idle` must not complete while `pending_accepts > 0`. +/// After the accept completes and the connection is closed, `wait_idle` must resolve. +#[tokio::test] +async fn split_accept_wait_idle_and_open_connections() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + let client_conn = await_connection(connect).await.unwrap(); + let server_conn = await_connection(accept).await.unwrap(); + assert_eq!(server.open_connections(), 1); + + client_conn.close(0u32.into(), b"done"); + let _ = wait_closed(&client_conn).await; + let _ = wait_closed(&server_conn).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// A failed accept never becomes a connection, so no Drained event will wake idle waiters. +#[tokio::test] +async fn split_accept_failure_wakes_idle_waiter() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"server", b"client"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + let error = await_connection(accept).await.unwrap_err(); + assert!(matches!( + error, + crate::ConnectionError::TransportError(error) + if error.code == proto::TransportErrorCode::crypto(0x78) + )); + // Repolling wait_idle alone would hide a missing notification now that the endpoint is + // idle. Verify that the waiter registered before the failure was actually woken. + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + assert_eq!(server.open_connections(), 0); + + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +/// Verify that calling `Endpoint::close` while a connection is in the `Accepting` state +/// (between `start_accept` and `finish_accept`) produces `LocallyClosed` on the server side +/// and `ConnectionClosed` on the client side, and that `wait_idle` resolves afterward. +#[tokio::test] +async fn split_accept_close_during_pending_accept() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + match await_connection(accept).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + } + Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + } + + match await_connection(connect).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + Err(err) => { + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + } + + wait_idle(&server).await; + wait_idle(&client).await; + assert_eq!(server.open_connections(), 0); +} + +/// The endpoint driver may exit while an accept runs without the lock. The connection registered +/// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly +/// and `wait_idle` does not wait on a connection nobody drives. +#[tokio::test] +async fn split_accept_driver_lost_during_handshake() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_socket(b"test", b"test", socket); + + wait_for_blocked_handshake(blocker.clone()).await; + // Fail the server socket, which ends the endpoint driver while the handshake is blocked. + // `accept` yields `None` once the driver is gone. + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + // Register an idle waiter while the accept is still pending. Nothing but the accept's + // completion can wake it now that the driver is gone. + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + // Without the fix this waits for the handshake timeout, well past `await_connection`'s limit. + assert!(await_connection(accept).await.is_err()); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn split_accept_driver_lost_before_connection_spawn() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let runtime = Arc::new(DriverLossRuntime { + failure: failure.clone(), + stopped_on_first_poll: AtomicBool::new(false), + }); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); + + wait_for_blocked_handshake(blocker.clone()).await; + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + blocker.release(); + // The runtime polls the connection driver before registration returns. It must observe the + // closed event channel immediately, before transmitting any handshake packets. + assert!(await_connection(accept).await.is_err()); + assert!( + runtime.stopped_on_first_poll.load(Ordering::SeqCst), + "connection driver ran before its event channel was closed" + ); + wait_idle(&server).await; + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn remote_address_monitoring() { + let _guard = subscribe(); + + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let client = factory.endpoint(); + + let trigger_migration = Arc::new(tokio::sync::Notify::new()); + let trigger_migration_clone = trigger_migration.clone(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let old_remote = conn.remote_address(); + let mut addr_watch = conn.watch_remote_address(); + + trigger_migration_clone.notify_one(); + + let addr_a = addr_watch.wait().await.unwrap(); + assert_eq!(addr_a.port(), 23_001); + assert_ne!(old_remote, addr_a); + + trigger_migration_clone.notify_one(); + + let addr_b = addr_watch.wait().await.unwrap(); + assert_eq!(addr_b.port(), 23_003); + assert_ne!(addr_a, addr_b); + }); + + let client_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + + let mut port = 23_000; + let mut migrate = async || { + port += 1; + client + .rebind( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)).unwrap(), + ) + .unwrap(); + + // send data from the new socket to trigger migration on the server + let mut stream = client_conn.open_uni().await.unwrap(); + stream.write_all(b"hello").await.unwrap(); + stream.finish().unwrap(); + }; + + trigger_migration.notified().await; + migrate().await; + trigger_migration.notified().await; + migrate().await; + migrate().await; + + server_task.await.unwrap(); + client_conn.close(0u32.into(), b"done"); +} + +#[tokio::test] +async fn stream_id_flow_control() { + let _guard = subscribe(); + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + let endpoint = endpoint_with_config(cfg); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + // If `open_uni` doesn't get unblocked when the previous stream is dropped, this will time out. + tokio::join!( + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + server.accept_uni().await.unwrap(); + server.accept_uni().await.unwrap(); + } + ); +} + +#[tokio::test] +async fn two_datagram_readers() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + let done = tokio::sync::Notify::new(); + let (a, b, ()) = tokio::join!( + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + server.send_datagram(b"one"[..].into()).unwrap(); + done.notified().await; + server.send_datagram_wait(b"two"[..].into()).await.unwrap(); + } + ); + assert!(*a == *b"one" || *b == *b"one"); + assert!(*a == *b"two" || *b == *b"two"); +} + +#[tokio::test] +async fn multiple_conns_with_zero_length_cids() { + let _guard = subscribe(); + let mut factory = EndpointFactory::new(); + factory + .endpoint_config + .cid_generator(Arc::new(|| Box::new(RandomConnectionIdGenerator::new(0)))); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client1 = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + let client2 = { + let _guard = error_span!("client2").entered(); + factory.endpoint() + }; + + let client1 = async move { + let conn = client1 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client1")); + let client2 = async move { + let conn = client2 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client2")); + let server = async move { + let client1 = server.accept().await.unwrap().await.unwrap(); + let client2 = server.accept().await.unwrap().await.unwrap(); + // Both connections are now concurrently live. + client1.close(42u32.into(), &[]); + client2.close(42u32.into(), &[]); + } + .instrument(error_span!("server")); + tokio::join!(client1, client2, server); +} + +#[tokio::test] +async fn stream_stopped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + + let client = async move { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + let stopped1 = stream.stopped(); + let stopped2 = stream.stopped(); + let stopped3 = stream.stopped(); + + stream.write_all(b"hi").await.unwrap(); + // spawn one of the futures into a task + let stopped1 = tokio::task::spawn(stopped1); + // verify that both futures resolved + let (stopped1, stopped2) = tokio::join!(stopped1, stopped2); + assert!(matches!(stopped1, Ok(Ok(Some(val))) if val == 42u32.into())); + assert!(matches!(stopped2, Ok(Some(val)) if val == 42u32.into())); + // drop the stream + drop(stream); + // verify that a future also resolves after dropping the stream + let stopped3 = stopped3.await; + assert_eq!(stopped3, Ok(Some(42u32.into()))); + }; + let client = timeout(Duration::from_millis(100), client).instrument(error_span!("client")); + let server = async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + let mut buf = [0u8; 2]; + stream.read_exact(&mut buf).await.unwrap(); + stream.stop(42u32.into()).unwrap(); + conn + } + .instrument(error_span!("server")); + let (client, conn) = tokio::join!(client, server); + client.expect("timeout"); + drop(conn); +} + +#[tokio::test] +async fn stream_stopped_2() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (conn, _server_conn) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let send_stream = conn.open_uni().await.unwrap(); + let stopped = timeout(Duration::from_millis(100), send_stream.stopped()) + .instrument(error_span!("stopped")); + tokio::pin!(stopped); + // poll the future once so that the waker is registered. + tokio::select! { + biased; + _x = &mut stopped => {}, + _x = std::future::ready(()) => {} + } + // drop the send stream + drop(send_stream); + // make sure the stopped future still resolves + let res = stopped.await; + assert_eq!(res, Ok(Ok(None))); +} + +#[tokio::test] +async fn stream_drop_removes_blocked_reader() { + let _guard = subscribe(); + + for drop_stream in [false, true] { + let endpoint_factory = EndpointFactory::new(); + let server = endpoint_factory.endpoint(); + let server_address = server.local_addr().unwrap(); + let client = endpoint_factory.endpoint(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + + // read "hello" + let mut buf = [0u8; 5]; + stream.read_exact(&mut buf).await.unwrap(); + + let (waker, wake_counter) = new_count_waker(); + let mut cx = Context::from_waker(&waker); + // do a blocking read which will add the stream in conn.blocked_readers + { + let mut buf = [0u8; 64]; + let read_fut = stream.read(&mut buf); + tokio::pin!(read_fut); + assert!(matches!(read_fut.as_mut().poll(&mut cx), Poll::Pending)); + } + + if !drop_stream { + assert_eq!(wake_counter.wakes(), 0); + // We have a blocked reader, closing the connection should wake it. We use this as + // a proxy to assert that the stream is in conn.blocked_readers. + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 1); + } else { + // dropping the stream should remove it from conn.blocked_readers, so we don't + // expect any wakeups + drop(stream); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + } + }); + + let conn = client + .connect(server_address, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + // need to send some data to actually start the stream + stream.write_all(b"hello").await.unwrap(); + + server_task.await.unwrap(); + } +} + +/// Test that dropping a `RecvStream` after cancelling a read and then +/// explicitly `stop`ing it doesn't panic. +#[tokio::test] +async fn recv_stream_cancel_stop_drop() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client").entered(); + factory.endpoint() + }; + let recv_dropped = tokio::sync::SetOnce::new(); + join!( + async { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut recv = conn.accept_uni().await.unwrap(); + // Create a future to read from the stream, poll it once, then immediately drop it + { + let fut = pin!(recv.read_to_end(usize::MAX)); + let mut cx = Context::from_waker(Waker::noop()); + assert!(fut.poll(&mut cx).is_pending()); + } + recv_dropped.set(()).unwrap(); + recv.stop(0u32.into()).unwrap(); + }, + async { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut send = conn.open_uni().await.unwrap(); + _ = send.write_all(b"hello").await; + // Don't drop (finish) the send stream until the read has been + // cancelled by the server, ensuring that read_to_end can't complete + // immediately. + recv_dropped.wait().await; + }, + ); +} + +#[tokio::test(start_paused = true)] +async fn dropped_endpoint_cleans_up() { + let _guard = subscribe(); + + let mut endpoint_factory = EndpointFactory::new(); + let cid_generator = Arc::new(|| -> Box { + Box::::default() + }); + endpoint_factory + .endpoint_config + .cid_generator(cid_generator.clone()); + let endpoint = endpoint_factory.endpoint(); + drop(endpoint_factory); + assert_eq!(Arc::strong_count(&cid_generator), 2); + drop(endpoint); + // Let the driver task run; paused runtimes are guaranteed to drain pending work on sleep. + sleep(Duration::from_millis(1)).await; + assert_eq!(Arc::strong_count(&cid_generator), 1); +} + +#[tokio::test] +async fn dropped_connection_cleans_up() { + let _guard = subscribe(); + let endpoint = endpoint(); + join!( + async { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .unwrap() + }, + async { endpoint.accept().await.unwrap().await.unwrap() } + ); + endpoint.wait_idle().await; +} + +#[derive(Default)] +struct WakeCounter { + wakes: AtomicUsize, +} + +impl WakeCounter { + fn wakes(&self) -> usize { + self.wakes.load(Ordering::SeqCst) + } +} + +fn new_count_waker() -> (Waker, Arc) { + // instance of WakeCounter + let counter = Arc::new(WakeCounter::default()); + + // convert + let waker = unsafe { Waker::from_raw(raw_waker(counter.clone())) }; + (waker, counter) +} + +fn raw_waker(counter: Arc) -> RawWaker { + // Store an Arc behind the raw pointer. + let ptr = Arc::into_raw(counter) as *const (); + RawWaker::new(ptr, &VTABLE) +} + +static VTABLE: RawWakerVTable = + RawWakerVTable::new(clone_waker, wake_waker, wake_by_ref_waker, drop_waker); + +unsafe fn clone_waker(data: *const ()) -> RawWaker { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + let cloned = arc.clone(); + std::mem::forget(arc); + raw_waker(cloned) +} + +unsafe fn wake_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + // arc drops here +} + +unsafe fn wake_by_ref_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + std::mem::forget(arc); +} + +unsafe fn drop_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + drop(unsafe { Arc::::from_raw(data as *const WakeCounter) }); +} diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs new file mode 100644 index 0000000000..c3c3d3551d --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -0,0 +1,224 @@ +use crate::{Duration, Instant}; + +/// Limits the amount of time spent on a certain type of work in a cycle +/// +/// The limiter works dynamically: For a sampled subset of cycles it measures +/// the time that is approximately required for fulfilling 1 work item, and +/// calculates the amount of allowed work items per cycle. +/// The estimates are smoothed over all cycles where the exact duration is measured. +/// +/// In cycles where no measurement is performed the previously determined work limit +/// is used. +/// +/// For the limiter the exact definition of a work item does not matter. +/// It could for example track the amount of transmitted bytes per cycle, +/// or the amount of transmitted datagrams per cycle. +/// It will however work best if the required time to complete a work item is +/// constant. +#[derive(Debug)] +pub(crate) struct WorkLimiter { + /// Whether to measure the required work time, or to use the previous estimates + mode: Mode, + /// The current cycle number + cycle: u16, + /// The time the cycle started - only used in measurement mode + start_time: Option, + /// How many work items have been completed in the cycle + completed: usize, + /// The amount of work items which are allowed for a cycle + allowed: usize, + /// The desired cycle time + desired_cycle_time: Duration, + /// The estimated and smoothed time per work item in nanoseconds + smoothed_time_per_work_item_nanos: f64, +} + +impl WorkLimiter { + pub(crate) fn new(desired_cycle_time: Duration) -> Self { + Self { + mode: Mode::Measure, + cycle: 0, + start_time: None, + completed: 0, + allowed: 0, + desired_cycle_time, + smoothed_time_per_work_item_nanos: 0.0, + } + } + + /// Starts one work cycle + pub(crate) fn start_cycle(&mut self, now: impl Fn() -> Instant) { + self.completed = 0; + if let Mode::Measure = self.mode { + self.start_time = Some(now()); + } + } + + /// Returns whether more work can be performed inside the `desired_cycle_time` + /// + /// Requires that previous work was tracked using `record_work`. + pub(crate) fn allow_work(&mut self, now: impl Fn() -> Instant) -> bool { + match self.mode { + Mode::Measure => (now() - self.start_time.unwrap()) < self.desired_cycle_time, + Mode::HistoricData => self.completed < self.allowed, + } + } + + /// Records that `work` additional work items have been completed inside the cycle + /// + /// Must be called between `start_cycle` and `finish_cycle`. + pub(crate) fn record_work(&mut self, work: usize) { + self.completed += work; + } + + /// Finishes one work cycle + /// + /// For cycles where the exact duration is measured this will update the estimates + /// for the time per work item and the limit of allowed work items per cycle. + /// The estimate is updated using the same exponential averaging (smoothing) + /// mechanism which is used for determining QUIC path rtts: The last value is + /// weighted by 1/8, and the previous average by 7/8. + pub(crate) fn finish_cycle(&mut self, now: impl Fn() -> Instant) { + // If no work was done in the cycle drop the measurement, it won't be useful + if self.completed == 0 { + return; + } + + if let Mode::Measure = self.mode { + let elapsed = now() - self.start_time.unwrap(); + + let time_per_work_item_nanos = (elapsed.as_nanos()) as f64 / self.completed as f64; + + // Calculate the time per work item. We set this to at least 1ns to avoid + // dividing by 0 when calculating the allowed amount of work items. + self.smoothed_time_per_work_item_nanos = if self.allowed == 0 { + // Initial estimate + time_per_work_item_nanos + } else { + // Smoothed estimate + (7.0 * self.smoothed_time_per_work_item_nanos + time_per_work_item_nanos) / 8.0 + } + .max(1.0); + + // Allow at least 1 work item in order to make progress + self.allowed = (((self.desired_cycle_time.as_nanos()) as f64 + / self.smoothed_time_per_work_item_nanos) as usize) + .max(1); + self.start_time = None; + } + + self.cycle = self.cycle.wrapping_add(1); + self.mode = match self.cycle % SAMPLING_INTERVAL { + 0 => Mode::Measure, + _ => Mode::HistoricData, + }; + } +} + +/// We take a measurement sample once every `SAMPLING_INTERVAL` cycles +const SAMPLING_INTERVAL: u16 = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Measure, + HistoricData, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[test] + fn limit_work() { + const CYCLE_TIME: Duration = Duration::from_millis(500); + const BATCH_WORK_ITEMS: usize = 12; + const BATCH_TIME: Duration = Duration::from_millis(100); + + const EXPECTED_INITIAL_BATCHES: usize = + (CYCLE_TIME.as_nanos() / BATCH_TIME.as_nanos()) as usize; + const EXPECTED_ALLOWED_WORK_ITEMS: usize = EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS; + + let mut limiter = WorkLimiter::new(CYCLE_TIME); + reset_time(); + + // The initial cycle is measuring + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, EXPECTED_ALLOWED_WORK_ITEMS); + let initial_time_per_work_item = limiter.smoothed_time_per_work_item_nanos; + + // The next cycles are using historic data + const BATCH_SIZES: [usize; 4] = [1, 2, 3, 5]; + for &batch_size in &BATCH_SIZES { + limiter.start_cycle(get_time); + let mut allowed_work = 0; + while limiter.allow_work(get_time) { + limiter.record_work(batch_size); + allowed_work += batch_size; + } + limiter.finish_cycle(get_time); + + assert_eq!(allowed_work, EXPECTED_ALLOWED_WORK_ITEMS); + } + + // After `SAMPLING_INTERVAL`, we get into measurement mode again + for _ in 0..(SAMPLING_INTERVAL as usize - BATCH_SIZES.len() - 1) { + limiter.start_cycle(get_time); + limiter.record_work(1); + limiter.finish_cycle(get_time); + } + + // We now do more work per cycle, and expect the estimate of allowed + // work items to go up + const BATCH_WORK_ITEMS_2: usize = 96; + const TIME_PER_WORK_ITEMS_2_NANOS: f64 = + CYCLE_TIME.as_nanos() as f64 / (EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS_2) as f64; + + let expected_updated_time_per_work_item = + (initial_time_per_work_item * 7.0 + TIME_PER_WORK_ITEMS_2_NANOS) / 8.0; + let expected_updated_allowed_work_items = + (CYCLE_TIME.as_nanos() as f64 / expected_updated_time_per_work_item) as usize; + + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS_2); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, expected_updated_allowed_work_items); + } + + thread_local! { + /// Mocked time + pub static TIME: RefCell = RefCell::new(Instant::now()); + } + + fn reset_time() { + TIME.with(|t| { + *t.borrow_mut() = Instant::now(); + }) + } + + fn get_time() -> Instant { + TIME.with(|t| *t.borrow()) + } + + fn advance_time(duration: Duration) { + TIME.with(|t| { + *t.borrow_mut() += duration; + }) + } +} diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs new file mode 100644 index 0000000000..c3dcee477b --- /dev/null +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -0,0 +1,378 @@ +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use crate::proto::ConnectRequest; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use noq::crypto::rustls::QuicClientConfig; +use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; +use tokio::net::lookup_host; +use url::Host; + +use crate::crypto; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::ALPN; +use crate::{ClientError, Session}; + +/// Congestion control algorithm to use for the connection. +/// +/// Different algorithms make different tradeoffs between throughput and latency. +pub enum CongestionControl { + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Client] using sane defaults. +/// +/// This is optional; advanced users may use [Client::new] directly. +#[derive(Clone)] +pub struct ClientBuilder { + provider: crypto::Provider, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ClientBuilder { + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self + .builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates( + self, + certs: Vec, + ) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes( + self, + hashes: Vec>, + ) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); + + let mut transport = noq::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ClientBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Builder for dangerous TLS configuration options. +/// +/// This builder provides access to potentially insecure TLS configurations. +/// These options should only be used when you understand the security implications, +/// such as in local development or over a secure VPN connection. +pub struct DangerousClientBuilder { + inner: ClientBuilder, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl DangerousClientBuilder { + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } +} + +/// A client for connecting to a WebTransport server. +#[derive(Clone, Debug)] +pub struct Client { + endpoint: noq::Endpoint, + config: noq::ClientConfig, +} + +impl Client { + /// Manually create a client via a Noq endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect( + &self, + request: impl Into, + ) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self + .endpoint + .connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for Client { + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } +} + +#[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] +#[derive(Debug)] +struct ServerFingerprints { + provider: crypto::Provider, + fingerprints: Vec>, +} + +impl ServerCertVerifier for ServerFingerprints { + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider + .signature_verification_algorithms + .supported_schemes() + } +} + +#[derive(Debug)] +pub struct NoCertificateVerification(Arc); + +impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs new file mode 100644 index 0000000000..4307355d3f --- /dev/null +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -0,0 +1,158 @@ +use std::ops::Deref; + +use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum ConnectError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), + + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), + + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), +} + +/// An HTTP/3 CONNECT request/response for establishing a WebTransport session. +pub struct Connecting { + // The request that was sent by the client. + pub request: ConnectRequest, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connecting { + pub async fn accept(conn: &noq::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { + request, + send, + recv, + }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond( + mut self, + response: impl Into, + ) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol { + if !self.request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } +} + +impl Deref for Connecting { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.request + } +} + +pub struct Connected { + // The request that was sent by the client. + pub request: ConnectRequest, + + // The response sent by the server. + pub response: ConnectResponse, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connected { + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol { + if !request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. + // We don't use the noq::VarInt because that would mean a noq dependency in + // web-transport-proto + let stream_id = noq::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } +} diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs new file mode 100644 index 0000000000..1b8bb8ecf1 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -0,0 +1,67 @@ +//! Simple crypto provider utilities for rustls. +//! +//! This module provides helper functions for working with rustls crypto providers, +//! supporting both ring and aws-lc-rs backends. + +use std::sync::Arc; + +use rustls::crypto::hash::{self, HashAlgorithm}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::CertificateDer; + +/// A shared reference to a crypto provider. +pub type Provider = Arc; + +/// Returns the default crypto provider. +/// +/// This function checks for a process-wide default provider first, +/// then falls back to feature-enabled providers (aws-lc-rs or ring). +/// +/// # Panics +/// +/// Panics if no provider is available. Either call `CryptoProvider::set_default()` +/// or enable exactly one of the `ring` or `aws-lc-rs` features. +pub fn default_provider() -> Provider { + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } + + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!( + "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." + ); + } +} + +/// Computes the SHA-256 hash of a certificate using the provided crypto provider. +/// +/// # Panics +/// +/// Panics if the provider doesn't expose a SHA-256 hash algorithm. +pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } + + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); +} diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs new file mode 100644 index 0000000000..654260fc4c --- /dev/null +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -0,0 +1,405 @@ +use std::sync::{Arc, OnceLock}; + +use thiserror::Error; + +use crate::{ConnectError, SettingsError}; + +/// An error returned when connecting to a WebTransport endpoint. +#[derive(Error, Debug, Clone)] +pub enum ClientError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error: {0}")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write: {0}")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read: {0}")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), + + #[error("quic error: {0}")] + NoqError(#[from] noq::ConnectError), + + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// WebTransport errors. +#[derive(Clone, Error, Debug)] +pub enum SessionError { + #[error("connection error: {0}")] + ConnectionError(noq::ConnectionError), + + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), + + #[error("send datagram error: {0}")] + SendDatagramError(#[from] noq::SendDatagramError), +} + +impl From for SessionError { + fn from(e: noq::ConnectionError) -> Self { + match &e { + noq::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } +} + +/// Why a session closed, shared by the session and its streams. The first close wins. +/// +/// It also knows whether the session is raw QUIC, which decides how its close and stream +/// codes are encoded. +#[derive(Debug)] +pub(crate) struct CloseReason { + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, +} + +impl CloseReason { + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { + if self.raw { + return code.into(); + } + noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { + Some(conn) + } + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + Err(_) => err, + } + } + _ => err, + } + } +} + +/// An error that can occur when reading/writing the WebTransport stream header. +#[derive(Clone, Error, Debug)] +pub enum WebTransportError { + #[error("closed: code={0} reason={1}")] + Closed(u32, String), + + #[error("unknown session")] + UnknownSession, + + #[error("read error: {0}")] + ReadError(#[from] noq::ReadExactError), + + #[error("write error: {0}")] + WriteError(#[from] noq::WriteError), +} + +/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +#[derive(Clone, Error, Debug)] +pub enum WriteError { + #[error("STOP_SENDING: {0}")] + Stopped(u32), + + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(noq::VarInt), + + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("stream closed")] + ClosedStream, +} + +impl From for WriteError { + fn from(e: noq::WriteError) -> Self { + match e { + noq::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + noq::WriteError::ClosedStream => WriteError::ClosedStream, + noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadError { + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("RESET_STREAM: {0}")] + Reset(u32), + + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(noq::VarInt), + + #[error("stream already closed")] + ClosedStream, +} + +impl From for ReadError { + fn from(value: noq::ReadError) -> Self { + match value { + noq::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + noq::ReadError::ClosedStream => ReadError::ClosedStream, + noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadExactError { + #[error("finished early")] + FinishedEarly(usize), + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadExactError { + fn from(e: noq::ReadExactError) -> Self { + match e { + noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } +} + +/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadToEndError { + #[error("too long")] + TooLong, + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadToEndError { + fn from(e: noq::ReadToEndError) -> Self { + match e { + noq::ReadToEndError::TooLong => ReadToEndError::TooLong, + noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } +} + +/// An error indicating the stream was already closed. +#[derive(Clone, Error, Debug)] +#[error("stream closed")] +pub struct ClosedStream; + +impl From for ClosedStream { + fn from(_: noq::ClosedStream) -> Self { + ClosedStream + } +} + +/// An error returned when receiving a new WebTransport session. +#[derive(Error, Debug, Clone)] +pub enum ServerError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), + + #[error("io error: {0}")] + IoError(Arc), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +// #[derive(Clone, Error, Debug)] +// pub enum SendDatagramError { +// #[error("Unsupported peer")] +// UnsupportedPeer, + +// #[error("Datagram support Disabled by peer")] +// DatagramSupportDisabled, + +// #[error("Datagram Too large")] +// TooLarge, + +// #[error("Session errorr: {0}")] +// SessionError(#[from] SessionError), +// } + +// impl From for SendDatagramError { +// fn from(value: noq::SendDatagramError) -> Self { +// match value { +// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// noq::SendDatagramError::ConnectionLost(e) => +// SendDatagramError::SessionError(e.into()), } +// } +// } + +impl web_transport_trait::Error for SessionError { + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } + + None + } +} + +impl web_transport_trait::Error for WriteError { + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } +} + +impl web_transport_trait::Error for ReadError { + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs new file mode 100644 index 0000000000..e5f3706492 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -0,0 +1,71 @@ +//! WebTransport is a protocol for client-server communication over QUIC. +//! It's [available in the browser](https://caniuse.com/webtransport) as an alternative to HTTP and WebSockets. +//! +//! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. +//! This library hides that detail and tries to expose only the QUIC API, delegating as much as +//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! +//! QUIC provides two primary APIs: +//! +//! # Streams +//! QUIC streams are ordered, reliable, flow-controlled, and optionally bidirectional. +//! Both endpoints can create and close streams (including an error code) with no overhead. +//! You can think of them as TCP connections, but shared over a single QUIC connection. +//! +//! # Datagrams +//! QUIC datagrams are unordered, unreliable, and not flow-controlled. +//! Both endpoints can send datagrams below the MTU size (~1.2kb minimum) and they might arrive out +//! of order or not at all. They are basically UDP packets, except they are encrypted and congestion +//! controlled. +//! +//! # Limitations +//! WebTransport is able to be pooled with HTTP/3 and multiple WebTransport sessions. +//! This crate avoids that complexity, doing the bare minimum to support a single WebTransport +//! session that owns the entire QUIC connection. If you want to support HTTP/3 on the same +//! host/port, you should use another crate (ex. `h3-webtransport`). If you want to support multiple +//! WebTransport sessions over the same QUIC connection... you should just dial a new QUIC +//! connection instead. + +// External +mod client; +mod error; +mod recv; +mod send; +mod server; +mod session; + +pub use client::*; +pub use error::*; +pub use recv::*; +pub use send::*; +pub use server::*; +pub use session::*; + +// Internal +mod connect; +mod settings; + +use connect::*; +use settings::*; + +// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +pub use connect::ConnectError; + +/// The HTTP/3 ALPN is required when negotiating a QUIC connection. +pub const ALPN: &str = "h3"; + +/// Export our simple crypto provider. +pub mod crypto; + +/// Re-export the underlying QUIC implementation. +pub use noq; + +/// Re-export the http crate because it's in the public API. +pub use http; + +/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of +/// [`SessionAccept`]'s poll methods holds its place in the accept queue. +pub use kio; + +/// Re-export the WebTransport protocol implementation. +pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs new file mode 100644 index 0000000000..4c3d188172 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -0,0 +1,159 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::Bytes; + +use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; + +/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +#[derive(Debug)] +pub struct RecvStream { + inner: noq::RecvStream, + close: Arc, +} + +impl RecvStream { + pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { + Self { + inner: stream, + close, + } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::ReadError) -> ReadError { + match e { + noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + noq::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. + pub async fn read_many_chunks( + &mut self, + bufs: &mut [Bytes], + ) -> Result, ReadError> { + self.inner + .read_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner + .read_to_end(size_limit) + .await + .map_err(|e| match e { + noq::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`noq::RecvStream::received_reset`]. + /// + /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.inner.id() + } + + /// Returns the number of bytes read from this stream. + /// + /// This is the offset of the next byte to be read, i.e. the length of the contiguous + /// prefix of the stream consumed by the application. + pub fn bytes_read(&self) -> Result { + self.inner.bytes_read().map_err(|_| ClosedStream) + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl web_transport_trait::RecvStream for RecvStream { + type Error = ReadError; + + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } + + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } + + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } +} diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs new file mode 100644 index 0000000000..39af0d1fc7 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -0,0 +1,197 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::{Buf, Bytes}; + +use crate::{CloseReason, ClosedStream, SessionError, WriteError}; + +/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// +/// This wrapper is mainly needed for error codes, which is unfortunate. +/// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. +#[derive(Debug)] +pub struct SendStream { + stream: noq::SendStream, + close: Arc, +} + +impl SendStream { + pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::WriteError) -> WriteError { + match e { + noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`noq::SendStream::stopped`]. + /// + /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. + /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(noq::StoppedError::ConnectionLost(conn_err)) => { + Err(self.close.map(conn_err.into())) + } + Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream + .write_all(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write chunks of data to the stream, returning the number of bytes written. + /// + /// See [`noq::SendStream::write_many_chunks`]. + pub async fn write_many_chunks( + &mut self, + bufs: &mut &mut [Bytes], + ) -> Result { + self.stream + .write_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream + .write_chunk(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream + .write_all_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`noq::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.stream.id() + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + // We have to use this syntax because noq added its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + +impl web_transport_trait::SendStream for SendStream { + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + // NOTE: This used to require &mut in an older version of Noq. + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } +} diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs new file mode 100644 index 0000000000..8af3e81c98 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -0,0 +1,200 @@ +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use std::sync::Arc; + +use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::{crypto, CongestionControl}; +use crate::{ + proto::{ConnectRequest, ConnectResponse}, + Connecting, ServerError, Session, Settings, +}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Server] using sane defaults. +/// +/// This is optional; advanced users may use [Server::new] directly. +pub struct ServerBuilder { + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ServerBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ServerBuilder { + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard Noq setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important + + let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = noq::ServerConfig::with_crypto(Arc::new(config)); + + let server = + noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } +} + +/// A WebTransport server that accepts new sessions. +pub struct Server { + endpoint: noq::Endpoint, + accept: FuturesUnordered>>, +} + +impl core::ops::Deref for Server { + type Target = noq::Endpoint; + + fn deref(&self) -> &Self::Target { + &self.endpoint + } +} + +impl Server { + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. + pub fn new(endpoint: noq::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } +} + +/// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to +/// accept or reject the session based on the URL. +pub struct Request { + conn: noq::Connection, + settings: Settings, + connect: Connecting, +} + +impl Request { + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: noq::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond( + self, + response: impl Into, + ) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &noq::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } +} + +impl core::ops::Deref for Request { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.connect + } +} diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs new file mode 100644 index 0000000000..82a9149d84 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -0,0 +1,1331 @@ +use std::{ + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use bytes::{Bytes, BytesMut}; +use futures::stream::{FuturesUnordered, Stream, StreamExt}; +use kio::{Fan, Waiter}; + +use crate::{ + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, + WebTransportError, +}; + +/// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake +/// has not completed, or it is not valid UTF-8. +/// +/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// is owned rather than borrowed from the connection. +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +fn negotiated_alpn(conn: &noq::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data.downcast::().ok()?; + + String::from_utf8(data.protocol?).ok() +} + +/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// handshake data type to name here. +/// +/// This crate's own builders are gated the same way, so normally there is no connection +/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// brings its own endpoint can still establish one, and [`Session::protocol`] reports +/// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to +/// get the ALPN. +#[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] +fn negotiated_alpn(_conn: &noq::Connection) -> Option { + None +} + +/// An established WebTransport session, acting like a full QUIC connection. See +/// [`noq::Connection`]. +/// +/// It is important to remember that WebTransport is layered on top of QUIC: +/// 1. Each stream starts with a few bytes identifying the stream type and session ID. +/// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. +/// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. +/// +/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// These should be safe to use with WebTransport, but file a PR if you find one that isn't. +#[derive(Clone)] +pub struct Session { + conn: noq::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, +} + +impl Session { + pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = noq::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { + code, + reason, + })) => return Some((code, reason)), + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self + .conn + .accept_uni() + .await + .map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self + .conn + .read_datagram() + .await + .map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = + VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = + web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .unwrap_or(Duration::from_millis(100)); + let timeout = (rtt * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session + .close_with_capsule(send, capsule, code, timeout) + .await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: noq::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!( + ?e, + "connection ended before the close capsule was acknowledged" + ); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()) + .await + .is_err() + { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`noq::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`noq::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: noq::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + let path = self.conn.path_stats(noq::PathId::ZERO); + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .or_else(|| path.map(|p| p.rtt)); + SessionStats { + stats: self.conn.stats(), + path, + rtt, + } + } +} + +impl Deref for Session { + type Target = noq::Connection; + + fn deref(&self) -> &Self::Target { + &self.conn + } +} + +impl fmt::Debug for Session { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } +} + +impl PartialEq for Session { + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } +} + +impl Eq for Session {} + +// Poll the shared accept state, then wake the *other* accepters once the lock is +// released. +// +// `SessionAccept` does not wake them itself: a waker is free to resume its task +// inline, and the first thing a resumed accepter does is take this same lock. An +// arrival or a failure is exactly what the others are parked waiting to retry +// after, so `Ready` is the signal. +fn poll_accept_uni_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +fn poll_accept_bi_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +// Type aliases just so clippy doesn't complain about the complexity. +type AcceptUni = dyn Stream> + Send; +type AcceptBi = + dyn Stream> + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = + dyn Future, SessionError>> + Send; + +// Logic just for accepting streams, which is annoying because of the stream header. +pub struct SessionAccept { + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, +} + +impl SessionAccept { + pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::new(); + let uni_waiters = Fan::new(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, noq::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi( + &mut self, + waiter: &Waiter, + ) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: noq::SendStream, + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } +} + +// Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any +// failure as a truncation, which would hide a stream the peer reset before its header +// arrived. +async fn read_varint(recv: &mut noq::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; + + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; + + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) +} + +// Without reliable reset, a stream the peer resets early loses its header with it, which +// moq does routinely. Anything else is a misbehaving peer. +fn log_header_error(err: SessionError, direction: &'static str) { + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError( + noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } +} + +pub struct SessionStats { + stats: noq::ConnectionStats, + path: Option, + rtt: Option, +} + +impl web_transport_trait::Stats for SessionStats { + fn bytes_sent(&self) -> Option { + Some(self.stats.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.stats.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + self.path.map(|path| path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.stats.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.stats.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + self.path.map(|path| path.lost_packets) + } + + fn rtt(&self) -> Option { + self.rtt + } + + fn estimated_send_rate(&self) -> Option { + let path = self.path?; + let rtt = self.rtt?; + let rtt_secs = rtt.as_secs_f64(); + if path.cwnd > 0 && rtt_secs > 0.0 { + Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) + } else { + None + } + } +} + +impl web_transport_trait::Session for Session { + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } +} + +#[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] +mod tests { + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl noq::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &noq::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl noq::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [noq::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + noq::Connection, + noq::Connection, + [noq::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( + &signing_key, + ))); + + let server_config = + noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let runtime: Arc = Arc::new(noq::TokioRuntime); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: runtime.wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = noq::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + runtime, + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { + crate::Request::accept(server_conn) + .await + .unwrap() + .ok() + .await + }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert!( + matches!( + err, + SessionError::WebTransportError(WebTransportError::UnknownSession) + ), + "{err:?}" + ); + } +} diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs new file mode 100644 index 0000000000..08ea9a53a1 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -0,0 +1,70 @@ +use futures::try_join; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum SettingsError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), + + #[error("WebTransport is not supported")] + WebTransportUnsupported, + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), +} + +pub struct Settings { + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: noq::SendStream, + + #[allow(dead_code)] + recv: noq::RecvStream, +} + +impl Settings { + // Establish the H3 connection. + pub async fn connect(conn: &noq::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); + + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } + + async fn accept(conn: &noq::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; + + tracing::debug!(?settings, "received SETTINGS frame"); + + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } + + Ok(recv) + } + + async fn open(conn: &noq::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); + + tracing::debug!(?settings, "sending SETTINGS frame"); + + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; + + Ok(send) + } +} From e8195bdb5952909497cf01bab7ea514c4eaf519f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:35:17 -0700 Subject: [PATCH 03/37] feat(quic)!: run moq-tokio and moq-uring on moq-quic Drop every moq-noq dependency. moq-tokio's QUIC backend now drives moq-quic through quinn's async layer (`quic::driver`) with web-transport-moq folded in as `quic::web`; moq-uring's sans-IO driver moves from moq-noq-proto to moq-quic. Upstream noq stays only behind moq-tokio's `iroh` feature. - The `noq` cargo feature is `quic` everywhere it was forwarded. - `moq_tokio::noq::Error` is `moq_tokio::quic::Error`; `Error::Noq` is `Error::Quic`. - moq-quic takes `moq_clock::Instant` and raises a once-per-connection Draining endpoint event, which backs the driver's `wait_all_draining`. - qlog writes one trace per endpoint (per dial on moq-uring's connect side), since moq-quic holds one sink per transport config. - Session stats report the congestion controller's bandwidth estimate. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 114 +- Cargo.toml | 12 +- doc/bin/cli.md | 2 +- doc/bin/relay/config.md | 2 +- doc/setup/upgrade.md | 7 + rs/hang/Cargo.toml | 2 +- rs/justfile | 10 +- rs/moq-bench/Cargo.toml | 4 +- rs/moq-boy/Cargo.toml | 4 +- rs/moq-c/Cargo.toml | 2 +- rs/moq-cli/Cargo.toml | 8 +- rs/moq-gst/src/lib.rs | 2 +- rs/moq-hls/Cargo.toml | 2 +- rs/moq-quic/Cargo.toml | 1 + rs/moq-quic/README.md | 15 +- rs/moq-quic/src/config/transport.rs | 15 +- rs/moq-quic/src/connection/mod.rs | 15 + rs/moq-quic/src/connection/qlog.rs | 16 +- rs/moq-quic/src/endpoint.rs | 1 + rs/moq-quic/src/lib.rs | 6 +- rs/moq-quic/src/shared.rs | 11 + rs/moq-quic/src/tests/mod.rs | 31 + rs/moq-quic/src/tests/util.rs | 15 + rs/moq-relay/Cargo.toml | 7 +- rs/moq-relay/src/config.rs | 2 +- rs/moq-relay/src/relay.rs | 6 +- rs/moq-relay/src/shutdown.rs | 2 +- rs/moq-relay/tests/cluster_idle.rs | 2 +- rs/moq-relay/tests/drills.rs | 2 +- rs/moq-relay/tests/goaway_cluster.rs | 2 +- rs/moq-relay/tests/runtime_uring.rs | 23 +- rs/moq-sock/src/bind.rs | 6 +- rs/moq-tokio/Cargo.toml | 39 +- rs/moq-tokio/README.md | 2 +- rs/moq-tokio/src/client.rs | 122 +- rs/moq-tokio/src/connect.rs | 4 +- rs/moq-tokio/src/connection.rs | 10 +- rs/moq-tokio/src/error.rs | 26 +- rs/moq-tokio/src/iroh.rs | 6 +- rs/moq-tokio/src/lib.rs | 12 +- rs/moq-tokio/src/listen.rs | 14 +- rs/moq-tokio/src/log.rs | 2 +- rs/moq-tokio/src/quic.rs | 33 +- rs/moq-tokio/src/{noq.rs => quic/backend.rs} | 289 +- rs/moq-tokio/src/quic/driver/.rustfmt.toml | 2 + rs/moq-tokio/src/quic/driver/connection.rs | 99 +- rs/moq-tokio/src/quic/driver/endpoint.rs | 178 +- rs/moq-tokio/src/quic/driver/incoming.rs | 8 +- rs/moq-tokio/src/quic/driver/mod.rs | 54 +- rs/moq-tokio/src/quic/driver/mutex.rs | 111 - rs/moq-tokio/src/quic/driver/recv_stream.rs | 59 +- rs/moq-tokio/src/quic/driver/runtime/mod.rs | 70 +- rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 - rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 53 +- rs/moq-tokio/src/quic/driver/send_stream.rs | 31 +- rs/moq-tokio/src/quic/driver/tests.rs | 205 +- rs/moq-tokio/src/quic/driver/work_limiter.rs | 2 +- rs/moq-tokio/src/quic/web/client.rs | 577 ++-- rs/moq-tokio/src/quic/web/connect.rs | 238 +- rs/moq-tokio/src/quic/web/crypto.rs | 64 +- rs/moq-tokio/src/quic/web/error.rs | 547 ++-- rs/moq-tokio/src/quic/web/fan.rs | 140 + rs/moq-tokio/src/quic/web/mod.rs | 12 +- rs/moq-tokio/src/quic/web/recv.rs | 246 +- rs/moq-tokio/src/quic/web/send.rs | 323 +-- rs/moq-tokio/src/quic/web/server.rs | 302 +-- rs/moq-tokio/src/quic/web/session.rs | 2399 ++++++++--------- rs/moq-tokio/src/quic/web/settings.rs | 84 +- rs/moq-tokio/src/resolve.rs | 2 +- rs/moq-tokio/src/server.rs | 146 +- rs/moq-tokio/src/tcp.rs | 2 +- rs/moq-tokio/src/tls.rs | 34 +- rs/moq-tokio/src/transport.rs | 2 +- rs/moq-tokio/src/websocket.rs | 8 +- rs/moq-tokio/src/worker/group.rs | 8 +- rs/moq-tokio/src/worker/mod.rs | 4 +- rs/moq-tokio/tests/backend.rs | 100 +- rs/moq-tokio/tests/broadcast.rs | 2 +- rs/moq-tokio/tests/close_code.rs | 2 +- rs/moq-tokio/tests/stream_code.rs | 12 +- rs/moq-tokio/tests/worker.rs | 2 +- rs/moq-transcode/Cargo.toml | 2 +- rs/moq-uring/Cargo.toml | 23 +- rs/moq-uring/README.md | 24 +- .../benches/{echo_noq.rs => echo_quic.rs} | 10 +- rs/moq-uring/benches/session_lite.rs | 10 +- rs/moq-uring/src/lib.rs | 4 +- .../src/quic/{noq => driver}/connection.rs | 133 +- .../src/quic/{noq => driver}/endpoint.rs | 42 +- rs/moq-uring/src/quic/{noq => driver}/mod.rs | 134 +- .../src/quic/{noq => driver}/stream.rs | 38 +- rs/moq-uring/src/quic/endpoint.rs | 2 +- rs/moq-uring/src/quic/mod.rs | 17 +- rs/moq-uring/src/quic/qlog.rs | 37 +- rs/moq-uring/src/quic/web.rs | 2 +- rs/moq-uring/tests/echo.rs | 12 +- rs/moq-uring/tests/ecn.rs | 2 +- rs/moq-uring/tests/endpoint.rs | 10 +- rs/moq-uring/tests/identity.rs | 2 +- rs/moq-uring/tests/qlog.rs | 6 +- rs/moq-uring/tests/session.rs | 4 +- rs/moq-uring/tests/teardown.rs | 2 +- rs/moq-uring/tests/web.rs | 49 +- rs/moq-uring/tests/workers.rs | 2 +- sh/rs/select.test.py | 2 +- sh/rs/tokio-features.sh | 10 +- 106 files changed, 3656 insertions(+), 4064 deletions(-) rename rs/moq-tokio/src/{noq.rs => quic/backend.rs} (79%) create mode 100644 rs/moq-tokio/src/quic/driver/.rustfmt.toml delete mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/web/fan.rs rename rs/moq-uring/benches/{echo_noq.rs => echo_quic.rs} (93%) rename rs/moq-uring/src/quic/{noq => driver}/connection.rs (87%) rename rs/moq-uring/src/quic/{noq => driver}/endpoint.rs (92%) rename rs/moq-uring/src/quic/{noq => driver}/mod.rs (79%) rename rs/moq-uring/src/quic/{noq => driver}/stream.rs (90%) diff --git a/Cargo.lock b/Cargo.lock index 0447ee73e6..a2a305b404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4675,72 +4675,6 @@ dependencies = [ name = "moq-net-sim-macros" version = "0.0.0" -[[package]] -name = "moq-noq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20d9d1334f556a15e60f6d406020fc3a40409335ae8a2ad472f6bf074a01acfa" -dependencies = [ - "bytes", - "cfg_aliases", - "derive_more 2.1.1", - "moq-noq-proto", - "moq-noq-udp", - "pin-project-lite", - "rustc-hash 2.1.3", - "rustls", - "socket2 0.6.5", - "thiserror 2.0.21", - "tokio", - "tokio-stream", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-proto" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50ac444c6c1ed18d45536cd939be7f0fd664996e79c0519119b13d6ae9d3d5e2" -dependencies = [ - "aes-gcm", - "aws-lc-rs", - "bytes", - "derive_more 2.1.1", - "enum-assoc", - "fastbloom", - "getrandom 0.4.3", - "identity-hash", - "lru-slab", - "n0-qlog", - "rand 0.10.3", - "rand_pcg", - "ring", - "rustc-hash 2.1.3", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "slab", - "sorted-index-buffer", - "thiserror 2.0.21", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-udp" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e978cded333ca8a6d50b96a7a4388ccc8d98db51d89df7dbee598e9b849ee79" -dependencies = [ - "cfg_aliases", - "libc", - "socket2 0.6.5", - "tracing", - "windows-sys 0.61.2", -] - [[package]] name = "moq-nvenc" version = "0.1.2" @@ -4771,6 +4705,7 @@ dependencies = [ "getrandom 0.4.3", "hex-literal", "lru-slab", + "moq-clock", "qlog", "rand 0.10.3", "rand_pcg", @@ -4835,7 +4770,6 @@ dependencies = [ "tracing-subscriber", "url", "usage-rs", - "web-transport-moq", "web-transport-trait", ] @@ -4992,19 +4926,26 @@ dependencies = [ "futures", "hex", "hmac 0.13.0", + "http", "humantime", "jni 0.22.4", + "kio 0.6.2", "libc", "mdns-sd", + "moq-clock", "moq-net", + "moq-quic", "moq-sock", "noq-proto", "notify", + "pin-project-lite", "qmux", "rand 0.10.3", "rcgen", "reqwest", + "rustc-hash 2.1.3", "rustls", + "rustls-native-certs", "rustls-platform-verifier", "rustls-webpki", "serde", @@ -5027,7 +4968,6 @@ dependencies = [ "url", "usage-rs", "web-transport-iroh", - "web-transport-moq", "web-transport-proto", "web-transport-trait", "webpki-roots", @@ -5064,8 +5004,9 @@ dependencies = [ "io-uring", "kio 0.6.2", "libc", + "moq-clock", "moq-net", - "moq-noq-proto", + "moq-quic", "moq-sock", "moq-tokio", "nix", @@ -5081,7 +5022,6 @@ dependencies = [ "tokio", "tracing", "url", - "web-transport-moq", "web-transport-proto", "web-transport-trait", ] @@ -5292,19 +5232,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "n0-qlog" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1c2fc610d1facce51b0bf5d8245b309c91996a933defdf7742c92ac78a58cb" -dependencies = [ - "humantime", - "serde", - "serde_json", - "serde_with", - "strum", -] - [[package]] name = "n0-watcher" version = "1.0.0" @@ -10240,27 +10167,6 @@ dependencies = [ "web-transport-trait", ] -[[package]] -name = "web-transport-moq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc946e718371f88a79d849c352b6bdd95e0a36139e9b76818c1a101b45ed97a" -dependencies = [ - "bytes", - "futures", - "http", - "kio 0.5.9", - "moq-noq", - "rustls", - "rustls-native-certs", - "thiserror 2.0.21", - "tokio", - "tracing", - "url", - "web-transport-proto", - "web-transport-trait", -] - [[package]] name = "web-transport-proto" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 19a3faf7ee..80f784458f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,7 @@ mdns-sd = { version = "0.21", features = ["async"] } moq-archive = { version = "0.0.11", path = "rs/moq-archive" } moq-audio = { version = "0.1.10", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.2.1", path = "rs/moq-auth" } +moq-clock = { version = "0.11.0", path = "rs/moq-clock" } moq-clock-macros = { version = "0.1.0", path = "rs/moq-clock/macros" } moq-flate = { version = "0.2.0", path = "rs/moq-flate" } moq-hls = { version = "0.5.11", path = "rs/moq-hls", default-features = false } @@ -157,14 +158,13 @@ moq-loc = { version = "0.2.19", path = "rs/moq-loc" } moq-msf = { version = "0.5.1", path = "rs/moq-msf" } moq-mux = { version = "0.10.11", path = "rs/moq-mux" } moq-net = { version = "0.3.10", path = "rs/moq-net" } -# The MoQ fork of noq (moq-dev/noq). iroh keeps upstream noq, so a build with the -# iroh feature carries both stacks. -moq-noq-proto = { version = "2.0.5", default-features = false } # NVENC bindings, forked from ViliamVadocz/nvidia-video-codec-sdk to dlopen the # driver at runtime. Compiles on any platform (macOS included) but only actually # used by moq-video on Linux. moq-nvenc = { version = "0.1.2", path = "rs/moq-nvenc" } moq-pattern = { version = "0.1.2", path = "rs/moq-pattern" } +# default-features off so each consumer picks its crypto provider. +moq-quic = { version = "0.1.0", path = "rs/moq-quic", default-features = false } moq-relay = { version = "0.17.2", path = "rs/moq-relay", default-features = false } moq-rtc = { version = "0.3.12", path = "rs/moq-rtc" } moq-rtmp = { version = "0.3.11", path = "rs/moq-rtmp" } @@ -178,7 +178,7 @@ moq-tokio = { version = "0.19.23", path = "rs/moq-tokio", default-features = fal # VAAPI is opt-in everywhere; its decoder is hardware-validated, while its # encoder is not yet. moq-transcode = { version = "0.1.10", path = "rs/moq-transcode", default-features = false } -# default-features off (the noq backend) so the consumer picks which QUIC +# default-features off (the `quic` backend) so the consumer opts into the QUIC # stack the io_uring path compiles; cargo features are additive, so a default-on # backend could not be opted out of. moq-uring = { version = "0.0.12", path = "rs/moq-uring", default-features = false } @@ -240,10 +240,6 @@ usage = { package = "usage-rs", version = "6.11.1" } uuid = { version = "1.26", features = ["v7"] } web-async = { version = "0.1.5", features = ["tracing"] } web-transport-iroh = "0.8.3" -# default-features off so the QUIC crypto provider is chosen by moq-tokio's -# aws-lc-rs / ring features. -# The WebTransport adapter released with moq-noq, from the same repository. -web-transport-moq = { version = "2.0.5", default-features = false } web-transport-proto = "0.6" web-transport-trait = "0.5" web-transport-wasm = "0.8" diff --git a/doc/bin/cli.md b/doc/bin/cli.md index d127621494..ac4f645cf2 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -168,7 +168,7 @@ Playback is behind the `play` feature, since it pulls in windowing and audio-device dependencies: ```bash -cargo install moq-cli --no-default-features --features "noq,websocket,play" +cargo install moq-cli --no-default-features --features "quic,websocket,play" ``` `moq play` skips disabled renditions. If none are enabled, it waits for a later catalog. Disabling the current rendition stops it and selects an enabled replacement. diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index f4a5099962..9bd91a4a5e 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -122,7 +122,7 @@ io_uring = false # Drive them with io_uring instead of tokio ``` Packets are steered by connection ID, so a client that migrates stays with its -worker. `workers` needs the `noq` feature. Every worker and the TLS stream +worker. `workers` needs the `quic` feature. Every worker and the TLS stream listener serve one shared identity, generated or file-backed, so a certificate rotation reaches them together and `/certificate.sha256` stays accurate. diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index dd5c605fdc..e0e6502d99 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -23,6 +23,13 @@ These land with the next breaking release, not the 2026-09-23 train. with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` orders audio by enabled state, bitrate, sample rate, channel count, and name. Both helpers put disabled renditions last. +- **The `noq` cargo feature is `quic`.** Rename it wherever you enable it on + moq-tokio, moq-uring, moq-relay, moq-cli, or the other crates that forward + it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` + crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and + `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one + file per endpoint instead of one per connection; on io_uring workers, one per + endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC `idle_timeout` and `keep_alive`; backoff `initial`, `max`, and `timeout`; diff --git a/rs/hang/Cargo.toml b/rs/hang/Cargo.toml index dd2f9d495b..24dfae2525 100644 --- a/rs/hang/Cargo.toml +++ b/rs/hang/Cargo.toml @@ -33,7 +33,7 @@ features = ["from", "display", "debug"] [dev-dependencies] anyhow = { workspace = true } moq-mux = { path = "../moq-mux" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } tokio = { workspace = true, features = ["macros", "fs"] } tracing = { workspace = true } url = { workspace = true } diff --git a/rs/justfile b/rs/justfile index 8f1bdf33cc..92fd223564 100644 --- a/rs/justfile +++ b/rs/justfile @@ -46,8 +46,8 @@ check-test *args: lint *args: python3 sh/rs/stats-docs.py cargo fmt --all --check - # quinn-udp keeps quinn's formatting; cargo fmt skips it (see moq-sock's lib.rs). - rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs + # quinn-udp and quinn keep quinn's formatting; cargo fmt skips them (see moq-sock's lib.rs, moq-tokio's quic.rs). + rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps "$@" cargo shear cargo sort --workspace --check --no-format @@ -57,7 +57,7 @@ lint *args: fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets "$@" cargo fmt --all - rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs + rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs cargo shear --fix cargo sort --workspace --no-format @@ -312,11 +312,11 @@ bench-udp *args: cargo bench -p moq-uring --bench udp_tokio -- "$@" cargo bench -p moq-uring --bench udp_uring -- "$@" -# Measure a noq echo through the io_uring worker, ablating receive +# Measure a QUIC echo through the io_uring worker, ablating receive # batching, GRO, and GSO one at a time. Linux 6.12+ (skips loudly otherwise). [positional-arguments] bench-echo *args: - cargo bench -p moq-uring --bench echo_noq -- "$@" + cargo bench -p moq-uring --bench echo_quic -- "$@" # Measure a full moq-lite session (publisher to subscriber) through the # io_uring worker, same ablations and per-iteration unit as bench-echo. diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index 0de559a2e6..80e9ed8095 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -35,9 +35,9 @@ required-features = ["uring"] doc = false [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] iroh = ["moq-tokio/iroh"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] uring = ["dep:moq-uring"] websocket = ["moq-tokio/websocket"] diff --git a/rs/moq-boy/Cargo.toml b/rs/moq-boy/Cargo.toml index cffec17902..ca15ca12d1 100644 --- a/rs/moq-boy/Cargo.toml +++ b/rs/moq-boy/Cargo.toml @@ -12,9 +12,9 @@ edition = "2024" rust-version.workspace = true [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] websocket = ["moq-tokio/websocket"] [dependencies] diff --git a/rs/moq-c/Cargo.toml b/rs/moq-c/Cargo.toml index c7564df387..a7cd65b3aa 100644 --- a/rs/moq-c/Cargo.toml +++ b/rs/moq-c/Cargo.toml @@ -26,7 +26,7 @@ moq-audio = { workspace = true, features = ["aac"] } moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } -moq-tokio = { workspace = true, features = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] } +moq-tokio = { workspace = true, features = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] } # Select the OpenH264 fallback plus NVIDIA and VAAPI hardware codecs explicitly # because workspace defaults are off. The hardware backends dlopen their driver; # `vaapi` also runs bindgen, so Linux builds need libclang. diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9e8d106693..14401ce224 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,13 +26,13 @@ path = "src/main.rs" doc = false [features] -default = ["cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] +default = ["cluster-lan", "quic", "websocket", "nvidia", "s3", "gcs", "azure"] # Experimental transport with a heavy dependency tree, enabled only on request. iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] # qlog trace capture (`--client-quic-qlog `). Off by default: it compiles the # qlog machinery into the enabled backends and is only wanted for debugging or # congestion-control testing. @@ -63,7 +63,7 @@ play = ["dep:moq-video", "moq-video/render", "dep:moq-audio", "moq-audio/playbac # the matching moq-video feature, so it only bites when something else pulls in # moq-video: `capture`, `transcode`, or `play`. For a build with no CUDA # dependencies, drop it: -# cargo build -p moq-cli --no-default-features --features "noq,websocket,capture" +# cargo build -p moq-cli --no-default-features --features "quic,websocket,capture" nvidia = ["moq-video?/nvidia", "moq-transcode?/nvidia"] # Compatibility aliases for the pre-consolidation names, kept so a manifest that # already asks for the old split still resolves. Undocumented on purpose: the @@ -112,7 +112,7 @@ moq-hls = { workspace = true, features = ["server"] } moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the -# CLI's own feature set (iroh, noq, cluster-lan) is what selects them. +# CLI's own feature set (iroh, quic, cluster-lan) is what selects them. moq-relay = { workspace = true } moq-rtc = { workspace = true } moq-rtmp = { workspace = true } diff --git a/rs/moq-gst/src/lib.rs b/rs/moq-gst/src/lib.rs index 57c518542e..aaaa5f1f89 100644 --- a/rs/moq-gst/src/lib.rs +++ b/rs/moq-gst/src/lib.rs @@ -22,7 +22,7 @@ pub fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { .with_default_directive(LevelFilter::INFO.into()) .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse().unwrap()) - .add_directive("noq=info".parse().unwrap()) + .add_directive("moq_tokio::quic::driver=info".parse().unwrap()) .add_directive("tracing::span=off".parse().unwrap()) .add_directive("tracing::span::active=off".parse().unwrap()); diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 60ab13990e..4439f49b3c 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -40,6 +40,6 @@ url = { workspace = true } async-trait = "0.1" futures = { workspace = true } moq-archive = { path = "../moq-archive" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-quic/Cargo.toml b/rs/moq-quic/Cargo.toml index a34ddfdadc..3580b37e95 100644 --- a/rs/moq-quic/Cargo.toml +++ b/rs/moq-quic/Cargo.toml @@ -56,6 +56,7 @@ aws-lc-rs = { version = "1.9", default-features = false, optional = true } bytes = { workspace = true } fastbloom = { version = "0.17", default-features = false, optional = true } lru-slab = "0.1.2" +moq-clock = { workspace = true } qlog = { version = "0.18", optional = true } rand = { workspace = true } rand_pcg = "0.10" diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 789d1b1b44..d47b7629b0 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -34,6 +34,17 @@ Changes to its `Cargo.toml` and `build.rs` go into moq-sock's by hand, and its b The module logs through `tracing` only, so quinn-udp's `log` feature and no-op logger are dropped. A `.rustfmt.toml` in each imported directory keeps quinn's formatting; `cargo fmt` cannot apply it to a submodule, so moq-sock skips `udp` and `just rs fix` formats it separately. +quinn itself, the async layer from the same commit, is the `quic::driver` module of [moq-tokio](../moq-tokio), formatted the same way: + +```sh +git -C ../quinn format-patch -1 --stdout -- quinn/src \ + | sed -E 's#([ab])/quinn/src/lib\.rs#\1/rs/moq-tokio/src/quic/driver/mod.rs#g; s#([ab])/quinn/src/#\1/rs/moq-tokio/src/quic/driver/#g; s/\bcrate::/crate::quic::driver::/g; s/\bproto::/moq_quic::/g; s/\budp::/moq_sock::udp::/g' \ + | git am -3 +``` + +The driver is tokio-only: the `Runtime` trait, smol, the `futures-io` impls, and the lock-tracking mutex are dropped, so hunks touching them are skipped. +moq-tokio's `quic::web` module is `web-transport-moq` from moq-dev/noq at [`e27c0ce0`](https://github.com/moq-dev/noq/commit/e27c0ce00741d81c8620b98e345d4f2c88e0768c), which has no upstream left to track. + ### Carried changes Changes on top of the upstream commit, besides the renames: @@ -51,7 +62,9 @@ Changes on top of the upstream commit, besides the renames: - Apple fast-path fallback preserves the segmentation of batches prepared before the fast path was disabled, including when a private symbol is unavailable. - Discarding Initial or Handshake keys resets the PTO backoff, as RFC 9002 A.11 does, so a client's Initial backoff no longer delays the retransmission of a lost Finished. Also [moq-dev/noq#32](https://github.com/moq-dev/noq/pull/32); not offered upstream yet. - `TransportConfig::handshake_idle_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. - quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the switch's import has to set both timeouts there. + quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. +- **Instants on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. +- **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/transport.rs b/rs/moq-quic/src/config/transport.rs index eb72d81385..48e26c5b66 100644 --- a/rs/moq-quic/src/config/transport.rs +++ b/rs/moq-quic/src/config/transport.rs @@ -1,16 +1,16 @@ use std::{fmt, sync::Arc}; #[cfg(feature = "qlog")] -use std::{io, sync::Mutex, time::Instant}; +use std::{io, sync::Mutex}; #[cfg(feature = "qlog")] use qlog::streamer::QlogStreamer; -#[cfg(feature = "qlog")] -use crate::QlogStream; use crate::{ Duration, INITIAL_MTU, MAX_UDP_PAYLOAD, VarInt, VarIntBoundsExceeded, congestion, connection::qlog::QlogSink, }; +#[cfg(feature = "qlog")] +use crate::{Instant, QlogStream}; /// Parameters governing the core QUIC state machine /// @@ -634,10 +634,12 @@ impl QlogConfig { vec![], ); + // The streamer's own epoch only times the events it stamps itself, which this + // crate never asks it to; `QlogStream` stamps them on MoQ's clock instead. let mut streamer = QlogStreamer::new( self.title, self.description, - self.start_time, + std::time::Instant::now(), trace, qlog::events::EventImportance::Core, // `Instant` should have sub-microsecond precision on most platforms @@ -646,7 +648,10 @@ impl QlogConfig { ); match streamer.start_log() { - Ok(()) => Some(QlogStream(Arc::new(Mutex::new(streamer)))), + Ok(()) => Some(QlogStream { + streamer: Arc::new(Mutex::new(streamer)), + start: self.start_time, + }), Err(e) => { warn!("could not initialize endpoint qlog streamer: {e}"); None diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 0ad0ddbbb4..e828b46055 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -171,6 +171,8 @@ pub struct Connection { retry_src_cid: Option, events: VecDeque, endpoint_events: VecDeque, + /// Whether `EndpointEventInner::Draining` was emitted, which happens once. + draining_reported: bool, /// Whether the spin bit is in use for this connection spin_enabled: bool, /// Outgoing spin bit state @@ -317,6 +319,7 @@ impl Connection { retry_src_cid: None, events: VecDeque::new(), endpoint_events: VecDeque::new(), + draining_reported: false, spin_enabled: config.allow_spin && rng.random_ratio(7, 8), spin: false, spaces: [initial_space, PacketSpace::new(now), PacketSpace::new(now)], @@ -1256,6 +1259,7 @@ impl Connection { match timer { Timer::Close => { self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } Timer::Idle => { @@ -2625,6 +2629,9 @@ impl Connection { self.set_close_timer(now); } } + if matches!(self.state, State::Draining | State::Drained) { + self.report_draining(); + } if !was_drained && self.state.is_drained() { self.endpoint_events.push_back(EndpointEventInner::Drained); // Close timer may have been started previously, e.g. if we sent a close and got a @@ -3968,9 +3975,17 @@ impl Connection { self.close_common(); self.error = Some(reason); self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } + /// Tell the endpoint the connection is no longer active, the first time it stops being so + fn report_draining(&mut self) { + if !mem::replace(&mut self.draining_reported, true) { + self.endpoint_events.push_back(EndpointEventInner::Draining); + } + } + /// Storage size required for the largest packet known to be supported by the current path /// /// Buffers passed to [`Connection::poll_transmit`] should be at least this large. diff --git a/rs/moq-quic/src/connection/qlog.rs b/rs/moq-quic/src/connection/qlog.rs index 6120e4e6f9..0294a69516 100644 --- a/rs/moq-quic/src/connection/qlog.rs +++ b/rs/moq-quic/src/connection/qlog.rs @@ -27,17 +27,23 @@ use crate::{ /// Shareable handle to a single qlog output stream #[cfg(feature = "qlog")] #[derive(Clone)] -pub struct QlogStream(pub(crate) Arc>); +pub struct QlogStream { + pub(crate) streamer: Arc>, + /// The epoch event times count from, on MoQ's clock rather than the std one the + /// streamer keeps. + pub(crate) start: Instant, +} #[cfg(feature = "qlog")] impl QlogStream { fn emit_event(&self, orig_rem_cid: ConnectionId, event: EventData, now: Instant) { - // Time will be overwritten by `add_event_with_instant` - let mut event = Event::with_time(0.0, event); + // Milliseconds since the epoch, at the nanosecond precision the streamer is built with. + let time = now.saturating_duration_since(self.start).as_nanos() as f64 / 1e6; + let mut event = Event::with_time(time, event); event.group_id = Some(Box::new(orig_rem_cid.to_string())); - let mut qlog_streamer = self.0.lock().unwrap(); - if let Err(e) = qlog_streamer.add_event_with_instant(event, now) { + let mut qlog_streamer = self.streamer.lock().unwrap(); + if let Err(e) = qlog_streamer.add_event(event) { warn!("could not emit qlog event: {e}"); } } diff --git a/rs/moq-quic/src/endpoint.rs b/rs/moq-quic/src/endpoint.rs index 616afffd82..19496b4478 100644 --- a/rs/moq-quic/src/endpoint.rs +++ b/rs/moq-quic/src/endpoint.rs @@ -126,6 +126,7 @@ impl Endpoint { } } } + Draining => {} Drained => { if let Some(conn) = self.connections.try_remove(ch.0) { self.index.remove(&conn); diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index 65ddf62da2..a5c39cb957 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -105,10 +105,12 @@ pub use token_memory_cache::TokenMemoryCache; use arbitrary::Arbitrary; // Deal with time +// MoQ's clock, so a caller-driven clock (a test's or the simulator's) reaches the state machine. +pub(crate) use moq_clock::Instant; #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] -pub(crate) use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(all(target_family = "wasm", target_os = "unknown"))] -pub(crate) use web_time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use web_time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(fuzzing)] pub mod fuzzing { diff --git a/rs/moq-quic/src/shared.rs b/rs/moq-quic/src/shared.rs index f2d0ad5d2a..047c6d7ce0 100644 --- a/rs/moq-quic/src/shared.rs +++ b/rs/moq-quic/src/shared.rs @@ -45,10 +45,21 @@ impl EndpointEvent { pub fn is_drained(&self) -> bool { self.0 == EndpointEventInner::Drained } + + /// Determine whether the connection has stopped being active + /// + /// Emitted once per connection, before [`Self::is_drained`]'s event: when the peer's close + /// arrives, or at the drain if it never does. Past it the peer has our close or never will, + /// so a shutdown can stop waiting without sitting out the closing period. + pub fn is_draining(&self) -> bool { + self.0 == EndpointEventInner::Draining + } } #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum EndpointEventInner { + /// The connection has stopped being active; see [`EndpointEvent::is_draining`] + Draining, /// The connection has been drained Drained, /// The reset token and/or address eligible for generating resets has been updated diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index fedc9af91d..60a8e74199 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -2004,6 +2004,37 @@ fn initial_retransmit() { ); } +/// Each side tells its endpoint once that the connection stopped being active, before it drains, +/// so a shutdown waiting on every connection can stop at the first of these +#[test] +fn draining_is_reported_once_before_drained() { + let _guard = subscribe(); + let mut pair = Pair::default(); + let (client_ch, server_ch) = pair.connect(); + let now = pair.time; + pair.client_conn_mut(client_ch) + .close(now, VarInt(0), Bytes::new()); + pair.drive(); + + let expected = [Lifecycle::Draining, Lifecycle::Drained]; + let client: Vec<_> = pair + .client + .lifecycle + .iter() + .filter(|(ch, _)| *ch == client_ch) + .map(|(_, l)| *l) + .collect(); + let server: Vec<_> = pair + .server + .lifecycle + .iter() + .filter(|(ch, _)| *ch == server_ch) + .map(|(_, l)| *l) + .collect(); + assert_eq!(client, expected); + assert_eq!(server, expected); +} + #[test] fn instant_close_1() { let _guard = subscribe(); diff --git a/rs/moq-quic/src/tests/util.rs b/rs/moq-quic/src/tests/util.rs index e64c1ccf09..f98c7f5eca 100644 --- a/rs/moq-quic/src/tests/util.rs +++ b/rs/moq-quic/src/tests/util.rs @@ -397,6 +397,14 @@ pub(super) struct TestEndpoint { pub(super) capture_inbound_packets: bool, pub(super) handle_incoming: Box IncomingConnectionBehavior>, pub(super) waiting_incoming: Vec, + /// Every draining and drained endpoint event, in the order connections raised them + pub(super) lifecycle: Vec<(ConnectionHandle, Lifecycle)>, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum Lifecycle { + Draining, + Drained, } #[derive(Debug, Copy, Clone)] @@ -441,6 +449,7 @@ impl TestEndpoint { capture_inbound_packets: false, handle_incoming: Box::new(|_| IncomingConnectionBehavior::Accept), waiting_incoming: Vec::new(), + lifecycle: Vec::new(), } } @@ -536,6 +545,12 @@ impl TestEndpoint { } for (ch, event) in endpoint_events { + if event.is_draining() { + self.lifecycle.push((ch, Lifecycle::Draining)); + } + if event.is_drained() { + self.lifecycle.push((ch, Lifecycle::Drained)); + } if let Some(event) = self.handle_event(ch, event) && let Some(conn) = self.connections.get_mut(&ch) { diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 8b1a73eb74..7791f78594 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,12 +25,12 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] -default = ["cluster-lan", "noq", "websocket", "uds"] +default = ["cluster-lan", "quic", "websocket", "uds"] iroh = ["moq-tokio/iroh"] # Discover cluster peers on the LAN with mDNS (`--cluster-lan`). cluster-lan = ["moq-tokio/mdns"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["_quic", "moq-tokio/noq"] +quic = ["_quic", "moq-tokio/quic"] # The tokio QUIC listener, and the thread-per-core worker group behind # `runtime.workers`. Not one to enable on its own: it names no backend, so # moq-tokio would have nothing to bind a socket with. @@ -46,7 +46,7 @@ websocket = ["moq-tokio/websocket", "dep:qmux", "axum/ws"] uds = ["moq-tokio/uds"] # Serve QUIC from io_uring workers (`--runtime-io-uring`), Linux-only. # This is off by default: the relay is useful without io_uring. -io-uring = ["_uring", "moq-uring/noq"] +io-uring = ["_uring", "moq-uring/quic"] # The listener machinery shared by the feature above. _uring = ["dep:moq-sock", "dep:moq-uring"] @@ -101,7 +101,6 @@ rcgen = "0.14" tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } -web-transport-moq = { workspace = true } # Which sockets the process still holds, so the embed tests can prove a stopped # relay closed its own without rebinding a port another test could take. diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 2750f1c024..707ffbc379 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1570,7 +1570,7 @@ uid = [1001] "cannot be combined", ); - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] { let path = dir.join("typed.toml"); std::fs::write(&path, "[listen]\nload_balancer = { id = \"ab\", nonce = 8 }\n").unwrap(); diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index fdf4908f6d..8f463f4c3a 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -171,7 +171,7 @@ impl Relay { #[cfg(not(feature = "_quic"))] anyhow::ensure!( io_uring || config.runtime.workers().is_none(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); // Workers and the stream server split one relay across listeners, so they share @@ -552,7 +552,7 @@ impl Relay { // Before the readiness notify, for the same reason the cluster starts // before it: an unsupported kernel, a refused ring, or a certificate - // certificate identity noq will not load fails here, and reporting ready first would + // certificate identity rustls will not load fails here, and reporting ready first would // release the units depending on a relay that is about to exit. #[cfg(all(target_os = "linux", feature = "_uring"))] let mut uring = uring; @@ -929,7 +929,7 @@ mod tests { }; assert_eq!( error.to_string(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); } } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 12476d1453..1cc1461dc8 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -285,7 +285,7 @@ mod tests { /// A WebSocket peer that upgrades and never sends SETUP ends at the drain /// deadline even with the SETUP timeout disabled, so its task does not /// outlive the relay. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test] async fn withheld_websocket_setup_ends_at_the_deadline() { let window = Duration::from_secs(1); diff --git a/rs/moq-relay/tests/cluster_idle.rs b/rs/moq-relay/tests/cluster_idle.rs index fc90ee1b7e..981fb408fe 100644 --- a/rs/moq-relay/tests/cluster_idle.rs +++ b/rs/moq-relay/tests/cluster_idle.rs @@ -2,7 +2,7 @@ //! //! A crashed or partitioned peer sends no close, so the idle timeout is the whole //! outage window for subscribes routed through it. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::time::{Duration, Instant}; diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index e699dc9c4e..bce5ce6144 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -23,7 +23,7 @@ //! See `test/drill/README.md` for the recipe, the sensitivity proof that each //! drill fails when its recovery behavior is removed, and the loom/fuzz cases //! covering the primitives underneath. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 726479103b..c7c0f47ac1 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -33,7 +33,7 @@ async fn within(step: &str, fut: impl std::future::Future) -> T { /// Run an integration-test future on a dedicated thread with a large stack. /// /// Under `--all-features`, `moq-tokio` compiles every transport -/// (noq, iroh, websocket, and qmux) into its `Session`/`Client` types. +/// (QUIC, iroh, websocket, and qmux) into its `Session`/`Client` types. /// These multi-relay tests hold several such values live across await points, /// so the single test future's state machine is large, and in an unoptimized /// build it overflows libtest's default 2 MiB per-test thread stack (a SIGABRT diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 5494fc4eea..5d66675801 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -763,10 +763,10 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { /// `listen.timeout` bounds every handshake phase from the first packet: TLS, /// HTTP/3 before CONNECT, and MoQ SETUP over raw QUIC and WebTransport. Each /// phase closes in its own error space. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] async fn uring_workers_bound_stalled_handshakes() { - use web_transport_moq::noq; + use moq_tokio::quic::{driver, web}; use web_transport_trait::Error as _; let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -788,10 +788,10 @@ async fn uring_workers_bound_stalled_handshakes() { let crypto = |alpn: &[u8]| { let mut tls = tls.clone(); tls.alpn_protocols = vec![alpn.to_vec()]; - let crypto = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); - noq::ClientConfig::new(std::sync::Arc::new(crypto)) + let crypto = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); + driver::ClientConfig::new(std::sync::Arc::new(crypto)) }; - let client = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); + let client = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); // TLS: the relay never hears more than the ClientHello, so only its own // deadline can end the handshake before the idle timeout does. @@ -806,7 +806,7 @@ async fn uring_workers_bound_stalled_handshakes() { .await .expect("listener did not bound the QUIC handshake"); assert!( - !matches!(error, noq::ConnectionError::TimedOut), + !matches!(error, driver::ConnectionError::TimedOut), "the relay must close a stalled QUIC handshake: {error}" ); @@ -823,11 +823,10 @@ async fn uring_workers_bound_stalled_handshakes() { .unwrap(); if scheme == "https" { let url: url::Url = format!("https://{addr}/stalled").parse().unwrap(); - let request = - web_transport_moq::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); - web_transport_moq::Session::connect(conn, request).await.unwrap() + let request = web::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); + web::Session::connect(conn, request).await.unwrap() } else { - web_transport_moq::Session::raw(conn) + web::Session::raw(conn) } }) .await @@ -842,7 +841,7 @@ async fn uring_workers_bound_stalled_handshakes() { "{scheme}: {error}" ); } - client.close(noq::VarInt::from_u32(0), b"test complete"); + client.close(driver::VarInt::from_u32(0), b"test complete"); client.wait_idle().await; running.abort(); let _ = running.await; @@ -851,7 +850,7 @@ async fn uring_workers_bound_stalled_handshakes() { /// A UDP hop to `server` that forwards one client's datagrams until the server /// first answers, and every answer back, so the server never sees the client /// finish its handshake. Returns the address to dial. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn stall_after_hello(server: SocketAddr) -> SocketAddr { let socket = tokio::net::UdpSocket::bind("127.0.0.1:0").await.expect("bind hop"); let addr = socket.local_addr().expect("hop address"); diff --git a/rs/moq-sock/src/bind.rs b/rs/moq-sock/src/bind.rs index ebd1e30851..13471fb997 100644 --- a/rs/moq-sock/src/bind.rs +++ b/rs/moq-sock/src/bind.rs @@ -27,7 +27,7 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); /// UDP socket buffer size requested in each direction, in bytes. /// /// A QUIC stack absorbs bursts in the kernel socket buffer: whatever doesn't fit -/// while the process is off the CPU is dropped before noq ever sees it, and +/// while the process is off the CPU is dropped before QUIC ever sees it, and /// congestion control reads those drops as congestion. The OS defaults are sized /// for a chatty TCP-era socket (208 KiB on Linux), which a single relay socket /// carrying every connection blows through in milliseconds. 8 MiB is roughly 64ms @@ -317,7 +317,7 @@ pub fn tcp(addr: SocketAddr) -> io::Result { /// A specific IPv6 address such as `[::1]` can never carry IPv4, so it is set /// v6-only instead, which Linux does on bind anyway. Windows keeps the cleared /// flag, so the socket claims an IPv4 side it lacks and then rejects the -/// IPv4-level options noq sets on a dual-stack socket with `WSAEINVAL`, failing +/// IPv4-level options the QUIC driver sets on a dual-stack socket with `WSAEINVAL`, failing /// endpoint setup. fn make_dual_stack(socket: &Socket, addr: SocketAddr) { let SocketAddr::V6(addr) = addr else { return }; @@ -364,7 +364,7 @@ mod tests { } /// A specific IPv6 address can't carry IPv4, so the socket must not claim to: - /// Windows rejects noq's IPv4 socket options on a dual-stack `[::1]` socket. + /// Windows rejects the QUIC driver's IPv4 socket options on a dual-stack `[::1]` socket. /// Linux forces v6-only on such a bind itself, so only Windows can regress. #[test] fn udp_ipv6_loopback_is_v6_only() { diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 45ad002eab..9664cb7787 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -13,8 +13,24 @@ keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] [features] -default = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] -noq = ["_transport", "_certs", "dep:web-transport-moq", "dep:reqwest", "watch", "dep:dns-lookup"] +default = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] +# QUIC and WebTransport over moq-quic, for `https://`, `moqt://`, and `moql://`. +quic = [ + "_transport", + "_certs", + "dep:moq-quic", + "dep:moq-clock", + "dep:http", + "dep:kio", + "dep:pin-project-lite", + "dep:rustc-hash", + "dep:rustls-native-certs", + "dep:socket2", + "dep:web-transport-proto", + "dep:reqwest", + "watch", + "dep:dns-lookup", +] # Server-side certificate handling: chains loaded from disk, self-signed generation, # in-memory identities, and per-SNI selection. Not one to enable on its own, since # nothing turns it into a listener: every QUIC backend needs it for its own TLS, and @@ -28,12 +44,12 @@ _transport = [] watch = ["dep:notify"] # Capture qlog traces (`--*-quic-qlog `). Off by default so production # builds do not pull in the qlog crate and its serde_json encoder. -qlog = ["web-transport-moq?/qlog"] +qlog = ["moq-quic?/qlog"] # The QUIC backend pulls in its crypto provider from these features (default-features # are off on the dependency), so a backend without aws-lc-rs/ring has no initial cipher. # A provider is also all it takes to sign and serve certificates, so each turns on # `_certs` even where there is no QUIC backend to serve them for. -aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "web-transport-moq?/aws-lc-rs"] +aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "moq-quic?/rustls-aws-lc-rs"] iroh = ["_transport", "dep:web-transport-iroh", "dep:web-transport-proto", "dep:noq-proto"] # LAN peer discovery over mDNS (the `mdns` module). mdns = ["dep:mdns-sd", "dep:hmac", "dep:sha2", "dep:subtle"] @@ -44,7 +60,7 @@ tcp = ["_transport", "dep:qmux", "dep:dns-lookup"] # Unix-domain-socket qmux transport (`unix://`), unix-only. Adds peer-credential # inspection on accept. Pulls in `tcp` (shares qmux's stream transport). uds = ["tcp", "qmux/uds"] -ring = ["_certs", "rustls/ring", "rcgen/ring", "web-transport-moq?/ring"] +ring = ["_certs", "rustls/ring", "rcgen/ring", "moq-quic?/rustls-ring"] [dependencies] bytes = "1" @@ -54,27 +70,37 @@ dns-lookup = { version = "4", optional = true } futures = "0.3" hex = { workspace = true } hmac = { workspace = true, optional = true } +http = { version = "1", optional = true } humantime = "2.3" +kio = { workspace = true, optional = true } mdns-sd = { workspace = true, optional = true } +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } +# The QUIC state machine the `quic` module drives; the crypto-provider features pick its provider. +moq-quic = { workspace = true, features = ["bloom"], optional = true } moq-sock = { workspace = true } -# iroh runs on upstream noq, not the moq-noq fork, and re-exports only the +# iroh runs on upstream noq rather than moq-quic, and re-exports only the # ControllerFactory trait, not the concrete congestion configs. Version matched to # the copy iroh pulls in. noq-proto = { workspace = true, optional = true } notify = { version = "8", optional = true } +pin-project-lite = { version = "0.2", optional = true } qmux = { workspace = true, features = ["wss", "tls", "tcp"], optional = true } rand = "0.10.1" rcgen = { version = "0.14", default-features = false, optional = true } reqwest = { version = "0.13", default-features = false, optional = true } +rustc-hash = { workspace = true, optional = true } rustls = "0.23" +# The WebTransport client trusts the OS store alongside its custom roots. +rustls-native-certs = { version = "0.8", optional = true } # OS-native server-cert verifier for the rustls backend. rustls-platform-verifier = "0.7" rustls-webpki = { version = "0.103", features = ["aws-lc-rs"], optional = true } serde = { version = "1", features = ["derive"] } serde_with = { version = "3", features = ["hex"] } sha2 = { workspace = true, optional = true } +socket2 = { workspace = true, optional = true } subtle = { workspace = true, optional = true } thiserror = "2" tikv-jemalloc-ctl = { version = "0.6", optional = true } @@ -87,7 +113,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = { version = "2", features = ["serde"] } usage = { workspace = true, features = ["config"] } web-transport-iroh = { workspace = true, optional = true } -web-transport-moq = { workspace = true, optional = true } web-transport-proto = { workspace = true, optional = true } web-transport-trait = { workspace = true } x509-parser = "0.18" diff --git a/rs/moq-tokio/README.md b/rs/moq-tokio/README.md index 4802a6760b..3ae2db1988 100644 --- a/rs/moq-tokio/README.md +++ b/rs/moq-tokio/README.md @@ -12,7 +12,7 @@ Tokio-based connection helpers for native [Media over QUIC](https://moq.dev) app Establishes MoQ connections over a few different transports, selectable via cargo features: -- **WebTransport** (HTTP/3) via [noq](https://crates.io/crates/noq) +- **WebTransport** (HTTP/3) via [moq-quic](https://crates.io/crates/moq-quic) - **Raw QUIC** with ALPN negotiation - **WebSocket** as a fallback when QUIC isn't available - **Iroh** P2P (`iroh` feature) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 20c71f8b58..66166b129b 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -6,7 +6,7 @@ use crate::connection::Goaway; use crate::{Addrs, Backoff, Connection, Error}; -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] use std::future::Future; use url::Url; @@ -47,7 +47,7 @@ pub struct Client { /// raw TCP/UDS qmux and WebSocket). Resolved once in [`Client::new`] so the ALPN list /// can't diverge between transports. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -81,10 +81,10 @@ pub struct Client { #[cfg(feature = "websocket")] tls_host_name: Option, /// Only the TLS-based dials read this; the plaintext qmux transports have none. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls: rustls::ClientConfig, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "iroh")] iroh: Option, #[cfg(feature = "iroh")] @@ -98,7 +98,7 @@ impl Client { #[cfg(not(feature = "_transport"))] pub fn new(_config: Config) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -121,11 +121,11 @@ impl Client { // Only the rustls-backed transports use this. Iroh and the plaintext // qmux transports must not require a rustls crypto provider. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] let tls = config.tls.build()?; - #[cfg(feature = "noq")] - let noq = Some(crate::noq::NoqClient::new(&config, &quic)?); + #[cfg(feature = "quic")] + let quic = Some(crate::quic::Client::new(&config, &quic)?); let versions = config.versions(); // Read before the struct literal below moves fields out of `config`. @@ -145,7 +145,7 @@ impl Client { .with_versions(versions.clone()) .with_extensions(config.extensions), #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -168,10 +168,10 @@ impl Client { websocket: config.websocket, #[cfg(feature = "websocket")] tls_host_name, - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "iroh")] iroh: None, #[cfg(feature = "iroh")] @@ -278,13 +278,13 @@ impl Client { /// This closes at once, discarding stream data the peer has not acknowledged yet. /// Call [`Connection::close`] on each connection first to deliver it. /// - /// Only the noq endpoint is closed. WebSocket, TCP, and UDS sessions end when their + /// Only the QUIC endpoint is closed. WebSocket, TCP, and UDS sessions end when their /// [`Connection`] is dropped (the kernel closes the socket on exit), and an iroh /// endpoint passed to `with_iroh` is closed by its owner. pub async fn close(self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq { - noq.close().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic { + quic.close().await; } } @@ -315,7 +315,7 @@ impl Client { /// /// Errors if no transport feature is compiled in. #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -323,7 +323,7 @@ impl Client { )))] pub(crate) async fn dial(&self, _addr: crate::connect::Addr) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -334,7 +334,7 @@ impl Client { /// protocol driver is spawned on the current tokio runtime; the session /// closes once the last returned handle drops. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -362,7 +362,7 @@ impl Client { /// The moq client builder, advertising `path` in the SETUP when there is one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -376,7 +376,7 @@ impl Client { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -472,15 +472,15 @@ impl Client { return Ok(Dialed::new(session, crate::Transport::Iroh)); } - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { let quic_addr = addr.clone(); let binding = match quic_transport(&url) { crate::Transport::Quic => moq_net::transport::Binding::InBand, _ => moq_net::transport::Binding::Uri, }; let quic_handle = async { - noq.connect(&self.tls, quic_addr, &self.versions) + quic.connect(&self.tls, quic_addr, &self.versions) .await .map(|session| crate::transport::Session::new(session, binding)) .map_err(Error::from) @@ -530,7 +530,7 @@ impl Client { /// /// Only compiled when there is a QUIC dial to race: a WebSocket-only build connects /// over the fallback directly. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] async fn race_moq_connect( &self, moq: &moq_net::Client, @@ -574,7 +574,7 @@ impl Client { /// `None` when the result is empty, which means the same as omitting the parameter: the /// server's default path. A peer on published lite-05 rejects an empty value outright. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -600,7 +600,7 @@ fn request_target(url: &Url) -> Option { /// picked by ALPN negotiation rather than by the scheme; that dial reads the negotiated /// [`crate::iroh::Binding`] and calls [`request_target`] itself. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -623,9 +623,9 @@ fn setup_path(url: &Url) -> Option { } } -/// What a noq dial to `url` runs on: WebTransport for `https://` (and the `http://` +/// What a QUIC dial to `url` runs on: WebTransport for `https://` (and the `http://` /// bootstrap), raw QUIC for `moqt://` and `moql://`. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn quic_transport(url: &Url) -> crate::Transport { match url.scheme() { "moqt" | "moql" => crate::Transport::Quic, @@ -643,7 +643,7 @@ pub(crate) struct Dialed { impl Dialed { #[cfg_attr( not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -662,7 +662,7 @@ impl Dialed { /// from the host and port so URL userinfo never goes on the wire. `None` for every other /// scheme, and when the URL has no host, since an empty authority is meaningless. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -681,7 +681,7 @@ fn setup_authority(url: &Url) -> Option { } /// The arm that won a [`race_transport_connect`]; the other was dropped, cancelling it. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] enum TransportRace { Quic(Q), WebSocket(W), @@ -689,7 +689,7 @@ enum TransportRace { /// Race the QUIC dial to `url` against the WebSocket fallback, returning whichever /// transport comes up first. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] async fn race_transport_connect(url: &Url, quic: Q, websocket: W) -> crate::Result> where Q: Future>, @@ -751,7 +751,7 @@ where /// Auth is terminal only when both arms refused. A WebTransport-only endpoint /// answers the fallback with 403 while QUIC is still in flight, and reconnect /// treats is_auth() as terminal, so a mixed pair reports the retryable error. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] fn race_error(quic: Error, websocket: Error) -> Error { match (quic.is_auth(), websocket.is_auth()) { (false, false) => Error::TransportRace { @@ -764,7 +764,7 @@ fn race_error(quic: Error, websocket: Error) -> Error { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -791,25 +791,25 @@ mod tests { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn fixed_target_preserves_request_and_refuses_redirect() { check_fixed_redirect(false, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn one_shot_fixed_target_refuses_redirect() { check_fixed_redirect(true, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn unused_fixed_fallback_does_not_restrict_connected_target() { check_fixed_redirect(true, false).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] async fn check_fixed_redirect(once: bool, pinned: bool) { let mut listen = crate::listen::Config { bind: Some("127.0.0.1:0".parse().unwrap()), @@ -886,7 +886,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -937,7 +937,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -976,7 +976,7 @@ mod tests { /// The iroh dial derives its target here rather than through [`setup_path`], since /// only the negotiated binding says whether to send one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1351,7 +1351,7 @@ mod tests { assert_eq!(config.versions().alpns().len(), moq_net::ALPNS.len()); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_auth_error() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1367,7 +1367,7 @@ mod tests { assert!(matches!(value, super::TransportRace::WebSocket(1))); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_quic_after_websocket_forbidden() { let quic = async { @@ -1389,7 +1389,7 @@ mod tests { /// Inline rather than in `tests/` so each arm dials its own ephemeral port: the /// public connect sends the fallback to the QUIC port, which nothing reserves over /// TCP as well. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn websocket_forbidden_does_not_end_a_quic_connect() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -1431,14 +1431,14 @@ mod tests { // The same race `connect_inner` runs, except the fallback dials its own port, // as plain ws:// so the listener above can answer without TLS. - let noq = client.noq.clone().unwrap(); + let quic = client.quic.clone().unwrap(); let (tls, versions) = (client.tls.clone(), client.versions.clone()); let quic_addr: crate::connect::Addr = Url::parse(&format!("https://localhost:{quic_port}")).unwrap().into(); let ws_addr: crate::connect::Addr = Url::parse(&format!("http://localhost:{ws_port}")).unwrap().into(); // Hold QUIC until the fallback has been refused, so the 403 is always exercised. let quic = Box::pin(async move { forbid.await.unwrap().expect("fallback listener failed"); - noq.connect(&tls, quic_addr, &versions) + quic.connect(&tls, quic_addr, &versions) .await .map(|session| crate::transport::Session::new(session, moq_net::transport::Binding::Uri)) .map_err(Error::from) @@ -1460,7 +1460,7 @@ mod tests { /// QUIC wins while the fallback handshake is held until QUIC connects. /// Each transport owns its ephemeral port; this exercises the same race as /// connect_inner without depending on which real handshake runs faster. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tracing_test::traced_test] #[tokio::test] async fn broadcast_race_quic_wins() { @@ -1530,8 +1530,8 @@ mod tests { let client = client.with_subscriber(sub_origin); let dialer = client.clone(); let quic = Box::pin(async move { - let noq = dialer.noq.as_ref().expect("QUIC backend"); - let session = noq.connect(&dialer.tls, quic_addr, &dialer.versions).await?; + let quic = dialer.quic.as_ref().expect("QUIC backend"); + let session = quic.connect(&dialer.tls, quic_addr, &dialer.versions).await?; connected.send(()).expect("fallback listener alive"); Ok::<_, crate::Error>(crate::transport::Session::new( session, @@ -1577,7 +1577,7 @@ mod tests { .expect("server task failed"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_auth_when_both_refuse() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1589,7 +1589,7 @@ mod tests { assert!(err.is_auth(), "unexpected error: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_quic_error_when_websocket_forbidden() { let quic = async { @@ -1605,7 +1605,7 @@ mod tests { assert!(!err.is_auth(), "mixed auth/non-auth must stay retryable: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_non_auth_error() { let quic = async { Err::(Error::ConnectFailed) }; @@ -1619,14 +1619,14 @@ mod tests { /// The URL a race unit test dials. Each test runs in its own process under nextest, /// so the process-wide [`crate::websocket::won`] memo starts empty. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn race_url() -> Url { Url::parse("https://race.invalid/").unwrap() } /// QUIC winning a race clears WebSocket's earlier win, so the next dial gives QUIC /// its head start again. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_restores_its_head_start() { crate::websocket::remember(&race_url()); @@ -1644,7 +1644,7 @@ mod tests { } /// A race arm that never finishes, and reports through `dropped` once it is dropped. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn loser() -> ( impl Future, tokio::sync::oneshot::Receiver, @@ -1659,7 +1659,7 @@ mod tests { /// WebSocket winning cancels the QUIC dial before the race returns, so no QUIC /// session can come up behind the one WebSocket serves. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn websocket_winning_cancels_the_quic_dial() { let (quic, mut dropped) = loser::>(); @@ -1677,7 +1677,7 @@ mod tests { } /// QUIC winning cancels the WebSocket dial before the race returns. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_cancels_the_websocket_dial() { let quic = async { Ok("quic") }; @@ -1694,7 +1694,7 @@ mod tests { ); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_returns_when_quic_transport_connects() { let quic = async { Ok("quic") }; @@ -1721,7 +1721,7 @@ mod tests { /// Iroh carries no rustls state, so constructing its client must not require an /// application-installed crypto provider. - #[cfg(all(feature = "iroh", not(any(feature = "noq", feature = "websocket"))))] + #[cfg(all(feature = "iroh", not(any(feature = "quic", feature = "websocket"))))] #[test] fn iroh_only_client_does_not_require_a_tls_provider() { crate::connect::Config::default().init().expect("iroh-only client"); diff --git a/rs/moq-tokio/src/connect.rs b/rs/moq-tokio/src/connect.rs index 789efcb0df..cd67fa6678 100644 --- a/rs/moq-tokio/src/connect.rs +++ b/rs/moq-tokio/src/connect.rs @@ -341,7 +341,7 @@ pub enum ConnectError { impl ConnectError { /// Only the transports that carry an HTTP status (WebTransport, WebSocket) can /// classify one; qmux over tcp/unix has no such response. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] pub(crate) fn from_status_u16(status: u16) -> Option { match status { 401 => Some(Self::Unauthorized), @@ -357,7 +357,7 @@ impl ConnectError { } } -#[cfg(all(test, any(feature = "noq", feature = "websocket")))] +#[cfg(all(test, any(feature = "quic", feature = "websocket")))] mod tests { use super::*; diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f98e88f8e1..e44fbaa760 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2070,7 +2070,7 @@ mod tests { } /// Invalid local configuration must end an unlimited reconnect loop without waiting. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_unsupported_scheme_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2088,7 +2088,7 @@ mod tests { } /// A URL host that parses but can never be a TLS server name ends an unlimited reconnect loop. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_invalid_tls_server_name_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2200,12 +2200,12 @@ mod tests { /// /// Holding is what decides the transport race without depending on timing: the /// QUIC dial cannot finish until the test says so. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] struct Forwarder { gate: tokio::sync::watch::Sender, } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] impl Forwarder { async fn start(front: tokio::net::UdpSocket, server: std::net::SocketAddr) -> Self { use std::sync::Arc; @@ -2264,7 +2264,7 @@ mod tests { /// When QUIC wins the race the WebSocket dial is cancelled at once: its socket /// closes without carrying a session, and the one session runs over QUIC. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn quic_winning_cancels_the_websocket_dial() { use tokio::io::AsyncReadExt; diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index e1dbb14064..21aff0f4db 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -206,10 +206,10 @@ pub enum Error { #[error(transparent)] Tls(Arc), - /// The noq backend failed. - #[cfg(feature = "noq")] + /// The QUIC backend failed. + #[cfg(feature = "quic")] #[error(transparent)] - Noq(Arc), + Quic(Arc), /// The Iroh backend failed. #[cfg(feature = "iroh")] @@ -252,8 +252,8 @@ impl Error { | Self::WindowRange(_) | Self::QlogUnsupported | Self::Connect(_) => true, - #[cfg(feature = "noq")] - Self::Noq(err) => err.is_terminal(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.is_terminal(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.is_terminal(), #[cfg(feature = "tcp")] @@ -285,8 +285,8 @@ impl Error { Self::MoqNet( moq_net::Error::Unauthorized | moq_net::Error::Session(moq_net::SessionError::Unauthorized), ) => Some(crate::ConnectError::Unauthorized), - #[cfg(feature = "noq")] - Self::Noq(err) => err.connect_error(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.connect_error(), #[cfg(feature = "websocket")] Self::TransportRace { quic, websocket } => quic.connect_error().or_else(|| websocket.connect_error()), #[cfg(feature = "websocket")] @@ -318,8 +318,8 @@ impl Error { _ => None, }, - #[cfg(feature = "noq")] - Self::Noq(err) => err.status(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.status(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.status(), _ => None, @@ -348,14 +348,14 @@ impl From for Error { } } -#[cfg(feature = "noq")] -impl From for Error { - fn from(err: crate::noq::Error) -> Self { +#[cfg(feature = "quic")] +impl From for Error { + fn from(err: crate::quic::Error) -> Self { if let Some(err) = err.connect_error() { return Self::Connect(err); } - Self::Noq(Arc::new(err)) + Self::Quic(Arc::new(err)) } } diff --git a/rs/moq-tokio/src/iroh.rs b/rs/moq-tokio/src/iroh.rs index 9e7350d1ad..058f33ca0c 100644 --- a/rs/moq-tokio/src/iroh.rs +++ b/rs/moq-tokio/src/iroh.rs @@ -9,7 +9,7 @@ use std::{net, path::PathBuf, str::FromStr, sync::Arc}; use crate::quic::CongestionControl; use url::Url; use web_transport_iroh::iroh::{self, SecretKey}; -// NOTE: web-transport-iroh should re-export proto like web-transport-moq does. +// NOTE: web-transport-iroh should re-export proto like `crate::quic::web` does. use web_transport_proto::{ConnectRequest, ConnectResponse}; pub use iroh::Endpoint; @@ -112,7 +112,7 @@ pub enum Error { RecvRequest(String), /// GSO is always on for iroh, so `--quic-gso=false` cannot be honored. - #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the noq backend")] + #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the quic backend")] GsoUnsupported, } @@ -531,7 +531,7 @@ mod tests { /// Build a controller from each family's factory and downcast it to the /// concrete implementation it must map to. iroh runs on noq, so this is the - /// same pairing as the noq backend. + /// same pairing as the quic backend. #[test] fn congestion_factory_maps_each_family() { let now = std::time::Instant::now(); diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index 29a9b356e9..1dc43f9694 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -11,7 +11,7 @@ //! See [`Client`] for connecting to relays and [`Server`] for accepting //! connections. The `mdns` feature finds peers to connect to on the local network. //! -//! With `default-features = false`, the `noq` backend must be paired +//! With `default-features = false`, the `quic` backend must be paired //! with the `aws-lc-rs` or `ring` crypto-provider feature. Every other subset //! compiles, including no transport at all: such a build cannot connect to //! anything, though a crypto provider on its own is still enough to configure TLS @@ -22,7 +22,7 @@ // The protocol crates need a compiled provider for reset and retry-token keys. A rustls provider // installed at runtime cannot supply constructors removed by their compile-time feature gates. -#[cfg(all(feature = "noq", not(any(feature = "aws-lc-rs", feature = "ring"))))] +#[cfg(all(feature = "quic", not(any(feature = "aws-lc-rs", feature = "ring"))))] compile_error!("a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature"); mod abort; @@ -34,18 +34,16 @@ pub mod connect; pub mod connection; pub mod crypto; mod error; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] pub mod failover; #[cfg(feature = "jemalloc")] pub mod jemalloc; pub mod listen; mod log; -#[cfg(feature = "noq")] -pub mod noq; pub mod origin; pub mod panic; pub mod quic; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] mod resolve; #[cfg(feature = "_transport")] pub mod server; @@ -60,7 +58,7 @@ pub mod unix; pub mod worker; // Resolving a `host:port` bind string is a QUIC-listener concern; the stream // listeners take a `SocketAddr`/path straight from their config. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod util; #[cfg(feature = "watch")] pub mod watch; diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 20e6fa4f7a..16959fd75c 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -14,7 +14,7 @@ pub enum Bind { } impl Bind { - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn resolve(&self) -> std::io::Result { match self { Self::Addr(addr) => Ok(*addr), @@ -186,11 +186,11 @@ pub struct Config { /// IPv4 address advertised as the QUIC preferred_address. /// - /// Supporting clients (Chrome M131+, native noq) migrate to this address + /// Supporting clients (Chrome M131+, this crate) migrate to this address /// shortly after the handshake completes. Typical use: handshake on an /// anycast IP, steady-state on this host's unicast IP. /// - /// Honored by noq. Accept-only, which is why it lives + /// Honored by the QUIC backend. Accept-only, which is why it lives /// here rather than in the shared [`crate::quic::Config`]. #[usage( name = "listen-preferred-v4", @@ -292,7 +292,7 @@ impl Default for Config { } } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) use moq_sock::shard::Shard; /// One server's socket in a complete `SO_REUSEPORT` group. /// @@ -527,7 +527,7 @@ impl Config { (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { self.lb_id @@ -666,7 +666,7 @@ mod tests { assert!("relay.example.com:443:8443".parse::().is_err()); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn cli_load_balancer_survives_the_merge_round_trip() { let config = config_from(["test", "--listen-quic-lb-id", "ab", "--listen-quic-lb-nonce", "9"]); @@ -726,7 +726,7 @@ load_balancer = { id = "cd", nonce = 4 } } /// Programmatic configuration keeps the QUIC-LB id and nonce paired. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn load_balancer_is_a_single_typed_value() { assert_eq!(Config::default().load_balancer(), None); diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index ddf33bbcbe..5d8a5e4f19 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,7 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) - .add_directive("noq=info".parse()?) + .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) .add_directive("tracing::span=off".parse()?) diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 4935765246..4b28ac64ef 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -14,6 +14,29 @@ use std::time::Duration; use crate::cli::Duration as CliDuration; +#[cfg(feature = "quic")] +mod backend; +#[cfg(feature = "quic")] +pub use backend::Error; +#[cfg(feature = "quic")] +pub(crate) use backend::{Client, Options, Server, accept}; + +// The async QUIC layer driving `moq-quic` on tokio, forked from quinn (see its module docs). +// Public only so the workspace's tests can dial a bare endpoint; not a stable API. +// cargo fmt formats submodules with the crate root's config, so `just rs fix` +// formats this one separately with its own (quinn's). +#[cfg(feature = "quic")] +#[doc(hidden)] +#[rustfmt::skip] +pub mod driver; + +/// WebTransport over [`driver`], formerly the `web-transport-moq` crate. +/// +/// Public only so the workspace's tests can dial a bare session; not a stable API. +#[cfg(feature = "quic")] +#[doc(hidden)] +pub mod web; + /// The routable server ID a QUIC-LB load balancer encodes into connection IDs. /// /// Parsed from, and serialized as, a hex string. Its length must match the load @@ -55,7 +78,7 @@ pub struct LoadBalancer { /// The congestion control family for a QUIC connection. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. #[derive(Clone, Copy, Debug, PartialEq, Eq, usage::ValueEnum, serde::Serialize, serde::Deserialize)] #[usage(ignore_case)] #[serde(rename_all = "kebab-case")] @@ -87,7 +110,7 @@ pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Bounds how much unread data one peer can make us buffer across all of its streams, /// which the backends otherwise leave unlimited. Sized so a relay-to-relay session, which /// shares one connection, can still sustain several Gbps at intercontinental RTTs. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) const DEFAULT_RECEIVE_WINDOW: u64 = 64 << 20; /// Default idle timeout before an inactive connection is dropped. @@ -250,7 +273,7 @@ pub struct Config { /// Write qlog traces into this directory, which must already exist. /// - /// Noq writes one file per connection. + /// Each endpoint writes one file, shared by its connections. /// /// Requires the `qlog` feature; setting it errors at init otherwise. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -648,7 +671,7 @@ impl Resolved { /// Delay-based unless the operator says otherwise: BBR keeps queues short and the /// send rate steady enough for a live encoder to track, which is what this stack /// carries. - #[cfg_attr(not(any(feature = "noq", feature = "iroh")), allow(dead_code))] + #[cfg_attr(not(any(feature = "quic", feature = "iroh")), allow(dead_code))] pub(crate) fn congestion(&self) -> CongestionControl { self.congestion_control.unwrap_or(CongestionControl::Delay) } @@ -657,7 +680,7 @@ impl Resolved { /// /// Only meaningful once [`Config::validate`] has passed; a build without the /// `qlog` feature never gets here with a directory set. - #[cfg_attr(not(feature = "noq"), allow(dead_code))] + #[cfg_attr(not(feature = "quic"), allow(dead_code))] pub(crate) fn qlog_dir(&self) -> Option<&std::path::Path> { self.qlog.as_deref() } diff --git a/rs/moq-tokio/src/noq.rs b/rs/moq-tokio/src/quic/backend.rs similarity index 79% rename from rs/moq-tokio/src/noq.rs rename to rs/moq-tokio/src/quic/backend.rs index 27634e1648..b0b5b08414 100644 --- a/rs/moq-tokio/src/noq.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -1,5 +1,6 @@ -//! The noq QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +//! The QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +use super::{driver, web}; use crate::RedactedUrl; use crate::connect; use crate::listen; @@ -12,13 +13,12 @@ use std::pin::Pin; use std::sync::{Arc, Mutex, RwLock}; use std::task::{Context, Poll, Waker, ready}; use std::time::Duration; -use web_transport_moq::noq; -/// Attach a qlog factory writing into the configured directory, if any. +/// Attach a qlog stream writing into the configured directory, if any. /// -/// noq's factory opens one file per connection, named after its initial destination -/// connection ID, so nothing needs to be unique per endpoint here. -fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { +/// A [`driver::QlogStream`] is a shared handle: every connection on the endpoint writes into +/// it, tagged with its own qlog `group_id`, so this is one file per endpoint. +fn apply_qlog(transport: &mut driver::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { // `Client::validate` already rejected a directory this build can't honor, so the // block below is only reached where capture actually works. let Some(dir) = quic.qlog_dir() else { @@ -27,8 +27,21 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) #[cfg(feature = "qlog")] { - transport.qlog_from_path(dir, &format!("moq-{role}")); - tracing::info!(dir = %dir.display(), "writing qlog"); + // The pid and sequence keep endpoints sharing a directory from clobbering each other. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = dir.join(format!("moq-{role}-{}-{seq}.sqlog", std::process::id())); + let file = std::fs::File::create(&path).map_err(Error::CreateQlog)?; + + // Deliberately unbuffered: qlog's streamer only flushes on `finish_log`, which is + // never called per event, so a BufWriter would hold every trace in memory until the + // endpoint drops and lose the lot if the process is killed. Killing a stuck process + // is exactly when these traces are worth having. + let mut config = driver::QlogConfig::default(); + config.writer(Box::new(file)).title(Some(format!("moq-tokio {role}"))); + + transport.qlog_stream(config.into_stream()); + tracing::info!(path = %path.display(), "writing qlog"); } #[cfg(not(feature = "qlog"))] @@ -37,17 +50,17 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) Ok(()) } -/// Apply the resolved quic knobs to a noq transport config. -fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { +/// Apply the resolved quic knobs to a transport config. +fn apply_transport(transport: &mut driver::TransportConfig, quic: &Resolved) { transport.max_idle_timeout(Some(quic.idle_timeout.try_into().expect("idle timeout out of range"))); transport.keep_alive_interval(quic.keep_alive); - // noq enables MTU discovery by default; disable it unless asked. + // MTU discovery is enabled by default; disable it unless asked. if !quic.mtu_discovery { transport.mtu_discovery_config(None); } - let max_streams = noq::VarInt::from_u64(quic.max_streams).unwrap_or(noq::VarInt::MAX); + let max_streams = driver::VarInt::from_u64(quic.max_streams).unwrap_or(driver::VarInt::MAX); transport.max_concurrent_bidi_streams(max_streams); transport.max_concurrent_uni_streams(max_streams); @@ -62,28 +75,28 @@ fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { /// Apply the flow-control windows. The connection window defaults to ours, since the /// backend's is unlimited; the others stay at the backend default when unset. -fn apply_windows(transport: &mut noq::TransportConfig, quic: &Resolved) { +fn apply_windows(transport: &mut driver::TransportConfig, quic: &Resolved) { // Saturating rather than erroring: `Config::validate` already rejects a window // past the varint, so this only bites a `Resolved` built without it. let window = quic.receive_window.unwrap_or(crate::quic::DEFAULT_RECEIVE_WINDOW); - transport.receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); if let Some(window) = quic.stream_receive_window { - transport.stream_receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.stream_receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); } if let Some(window) = quic.send_window { transport.send_window(window); } } -/// The noq controller factory for a congestion control family. noq's BBR is v3. -fn congestion_factory(family: CongestionControl) -> Arc { +/// The controller factory for a congestion control family. moq-quic's BBR is v3. +fn congestion_factory(family: CongestionControl) -> Arc { match family { - CongestionControl::Loss => Arc::new(noq::congestion::CubicConfig::default()), - CongestionControl::Delay => Arc::new(noq::congestion::Bbr3Config::default()), + CongestionControl::Loss => Arc::new(driver::congestion::CubicConfig::default()), + CongestionControl::Delay => Arc::new(driver::congestion::Bbr3Config::default()), } } -/// Errors specific to the noq QUIC backend. +/// Errors specific to the QUIC backend. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -95,6 +108,11 @@ pub enum Error { #[error("failed to create QUIC endpoint")] CreateEndpoint(#[source] std::io::Error), + /// The qlog trace file couldn't be created, usually a missing directory. + #[cfg(feature = "qlog")] + #[error("failed to create qlog file")] + CreateQlog(#[source] std::io::Error), + /// No async runtime was found. Construct the client or server from within a tokio runtime. #[error("no async runtime")] NoRuntime, @@ -203,7 +221,7 @@ pub enum Error { message: String, /// The HTTP status the server answered the CONNECT with, when it answered with one. /// - /// Read at conversion time rather than kept as a `web-transport-moq` error, so the + /// Read at conversion time rather than kept as a [`web`] error, so the /// classification survives without that crate appearing in this crate's public API. status: Option, }, @@ -251,20 +269,20 @@ impl crate::failover::Aggregate for Error { } crate::error::from_message! { - noq::ConnectError => Connect, - noq::ConnectionError => Connection, - web_transport_moq::ServerError => Server, + driver::ConnectError => Connect, + driver::ConnectionError => Connection, + web::ServerError => Server, hex::FromHexError => InvalidFingerprint, } -impl From for Error { - fn from(_: noq::crypto::rustls::NoInitialCipherSuite) -> Self { +impl From for Error { + fn from(_: driver::crypto::rustls::NoInitialCipherSuite) -> Self { Self::NoInitialCipherSuite } } -impl From for Error { - fn from(err: web_transport_moq::ClientError) -> Self { +impl From for Error { + fn from(err: web::ClientError) -> Self { Self::Client { status: client_status(&err), message: crate::error::message(err), @@ -277,9 +295,9 @@ type Result = std::result::Result; // ── Client ────────────────────────────────────────────────────────── #[derive(Clone)] -pub(crate) struct NoqClient { - pub quic: noq::Endpoint, - pub transport: Arc, +pub(crate) struct Client { + pub quic: driver::Endpoint, + pub transport: Arc, /// Whether an `http://` URL may bootstrap a pin (see [crate::tls::Connect::allows_http_bootstrap]). pub http_bootstrap: bool, /// Optional TLS SNI / verification hostname override (from config). @@ -295,24 +313,25 @@ pub(crate) struct NoqClient { dual_stack: bool, } -impl NoqClient { +impl Client { pub fn new(config: &connect::Config, quic: &crate::quic::Config) -> Result { let resolved = config.resolve(); let socket = crate::bind::udp(crate::bind::Udp::new(resolved.bind)).map_err(Error::BindSocket)?; let dual_stack = crate::bind::udp_is_dual_stack(&socket); - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "client")?; let transport = Arc::new(transport); // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; - let endpoint_config = noq::EndpointConfig::default(); + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; + let endpoint_config = driver::EndpointConfig::default(); // Create the generic QUIC endpoint. - let quic = noq::Endpoint::new(endpoint_config, None, socket, runtime).map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new(endpoint_config, None, socket).map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -327,7 +346,7 @@ impl NoqClient { /// Close every connection, then wait until each has sent its close to the peer. pub async fn close(self) { - self.quic.close(noq::VarInt::from_u32(0), b"client shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"client shutdown"); // Not `wait_idle`, which also sits out each connection's 3 PTO closing // period: that only repeats the close to a peer that already has it. self.quic.wait_all_draining().await; @@ -338,7 +357,7 @@ impl NoqClient { tls: &rustls::ClientConfig, addr: crate::connect::Addr, versions: &moq_net::Versions, - ) -> Result { + ) -> Result { let mut url = addr.url().clone(); let mut config = tls.clone(); @@ -401,7 +420,7 @@ impl NoqClient { } let alpns: Vec> = match url.scheme() { - "https" => vec![web_transport_moq::ALPN.as_bytes().to_vec()], + "https" => vec![web::ALPN.as_bytes().to_vec()], "moqt" | "moql" => versions.alpns().iter().map(|alpn| alpn.as_bytes().to_vec()).collect(), _ => return Err(Error::InvalidScheme), }; @@ -409,8 +428,8 @@ impl NoqClient { config.alpn_protocols = alpns; config.key_log = Arc::new(rustls::KeyLogFile::new()); - let config: noq::crypto::rustls::QuicClientConfig = config.try_into()?; - let mut config = noq::ClientConfig::new(Arc::new(config)); + let config: driver::crypto::rustls::QuicClientConfig = config.try_into()?; + let mut config = driver::ClientConfig::new(Arc::new(config)); config.transport_config(self.transport.clone()); tracing::debug!(peer = %crate::connect::Endpoint(&url), "connecting"); @@ -429,15 +448,15 @@ impl NoqClient { let session = match url.scheme() { "https" => { - let mut request = web_transport_moq::proto::ConnectRequest::new(url.clone()); + let mut request = web::proto::ConnectRequest::new(url.clone()); for alpn in versions.alpns() { request = request.with_protocol(alpn.to_string()); } - web_transport_moq::Session::connect(connection, request) + web::Session::connect(connection, request) .await .map_err(map_client_error)? } - "moqt" | "moql" => web_transport_moq::Session::raw(connection), + "moqt" | "moql" => web::Session::raw(connection), _ => return Err(Error::UnsupportedScheme(url.scheme().to_string())), }; @@ -502,7 +521,7 @@ impl Error { } } -fn map_client_error(err: web_transport_moq::ClientError) -> Error { +fn map_client_error(err: web::ClientError) -> Error { match client_status(&err).and_then(crate::ConnectError::from_status_u16) { Some(rejected) => rejected.into(), None => err.into(), @@ -516,48 +535,49 @@ fn map_client_error(err: web_transport_moq::ClientError) -> Error { /// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults /// the status. A `404` or `405` is the server's settled answer, so retrying /// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_moq::ClientError) -> Option { +fn client_status(err: &web::ClientError) -> Option { match err { - web_transport_moq::ClientError::HttpError(err) => connect_status(err), + web::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn connect_status(err: &web_transport_moq::ConnectError) -> Option { +fn connect_status(err: &web::ConnectError) -> Option { match err { - web_transport_moq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), - web_transport_moq::ConnectError::ProtoError(err) => proto_status(err), + web::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn proto_status(err: &web_transport_moq::proto::ConnectError) -> Option { +fn proto_status(err: &web::proto::ConnectError) -> Option { match err { - web_transport_moq::proto::ConnectError::ErrorStatus(status) - | web_transport_moq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + web::proto::ConnectError::ErrorStatus(status) | web::proto::ConnectError::WrongStatus(Some(status)) => { + Some(status.as_u16()) + } _ => None, } } // ── Server ────────────────────────────────────────────────────────── -pub(crate) struct NoqServer { - pub quic: noq::Endpoint, +pub(crate) struct Server { + pub quic: driver::Endpoint, tls: crate::tls::Serve, socket: Socket, } -/// The server's UDP socket, which [`NoqServer::shutdown`] closes even while noq +/// The server's UDP socket, which [`Server::shutdown`] closes even while the driver /// still holds references to it. /// -/// noq gives each connection a sender sharing the socket, and a connection lives +/// The driver gives each connection a sender sharing the socket, and a connection lives /// as long as anything holds its handle, so dropping the endpoint leaves the port /// bound until every accepted session is gone too. Taking the socket out of this /// slot closes it regardless. #[derive(Clone, Debug)] struct Socket { io: Arc>>>, - state: Arc, + state: Arc, local: net::SocketAddr, /// Wakes every sender blocked on a full send buffer from the socket's single /// write-readiness registration. @@ -566,7 +586,7 @@ struct Socket { impl Socket { fn new(socket: std::net::UdpSocket) -> std::io::Result { - let state = noq::udp::UdpSocketState::new((&socket).into())?; + let state = moq_sock::udp::UdpSocketState::new((&socket).into())?; let local = socket.local_addr()?; let io = tokio::net::UdpSocket::from_std(socket)?; Ok(Self { @@ -588,8 +608,8 @@ impl Socket { } } -impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { +impl driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { Box::pin(self.clone()) } @@ -597,7 +617,7 @@ impl noq::AsyncUdpSocket for Socket { &mut self, cx: &mut Context<'_>, bufs: &mut [std::io::IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { let io = self.io.read().unwrap(); // Released: nothing arrives again, and the endpoint stops through its close. @@ -617,7 +637,7 @@ impl noq::AsyncUdpSocket for Socket { Ok(self.local) } - fn max_receive_segments(&self) -> std::num::NonZeroUsize { + fn max_receive_segments(&self) -> usize { self.state.gro_segments() } @@ -626,10 +646,10 @@ impl noq::AsyncUdpSocket for Socket { } } -impl noq::UdpSender for Socket { +impl driver::UdpSender for Socket { fn poll_send( self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, + transmit: &moq_sock::udp::Transmit<'_>, cx: &mut Context<'_>, ) -> Poll> { let io = self.io.read().unwrap(); @@ -638,6 +658,9 @@ impl noq::UdpSender for Socket { return Poll::Ready(Ok(())); }; loop { + // `send` logs and drops a failed datagram rather than failing the connection, as + // UDP loss would; only `WouldBlock` comes back. + #[allow(deprecated)] match io.try_io(tokio::io::Interest::WRITABLE, || self.state.send(io.into(), transmit)) { Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} res => return Poll::Ready(res), @@ -650,7 +673,7 @@ impl noq::UdpSender for Socket { } } - fn max_transmit_segments(&self) -> std::num::NonZeroUsize { + fn max_transmit_segments(&self) -> usize { self.state.max_gso_segments() } } @@ -687,7 +710,7 @@ pub(crate) struct Options { pub tls: crate::tls::Serve, } -impl NoqServer { +impl Server { pub fn new(options: Options) -> Result { let Options { config, @@ -695,7 +718,7 @@ impl NoqServer { member, tls: served, } = options; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "server")?; @@ -708,17 +731,17 @@ impl NoqServer { .iter() .map(|alpn| alpn.as_bytes().to_vec()) .collect(); - alpns.push(web_transport_moq::ALPN.as_bytes().to_vec()); + alpns.push(web::ALPN.as_bytes().to_vec()); let mut tls = (*served.config(alpns)?).clone(); tls.key_log = Arc::new(rustls::KeyLogFile::new()); - let tls: noq::crypto::rustls::QuicServerConfig = tls.try_into()?; - let mut tls = noq::ServerConfig::with_crypto(Arc::new(tls)); + let tls: driver::crypto::rustls::QuicServerConfig = tls.try_into()?; + let mut tls = driver::ServerConfig::with_crypto(Arc::new(tls)); tls.transport_config(transport); // Advertise the preferred_address transport parameter (RFC 9000 §9.6). - // noq allocates a fresh CID + reset token for the address during the handshake. + // The driver allocates a fresh CID + reset token for the address during the handshake. if let Some(addr) = config.preferred_v4 { tls.preferred_address_v4(Some(addr)); } @@ -727,7 +750,8 @@ impl NoqServer { } // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; let listen = config .bind @@ -739,7 +763,7 @@ impl NoqServer { let load_balancer = config.load_balancer(); // Configure connection ID generator with server ID if provided - let mut endpoint_config = noq::EndpointConfig::default(); + let mut endpoint_config = driver::EndpointConfig::default(); if let Some(shard) = member.as_ref().map(listen::Socket::shard) { if load_balancer.is_some() { return Err(Error::ShardWithQuicLb); @@ -781,9 +805,8 @@ impl NoqServer { // Create the generic QUIC endpoint. let socket = Socket::new(socket).map_err(Error::CreateEndpoint)?; - let quic = - noq::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone()), runtime) - .map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone())) + .map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -792,7 +815,7 @@ impl NoqServer { }) } - pub fn accept(&self) -> impl std::future::Future> + '_ { + pub fn accept(&self) -> impl std::future::Future> + '_ { self.quic.accept() } @@ -805,7 +828,7 @@ impl NoqServer { } pub fn close(&self) { - self.quic.close(noq::VarInt::from_u32(0), b"server shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"server shutdown"); } /// Close every connection, wait until each has sent its close to the peer, @@ -824,32 +847,34 @@ impl NoqServer { } } -// ── NoqRequest ────────────────────────────────────────────────────── +// ── QuicRequest ───────────────────────────────────────────────────── -/// A raw QUIC connection request without WebTransport framing (noq backend). +/// A raw QUIC connection request without WebTransport framing (QUIC backend). /// Accept a QUIC connection, negotiate WebTransport or raw moq, and complete the /// handshake (a `200 OK` for WebTransport). Returns the established session, the request /// URL and validated mTLS identity (both captured before the response consumes the /// request), and the dialed authority (the CONNECT authority on WebTransport, the TLS /// SNI on raw QUIC). Raw QUIC carries no request URL (the path rides the SETUP instead). pub(crate) async fn accept( - conn: noq::Incoming, + conn: driver::Incoming, alpns: Vec<&'static str>, -) -> Result> { +) -> Result> { let mut conn = conn.accept()?; - let handshake = conn - .handshake_data() - .await? - .downcast::() - .unwrap(); + // Scoped so the `!Send` boxed handshake data is gone before the next await. + let (alpn, host) = { + let handshake = conn + .handshake_data() + .await? + .downcast::() + .unwrap(); + (handshake.protocol, handshake.server_name) + }; - let alpn = handshake.protocol.ok_or(Error::MissingAlpn)?; + let alpn = alpn.ok_or(Error::MissingAlpn)?; let alpn = String::from_utf8(alpn)?; - let host = handshake.server_name.unwrap_or_default(); + let host = host.unwrap_or_default(); - // The established Connection no longer exposes a single peer address (noq 1.0 - // supports multipath), so capture it from the Connecting before awaiting. let remote = conn.remote_address(); tracing::debug!(%host, ip = %remote, %alpn, "accepting"); @@ -868,10 +893,10 @@ pub(crate) async fn accept( }; match alpn.as_str() { - web_transport_moq::ALPN => { + web::ALPN => { // Wait for the CONNECT request, then capture its URL and mTLS identity before // the response consumes it. - let request = web_transport_moq::Request::accept(conn) + let request = web::Request::accept(conn) .await .map_err(|err| Error::RecvRequest(crate::error::message(err)))?; let url = Some(request.url.clone()); @@ -879,7 +904,7 @@ pub(crate) async fn accept( // The authority the client put in its CONNECT URL. let authority = request.url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); - let mut response = web_transport_moq::proto::ConnectResponse::OK; + let mut response = web::proto::ConnectResponse::OK; let mut link = link; if let Some(protocol) = request.protocols.iter().find(|p| alpns.contains(&p.as_str())) { response = response.with_protocol(protocol); @@ -906,7 +931,7 @@ pub(crate) async fn accept( // Raw QUIC carries no request URL; the path rides the SETUP. The TLS SNI is the // only authority the client can offer here, and it is optional. let authority = (!host.is_empty()).then_some(host); - let session = web_transport_moq::Session::raw(conn); + let session = web::Session::raw(conn); Ok(crate::server::Accepted { session, url: None, @@ -939,13 +964,13 @@ impl ShardIdGenerator { } } -impl noq::ConnectionIdGenerator for ShardIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ShardIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let mut cid = Vec::with_capacity(Self::LEN); cid.push(moq_sock::shard::cid_prefix(self.shard)); cid.extend(rand::rng().random_iter::().take(Self::LEN - 1)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -970,8 +995,8 @@ impl ServerIdGenerator { } } -impl noq::ConnectionIdGenerator for ServerIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ServerIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let cid_len = self.cid_len(); let mut cid = Vec::with_capacity(cid_len); @@ -979,7 +1004,7 @@ impl noq::ConnectionIdGenerator for ServerIdGenerator { cid.push((cid_len - 1) as u8); cid.extend(self.server_id.0.iter()); cid.extend(rand::rng().random_iter::().take(self.nonce_len)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -1022,7 +1047,7 @@ mod tests { ..Default::default() }; let tls = config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config, quic: quic.clone(), member: None, @@ -1040,7 +1065,7 @@ mod tests { ..Default::default() }; let tls = config.tls.build().expect("tls config"); - let client = NoqClient::new(&config, &quic).expect("client init"); + let client = Client::new(&config, &quic).expect("client init"); let url: Url = format!("moqt://{addr}/.cluster/test").parse().unwrap(); let versions = moq_net::Versions::default(); let dial = client.connect(&tls, url.into(), &versions); @@ -1054,18 +1079,20 @@ mod tests { let (_client, _server) = result.expect("pinned connection failed"); } - /// noq exposes no getters for the flow-control windows, but its `Debug` prints + /// moq-quic exposes no getters for the flow-control windows, but its `Debug` prints /// them, which is enough to prove each one reached the transport config, that an - /// unset connection window gets our finite default instead of noq's unlimited one, - /// and that the other unset knobs leave noq's own defaults in place. + /// unset connection window gets our finite default instead of moq-quic's unlimited one, + /// and that the other unset knobs leave moq-quic's own defaults in place. #[test] fn apply_windows_writes_each_field() { - let mut quic = crate::quic::Config::default(); - quic.receive_window = Some(128 << 20); - quic.stream_receive_window = Some(8 << 20); - quic.send_window = Some(32 << 20); + let quic = crate::quic::Config { + receive_window: Some(128 << 20), + stream_receive_window: Some(8 << 20), + send_window: Some(32 << 20), + ..Default::default() + }; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); apply_windows(&mut transport, &quic.resolve()); let applied = format!("{:?}", transport); @@ -1077,33 +1104,33 @@ mod tests { assert!(applied.contains(&field), "{field} missing from {applied}"); } - let mut untouched = noq::TransportConfig::default(); + let mut untouched = driver::TransportConfig::default(); apply_windows(&mut untouched, &crate::quic::Config::default().resolve()); - let mut expected = noq::TransportConfig::default(); - expected.receive_window(noq::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); + let mut expected = driver::TransportConfig::default(); + expected.receive_window(driver::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); assert_eq!(format!("{:?}", untouched), format!("{:?}", expected)); assert_ne!( format!("{:?}", untouched), - format!("{:?}", noq::TransportConfig::default()), - "noq's own connection window is unlimited, so ours must replace it" + format!("{:?}", driver::TransportConfig::default()), + "moq-quic's own connection window is unlimited, so ours must replace it" ); } /// Build a controller from each family's factory and downcast it to the - /// concrete noq implementation it must map to. + /// concrete moq-quic implementation it must map to. #[test] fn congestion_factory_maps_each_family() { - let now = std::time::Instant::now(); + let now = moq_clock::Instant::now(); let mtu = 1200; let loss = congestion_factory(CongestionControl::Loss).build(now, mtu); - assert!(loss.into_any().downcast::().is_ok()); + assert!(loss.into_any().downcast::().is_ok()); let delay = congestion_factory(CongestionControl::Delay).build(now, mtu); - assert!(delay.into_any().downcast::().is_ok()); + assert!(delay.into_any().downcast::().is_ok()); } - /// Loopback regression test: with the knob unset, live noq connections must run + /// Loopback regression test: with the knob unset, live QUIC connections must run /// BBRv3 on both ends. #[tokio::test] async fn default_reaches_the_live_connection() { @@ -1120,7 +1147,7 @@ mod tests { let quic = crate::quic::Config::default(); let tls = server_config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config: server_config, quic: quic.clone(), member: None, @@ -1146,7 +1173,7 @@ mod tests { }; let tls = client_config.tls.build().expect("tls config"); - let client = NoqClient::new(&client_config, &quic).expect("client init"); + let client = Client::new(&client_config, &quic).expect("client init"); // Dial the loopback IP directly so the system resolver is never involved. let url: Url = format!("moqt://127.0.0.1:{}", addr.port()).parse().unwrap(); @@ -1158,7 +1185,7 @@ mod tests { .await .expect("connect failed"); - // web_transport_moq::Session derefs to the noq connection. + // web::Session derefs to the driver connection. assert!(is_bbr3(&session), "client connection is not running BBRv3"); assert!( accepted.await.expect("server task panicked"), @@ -1169,15 +1196,11 @@ mod tests { .expect("test timed out"); } - /// Whether a live connection's initial path is running BBRv3. - /// - /// noq is multipath, so the controller is per path rather than per connection; - /// `PathId::ZERO` is the path the handshake came up on. - fn is_bbr3(conn: &noq::Connection) -> bool { - conn.congestion_state(noq::PathId::ZERO) - .expect("no controller on the initial path") + /// Whether a live connection is running BBRv3. + fn is_bbr3(conn: &driver::Connection) -> bool { + conn.congestion_state() .into_any() - .downcast::() + .downcast::() .is_ok() } } diff --git a/rs/moq-tokio/src/quic/driver/.rustfmt.toml b/rs/moq-tokio/src/quic/driver/.rustfmt.toml new file mode 100644 index 0000000000..daf6479bc8 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/.rustfmt.toml @@ -0,0 +1,2 @@ +# quinn's formatting (rustfmt defaults) rather than rs/.rustfmt.toml, so +# upstream fixes cherry-pick without whitespace conflicts. diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 6ce968db47..605163a673 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -23,15 +23,11 @@ use tokio::sync::{ }; use tracing::{Instrument, Span, debug_span}; -use crate::{ - ConnectionEvent, Duration, Instant, VarInt, - mutex::Mutex, - recv_stream::RecvStream, - runtime::{AsyncTimer, Runtime, UdpSender}, - send_stream::SendStream, - udp_transmit, +use crate::quic::driver::{ + ConnectionEvent, Duration, Instant, VarInt, mutex::Mutex, recv_stream::RecvStream, + runtime::UdpSender, send_stream::SendStream, udp_transmit, }; -use proto::{ +use moq_quic::{ ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, StreamId, TransportError, TransportErrorCode, congestion::Controller, }; @@ -53,11 +49,10 @@ pub struct Connecting { impl Connecting { pub(crate) fn new( handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, sender: Pin>, - runtime: Arc, ) -> Self { let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); @@ -69,21 +64,20 @@ impl Connecting { conn_events, on_handshake_data_send, sender, - runtime.clone(), )), shared: Shared::default(), })); let connected = Box::pin(conn.shared.connected.clone().notified_owned()); let driver = ConnectionDriver::new(conn.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {e}"); } } .instrument(Span::current()), - )); + ); Self { conn: Some(conn), @@ -104,7 +98,7 @@ impl Connecting { /// ## Outgoing /// /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends - /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig] /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may @@ -119,14 +113,14 @@ impl Connecting { /// relevant resumption state to be stored in the server, which servers may limit or lose for /// various reasons including not persisting resumption state across server restarts. /// - /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// If manually providing a [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Incoming /// /// For incoming connections, conversion to 0.5-RTT will always succeed. /// - /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// If manually providing a [`crypto::ServerConfig`][crate::quic::driver::crypto::ServerConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Security @@ -156,9 +150,9 @@ impl Connecting { /// Parameters negotiated during the handshake /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a - /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// [`crypto::rustls::HandshakeData`](crate::quic::driver::crypto::rustls::HandshakeData). /// /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. /// @@ -194,7 +188,7 @@ impl Connecting { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. /// /// Will panic if called after `poll` has returned `Ready`. @@ -341,9 +335,9 @@ impl Connection { /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing /// anything to [`SendStream`] will never succeed. /// - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream pub fn open_bi(&self) -> OpenBi<'_> { OpenBi { conn: &self.0, @@ -369,10 +363,10 @@ impl Connection { /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. /// - /// [`accept_bi()`]: crate::Connection::accept_bi - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`accept_bi()`]: crate::quic::driver::Connection::accept_bi + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream /// /// # Cancel safety /// @@ -462,8 +456,8 @@ impl Connection { /// undelivered to the application, including data that was acknowledged as received to /// the local endpoint. /// - /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed - /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`ConnectionError::LocallyClosed`]: crate::quic::driver::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::quic::driver::Endpoint::wait_idle /// [`close()`]: Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let conn = &mut *self.0.state.lock("close"); @@ -515,7 +509,7 @@ impl Connection { if let Some(x) = &conn.error { return Err(SendDatagramError::ConnectionLost(x.clone())); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match conn.inner.datagrams().send(data, true) { Ok(()) => { conn.wake(); @@ -614,7 +608,7 @@ impl Connection { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. pub fn local_ip(&self) -> Option { self.0.state.lock("local_ip").inner.local_ip() @@ -678,7 +672,7 @@ impl Connection { /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on /// the returned value. /// - /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + /// [`Connection::handshake_data()`]: crate::quic::driver::Connecting::handshake_data pub fn handshake_data(&self) -> Option> { self.0 .state @@ -691,7 +685,7 @@ impl Connection { /// Cryptographic identity of the peer /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. /// /// For custom asynchronous validation (e.g. checking a certificate against an external @@ -738,7 +732,7 @@ impl Connection { output: &mut [u8], label: &[u8], context: &[u8], - ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + ) -> Result<(), moq_quic::crypto::ExportKeyingMaterialError> { self.0 .state .lock("export_keying_material") @@ -758,14 +752,14 @@ impl Connection { conn.wake(); } - /// See [`proto::TransportConfig::send_window()`] + /// See [`moq_quic::TransportConfig::send_window()`] pub fn set_send_window(&self, send_window: u64) { let mut conn = self.0.state.lock("set_send_window"); conn.inner.set_send_window(send_window); conn.wake(); } - /// See [`proto::TransportConfig::receive_window()`] + /// See [`moq_quic::TransportConfig::receive_window()`] pub fn set_receive_window(&self, receive_window: VarInt) { let mut conn = self.0.state.lock("set_receive_window"); conn.inner.set_receive_window(receive_window); @@ -970,7 +964,7 @@ impl Future for SendDatagram<'_> { if let Some(e) = &state.error { return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match state .inner .datagrams() @@ -1084,13 +1078,13 @@ pub(crate) struct Shared { } pub(crate) struct State { - pub(crate) inner: proto::Connection, + pub(crate) inner: moq_quic::Connection, driver: Option, handle: ConnectionHandle, on_handshake_data: Option>, connected: bool, handshake_confirmed: bool, - timer: Option>>, + timer: Option, timer_deadline: Option, conn_events: mpsc::UnboundedReceiver, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, @@ -1100,23 +1094,21 @@ pub(crate) struct State { /// Always set to Some before the connection becomes drained pub(crate) error: Option, sender: Pin>, - runtime: Arc, send_buffer: Vec, /// We buffer a transmit when the underlying I/O would block - buffered_transmit: Option, + buffered_transmit: Option, remote_address: watch::Sender, } impl State { #[allow(clippy::too_many_arguments)] fn new( - inner: proto::Connection, + inner: moq_quic::Connection, handle: ConnectionHandle, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, on_handshake_data: oneshot::Sender<()>, sender: Pin>, - runtime: Arc, ) -> Self { let remote_address = inner.remote_address(); @@ -1136,7 +1128,6 @@ impl State { stopped: FxHashMap::default(), error: None, sender, - runtime, send_buffer: Vec::new(), buffered_transmit: None, remote_address: watch::Sender::new(remote_address), @@ -1144,7 +1135,7 @@ impl State { } fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { - let now = self.runtime.now(); + let now = Instant::now(); let mut transmits = 0; let max_datagrams = self @@ -1241,7 +1232,7 @@ impl State { fn forward_app_events(&mut self, shared: &Shared) { while let Some(event) = self.inner.poll() { - use proto::Event::*; + use moq_quic::Event::*; match event { HandshakeDataReady => { if let Some(x) = self.on_handshake_data.take() { @@ -1310,7 +1301,7 @@ impl State { // Use the clock rather than the async timer to detect expiry: Sleep::poll // respects Tokio's cooperative budget and can return Pending for elapsed // deadlines. - let now = self.runtime.now(); + let now = Instant::now(); if now >= deadline { self.inner.handle_timeout(now); self.timer_deadline = None; @@ -1320,26 +1311,22 @@ impl State { match &mut self.timer { // Avoid resetting the timer when the deadline is unchanged. Some(delay) if self.timer_deadline != Some(deadline) => { - delay.as_mut().reset(deadline); + delay.reset(deadline); } None => { - self.timer = Some(self.runtime.new_timer(deadline)); + self.timer = Some(moq_clock::sleep_until(deadline)); } _ => {} } self.timer_deadline = Some(deadline); - let delay = self - .timer - .as_mut() - .expect("timer must exist in this state") - .as_mut(); - if delay.poll(cx).is_pending() { + let delay = self.timer.as_mut().expect("timer must exist in this state"); + if Pin::new(delay).poll(cx).is_pending() { return false; } // The deadline elapsed in the window between the clock check and poll. - self.inner.handle_timeout(self.runtime.now()); + self.inner.handle_timeout(Instant::now()); self.timer_deadline = None; true } @@ -1372,7 +1359,7 @@ impl State { } fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { - self.inner.close(self.runtime.now(), error_code, reason); + self.inner.close(Instant::now(), error_code, reason); self.terminate(ConnectionError::LocallyClosed, shared); self.wake(); } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2012e415ff..2bae9031dc 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -14,35 +14,24 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] -use crate::runtime::default_runtime; -use crate::{ +use crate::quic::driver::{ Instant, - runtime::{AsyncUdpSocket, Runtime, UdpSender}, + runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}, udp_transmit, }; use bytes::{Bytes, BytesMut}; -use pin_project_lite::pin_project; -use proto::{ - self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, - EndpointEvent, ServerConfig, +use moq_quic::{ + ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, EndpointEvent, + ServerConfig, }; -use rustc_hash::FxHashMap; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] +use moq_sock::udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; +use pin_project_lite::pin_project; +use rustc_hash::{FxHashMap, FxHashSet}; use socket2::{Domain, Protocol, Socket, Type}; use tokio::sync::{Notify, futures::Notified, mpsc}; use tracing::{Instrument, Span}; -use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; -use crate::{ +use crate::quic::driver::{ ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, }; @@ -56,7 +45,6 @@ use crate::{ #[derive(Debug, Clone)] pub struct Endpoint { pub(crate) inner: EndpointRef, - runtime: Arc, } impl Endpoint { @@ -77,11 +65,6 @@ impl Endpoint { /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn client(addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -90,13 +73,10 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), None, - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } @@ -119,11 +99,6 @@ impl Endpoint { /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never /// dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -132,26 +107,21 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), Some(config), - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } /// Construct an endpoint with arbitrary configuration and socket - #[cfg(not(wasm_browser))] pub fn new( config: EndpointConfig, server_config: Option, socket: std::net::UdpSocket, - runtime: Arc, ) -> io::Result { - let socket = runtime.wrap_udp_socket(socket)?; - Self::new_with_abstract_socket(config, server_config, socket, runtime) + let socket = wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket) } /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket @@ -162,32 +132,30 @@ impl Endpoint { config: EndpointConfig, server_config: Option, socket: Box, - runtime: Arc, ) -> io::Result { let addr = socket.local_addr()?; let allow_mtud = !socket.may_fragment(); let rc = EndpointRef::new( socket, - proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + moq_quic::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), addr.is_ipv6(), - runtime.clone(), ); let driver = EndpointDriver(rc.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {}", e); } } .instrument(Span::current()), - )); - Ok(Self { inner: rc, runtime }) + ); + Ok(Self { inner: rc }) } /// Get the next incoming connection attempt from a client /// /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] - /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// can be `await`ed to obtain the final [`Connection`](crate::quic::driver::Connection), or used to e.g. /// filter connection attempts or force address validation, or converted into an intermediate /// `Connecting` future which can be used to e.g. send 0.5-RTT data. pub fn accept(&self) -> Accept<'_> { @@ -252,22 +220,21 @@ impl Endpoint { let (ch, conn) = endpoint .inner - .connect(self.runtime.now(), config, addr, server_name)?; + .connect(Instant::now(), config, addr, server_name)?; let sender = endpoint.socket.create_sender(); endpoint.stats.outgoing_handshakes += 1; Ok(endpoint .recv_state .connections - .insert(ch, conn, sender, false, self.runtime.clone())) + .insert(ch, conn, sender, false)) } /// Switch to a new UDP socket /// /// See [`Endpoint::rebind_abstract()`] for details. - #[cfg(not(wasm_browser))] pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { - self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + self.rebind_abstract(wrap_udp_socket(socket)?) } /// Switch to a new UDP socket @@ -322,7 +289,7 @@ impl Endpoint { /// /// See [`Connection::close()`] for details. /// - /// [`Connection::close()`]: crate::Connection::close + /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); let mut endpoint = self.inner.state.lock().unwrap(); @@ -360,6 +327,32 @@ impl Endpoint { .await; } } + + /// Wait until no connection on the endpoint is still active + /// + /// A connection stops being active once its peer's close arrives, or once its closing + /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out + /// the closing period for a peer that already has the close, nor wait for the + /// [`Connection`](crate::quic::driver::Connection) handles to drop. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + /// [`wait_idle()`]: Endpoint::wait_idle + pub async fn wait_all_draining(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.recv_state.connections.all_draining() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.all_draining.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -400,7 +393,7 @@ impl Future for EndpointDriver { None => endpoint.driver = Some(cx.waker().clone()), } - let now = endpoint.runtime.now(); + let now = Instant::now(); let mut keep_going = false; keep_going |= endpoint.drive_recv(cx, now)?; keep_going |= endpoint.handle_events(cx, &self.0.shared); @@ -432,6 +425,8 @@ impl Drop for EndpointDriver { // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); + endpoint.recv_state.connections.draining.clear(); + self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. @@ -450,7 +445,7 @@ pub(crate) struct EndpointInner { impl EndpointInner { pub(crate) fn accept( &self, - incoming: proto::Incoming, + incoming: moq_quic::Incoming, server_config: Option>, ) -> Result { let mut response_buffer = Vec::new(); @@ -458,7 +453,7 @@ impl EndpointInner { // Phase 1: reserve endpoint state for the connection under the lock. let accepting = { let mut state = self.state.lock().unwrap(); - let now = state.runtime.now(); + let now = Instant::now(); match state .inner .start_accept(incoming, now, &mut response_buffer, server_config) @@ -487,12 +482,11 @@ impl EndpointInner { Ok((handle, conn)) => { state.stats.accepted_handshakes += 1; let sender = state.socket.create_sender(); - let runtime = state.runtime.clone(); let driver_lost = state.driver_lost; Ok(state .recv_state .connections - .insert(handle, conn, sender, driver_lost, runtime)) + .insert(handle, conn, sender, driver_lost)) } Err(error) => { if let Some(transmit) = error.response { @@ -509,7 +503,7 @@ impl EndpointInner { result } - pub(crate) fn refuse(&self, incoming: proto::Incoming) { + pub(crate) fn refuse(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.refused_handshakes += 1; let mut response_buffer = Vec::new(); @@ -517,7 +511,7 @@ impl EndpointInner { respond(transmit, &response_buffer, &mut state.sender); } - pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + pub(crate) fn retry(&self, incoming: moq_quic::Incoming) -> Result<(), moq_quic::RetryError> { let mut state = self.state.lock().unwrap(); let mut response_buffer = Vec::new(); let transmit = state.inner.retry(incoming, &mut response_buffer)?; @@ -525,7 +519,7 @@ impl EndpointInner { Ok(()) } - pub(crate) fn ignore(&self, incoming: proto::Incoming) { + pub(crate) fn ignore(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.ignored_handshakes += 1; state.inner.ignore(incoming); @@ -539,13 +533,12 @@ pub(crate) struct State { /// During an active migration, abandoned_socket receives traffic /// until the first packet arrives on the new socket. prev_socket: Option>, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, recv_state: RecvState, driver: Option, ipv6: bool, events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, driver_lost: bool, - runtime: Arc, stats: EndpointStats, default_client_config: Option, /// Connections in the process of being accepted @@ -556,13 +549,14 @@ pub(crate) struct State { pub(crate) struct Shared { incoming: Notify, idle: Notify, + all_draining: Notify, /// Number of live handles that can be used to initiate or handle I/O; excludes the driver ref_count: AtomicUsize, } impl State { fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { - let get_time = || self.runtime.now(); + let get_time = Instant::now; self.recv_state.recv_limiter.start_cycle(get_time); if let Some(socket) = &mut self.prev_socket { // We don't care about the `PollProgress` from old sockets. @@ -571,7 +565,6 @@ impl State { &mut self.inner, &mut **socket, &mut self.sender, - &*self.runtime, now, ); if poll_res.is_err() { @@ -583,7 +576,6 @@ impl State { &mut self.inner, &mut *self.socket, &mut self.sender, - &*self.runtime, now, ); self.recv_state.recv_limiter.finish_cycle(get_time); @@ -606,12 +598,22 @@ impl State { } }; + let connections = &mut self.recv_state.connections; + if event.is_draining() && connections.senders.contains_key(&ch) { + connections.draining.insert(ch); + } if event.is_drained() { - self.recv_state.connections.senders.remove(&ch); + connections.senders.remove(&ch); + connections.draining.remove(&ch); if self.is_idle() { shared.idle.notify_waiters(); } } + if (event.is_draining() || event.is_drained()) + && self.recv_state.connections.all_draining() + { + shared.all_draining.notify_waiters(); + } let Some(event) = self.inner.handle_event(ch, event) else { continue; }; @@ -642,7 +644,7 @@ impl Drop for State { } fn respond( - transmit: proto::Transmit, + transmit: moq_quic::Transmit, response_buffer: &[u8], sender: &mut Pin>, ) { @@ -692,11 +694,11 @@ fn respond( } #[inline] -fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { +fn proto_ecn(ecn: moq_sock::udp::EcnCodepoint) -> moq_quic::EcnCodepoint { match ecn { - udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, - udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, - udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + moq_sock::udp::EcnCodepoint::Ect0 => moq_quic::EcnCodepoint::Ect0, + moq_sock::udp::EcnCodepoint::Ect1 => moq_quic::EcnCodepoint::Ect1, + moq_sock::udp::EcnCodepoint::Ce => moq_quic::EcnCodepoint::Ce, } } @@ -704,6 +706,9 @@ fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { struct ConnectionSet { /// Senders for communicating with the endpoint's connections senders: FxHashMap>, + /// The connections in `senders` that are no longer active: closed by the peer, or past + /// their closing period + draining: FxHashSet, /// Stored to give out clones to new ConnectionInners sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, /// Set if the endpoint has been manually closed @@ -714,10 +719,9 @@ impl ConnectionSet { fn insert( &mut self, handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, sender: Pin>, driver_lost: bool, - runtime: Arc, ) -> Connecting { let (send, recv) = mpsc::unbounded_channel(); if let Some((error_code, reason)) = &self.close { @@ -735,12 +739,17 @@ impl ConnectionSet { self.senders.insert(handle, send); } } - Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + Connecting::new(handle, conn, self.sender.clone(), recv, sender) } fn is_empty(&self) -> bool { self.senders.is_empty() } + + /// Whether no connection is still active + fn all_draining(&self) -> bool { + self.senders.len() == self.draining.len() + } } fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { @@ -795,9 +804,8 @@ pub(crate) struct EndpointRef(Arc); impl EndpointRef { pub(crate) fn new( socket: Box, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, ipv6: bool, - runtime: Arc, ) -> Self { let (sender, events) = mpsc::unbounded_channel(); let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); @@ -806,6 +814,7 @@ impl EndpointRef { shared: Shared { incoming: Notify::new(), idle: Notify::new(), + all_draining: Notify::new(), ref_count: AtomicUsize::new(0), }, state: Mutex::new(State { @@ -818,7 +827,6 @@ impl EndpointRef { driver: None, driver_lost: false, recv_state, - runtime, stats: EndpointStats::default(), default_client_config: None, pending_accepts: 0, @@ -858,7 +866,7 @@ impl std::ops::Deref for EndpointRef { /// State directly involved in handling incoming packets struct RecvState { - incoming: VecDeque, + incoming: VecDeque, connections: ConnectionSet, recv_buf: Box<[u8]>, /// Reused buffer that received datagrams are copied into and split into owned slices. @@ -870,7 +878,7 @@ impl RecvState { fn new( sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, max_receive_segments: usize, - endpoint: &proto::Endpoint, + endpoint: &moq_quic::Endpoint, ) -> Self { let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) as usize @@ -879,6 +887,7 @@ impl RecvState { Self { connections: ConnectionSet { senders: FxHashMap::default(), + draining: FxHashSet::default(), sender, close: None, }, @@ -892,10 +901,9 @@ impl RecvState { fn poll_socket( &mut self, cx: &mut Context<'_>, - endpoint: &mut proto::Endpoint, + endpoint: &mut moq_quic::Endpoint, socket: &mut dyn AsyncUdpSocket, sender: &mut Pin>, - runtime: &dyn Runtime, now: Instant, ) -> Result { let mut received_connection_packet = false; @@ -990,7 +998,7 @@ impl RecvState { return Err(e); } } - if !self.recv_limiter.allow_work(|| runtime.now()) { + if !self.recv_limiter.allow_work(Instant::now) { return Ok(PollProgress { received_connection_packet, keep_going: true, diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs index b5d60a08ec..64a08e493c 100644 --- a/rs/moq-tokio/src/quic/driver/incoming.rs +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -6,10 +6,10 @@ use std::{ task::{Context, Poll}, }; -use proto::{ConnectionError, ConnectionId, ServerConfig}; +use moq_quic::{ConnectionError, ConnectionId, ServerConfig}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ connection::{Connecting, Connection}, endpoint::EndpointRef, }; @@ -19,7 +19,7 @@ use crate::{ pub struct Incoming(Option); impl Incoming { - pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + pub(crate) fn new(inner: moq_quic::Incoming, endpoint: EndpointRef) -> Self { Self(Some(State { inner, endpoint })) } @@ -111,7 +111,7 @@ impl Drop for Incoming { #[derive(Debug)] struct State { - inner: proto::Incoming, + inner: moq_quic::Incoming, endpoint: EndpointRef, } diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index 649801fff8..d758cc9c7c 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -50,14 +50,11 @@ mod runtime; mod send_stream; mod work_limiter; -#[cfg(not(wasm_browser))] -pub(crate) use std::time::{Duration, Instant}; -#[cfg(wasm_browser)] -pub(crate) use web_time::{Duration, Instant}; +pub(crate) use moq_clock::Instant; +pub(crate) use std::time::Duration; -#[cfg(feature = "bloom")] -pub use proto::BloomTokenLog; -pub use proto::{ +pub use moq_quic::BloomTokenLog; +pub use moq_quic::{ AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, @@ -67,42 +64,37 @@ pub use proto::{ VarIntBoundsExceeded, Written, congestion, crypto, }; #[cfg(feature = "qlog")] -pub use proto::{QlogConfig, QlogStream}; -#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] -pub use rustls; -pub use udp; +pub use moq_quic::{QlogConfig, QlogStream}; -pub use crate::connection::{ +pub use crate::quic::driver::connection::{ AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, RemoteAddressWatcher, SendDatagram, SendDatagramError, }; -pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; -pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; -pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; -#[cfg(feature = "runtime-smol")] -pub use crate::runtime::SmolRuntime; -#[cfg(feature = "runtime-tokio")] -pub use crate::runtime::TokioRuntime; -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -pub use crate::runtime::default_runtime; -pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; -pub use crate::send_stream::{SendStream, StoppedError, WriteError}; +pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::quic::driver::recv_stream::{ + ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, +}; +#[cfg(test)] +pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; +pub use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender}; +pub use crate::quic::driver::send_stream::{SendStream, StoppedError, WriteError}; #[cfg(test)] mod tests; #[derive(Debug)] -enum ConnectionEvent { +pub(crate) enum ConnectionEvent { Close { error_code: VarInt, reason: bytes::Bytes, }, - Proto(proto::ConnectionEvent), + Proto(moq_quic::ConnectionEvent), Rebind(Pin>), } -fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { - udp::Transmit { +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> moq_sock::udp::Transmit<'a> { + moq_sock::udp::Transmit { destination: t.destination, ecn: t.ecn.map(udp_ecn), contents: buffer, @@ -111,11 +103,11 @@ fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { } } -fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { +fn udp_ecn(ecn: EcnCodepoint) -> moq_sock::udp::EcnCodepoint { match ecn { - EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, - EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, - EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + EcnCodepoint::Ect0 => moq_sock::udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => moq_sock::udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => moq_sock::udp::EcnCodepoint::Ce, } } diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs index 91962cb072..d88d40e32a 100644 --- a/rs/moq-tokio/src/quic/driver/mutex.rs +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -3,116 +3,6 @@ use std::{ ops::{Deref, DerefMut}, }; -#[cfg(feature = "lock_tracking")] -mod tracking { - use super::*; - use crate::{Duration, Instant}; - use std::collections::VecDeque; - use tracing::warn; - - #[derive(Debug)] - struct Inner { - last_lock_owner: VecDeque<(&'static str, Duration)>, - value: T, - } - - /// A Mutex which optionally allows to track the time a lock was held and - /// emit warnings in case of excessive lock times - pub(crate) struct Mutex { - inner: std::sync::Mutex>, - } - - impl Debug for Mutex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Debug::fmt(&self.inner, f) - } - } - - impl Mutex { - pub(crate) fn new(value: T) -> Self { - Self { - inner: std::sync::Mutex::new(Inner { - last_lock_owner: VecDeque::new(), - value, - }), - } - } - - /// Acquires the lock for a certain purpose - /// - /// The purpose will be recorded in the list of last lock owners - pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { - // We don't bother dispatching through Runtime::now because they're pure performance - // diagnostics. - let now = Instant::now(); - let guard = self.inner.lock().unwrap(); - - let lock_time = Instant::now(); - let elapsed = lock_time.duration_since(now); - - if elapsed > Duration::from_millis(1) { - warn!( - "Locking the connection for {} took {:?}. Last owners: {:?}", - purpose, elapsed, guard.last_lock_owner - ); - } - - MutexGuard { - guard, - start_time: lock_time, - purpose, - } - } - } - - pub(crate) struct MutexGuard<'a, T> { - guard: std::sync::MutexGuard<'a, Inner>, - start_time: Instant, - purpose: &'static str, - } - - impl Drop for MutexGuard<'_, T> { - fn drop(&mut self) { - if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { - self.guard.last_lock_owner.pop_back(); - } - - let duration = self.start_time.elapsed(); - - if duration > Duration::from_millis(1) { - warn!( - "Utilizing the connection for {} took {:?}", - self.purpose, duration - ); - } - - self.guard - .last_lock_owner - .push_front((self.purpose, duration)); - } - } - - impl Deref for MutexGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.guard.value - } - } - - impl DerefMut for MutexGuard<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.guard.value - } - } - - const MAX_LOCK_OWNERS: usize = 20; -} - -#[cfg(feature = "lock_tracking")] -pub(crate) use tracking::Mutex; - -#[cfg(not(feature = "lock_tracking"))] mod non_tracking { use super::*; @@ -159,5 +49,4 @@ mod non_tracking { } } -#[cfg(not(feature = "lock_tracking"))] pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index d3548a1a33..027349ae12 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -6,11 +6,11 @@ use std::{ }; use bytes::Bytes; -use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; use thiserror::Error; use tokio::io::ReadBuf; -use crate::{VarInt, connection::ConnectionRef}; +use crate::quic::driver::{VarInt, connection::ConnectionRef}; /// A stream that can only be used to receive data /// @@ -42,12 +42,12 @@ use crate::{VarInt, connection::ConnectionRef}; /// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver /// will be bidirectional stream 0. /// -/// [`ReadError`]: crate::ReadError +/// [`ReadError`]: crate::quic::driver::ReadError /// [`stop()`]: RecvStream::stop -/// [`SendStream::finish`]: crate::SendStream::finish -/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`SendStream::finish`]: crate::quic::driver::SendStream::finish +/// [`WriteError::Stopped`]: crate::quic::driver::WriteError::Stopped /// [`id`]: RecvStream::id -/// [`Connection::accept_bi`]: crate::Connection::accept_bi +/// [`Connection::accept_bi`]: crate::quic::driver::Connection::accept_bi #[derive(Debug)] pub struct RecvStream { conn: ConnectionRef, @@ -104,7 +104,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read( &mut self, cx: &mut Context<'_>, @@ -126,7 +126,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read_buf( &mut self, cx: &mut Context<'_>, @@ -256,7 +256,7 @@ impl RecvStream { /// /// This operation is *not* cancel-safe. /// - /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + /// [`ReadToEndError::TooLong`]: crate::quic::driver::ReadToEndError::TooLong pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { ReadToEnd { stream: self, @@ -286,13 +286,13 @@ impl RecvStream { /// Check if this stream predates completion of the handshake on an incoming connection. /// /// True only if the stream was accepted before the handshake completed, which is only possible - /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// if you successfully called [`Connecting::into_0rtt`](crate::quic::driver::Connecting::into_0rtt) and the /// client chose to send 0-RTT data. /// /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application /// data may be a replay attack. To guard against this, applications should not execute /// non-idempotent operations until - /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + /// [`Connection::authenticated`](crate::quic::driver::Connection::authenticated) succeeds. pub fn is_0rtt(&self) -> bool { self.is_0rtt } @@ -306,7 +306,7 @@ impl RecvStream { /// /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was - /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// [`finish()`](crate::quic::driver::SendStream::finish)ed by the peer and all data has been received, after /// which it is no longer meaningful for the stream to be reset. /// /// This operation is cancel-safe. @@ -360,7 +360,7 @@ impl RecvStream { where T: FnMut(&mut Chunks<'_>) -> ReadStatus, { - use proto::ReadError::*; + use moq_quic::ReadError::*; use std::collections::hash_map::Entry; if self.all_data_read { @@ -432,11 +432,11 @@ impl RecvStream { enum ReadStatus { Readable(T), Finished(Option), - Failed(Option, proto::ReadError), + Failed(Option, moq_quic::ReadError), } -impl From<(Option, Option)> for ReadStatus { - fn from(status: (Option, Option)) -> Self { +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { match status { (read, None) => Self::Finished(read), (read, Some(e)) => Self::Failed(read, e), @@ -446,7 +446,7 @@ impl From<(Option, Option)> for ReadStatus { /// Future produced by [`RecvStream::read_to_end()`]. /// -/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +/// [`RecvStream::read_to_end()`]: crate::quic::driver::RecvStream::read_to_end struct ReadToEnd<'a> { stream: &'a mut RecvStream, buffer: ReadToEndBuffer, @@ -515,19 +515,6 @@ pub enum ReadToEndError { TooLong, } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncRead for RecvStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut [u8], - ) -> Poll> { - let mut buf = ReadBuf::new(buf); - ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; - Poll::Ready(Ok(buf.filled().len())) - } -} - impl tokio::io::AsyncRead for RecvStream { fn poll_read( self: Pin<&mut Self>, @@ -594,7 +581,7 @@ pub enum ReadError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -640,7 +627,7 @@ pub enum ResetError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -658,7 +645,7 @@ impl From for io::Error { /// Future produced by [`RecvStream::read()`]. /// -/// [`RecvStream::read()`]: crate::RecvStream::read +/// [`RecvStream::read()`]: crate::quic::driver::RecvStream::read struct Read<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -679,7 +666,7 @@ impl Future for Read<'_> { /// Future produced by [`RecvStream::read_exact()`]. /// -/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +/// [`RecvStream::read_exact()`]: crate::quic::driver::RecvStream::read_exact struct ReadExact<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -715,7 +702,7 @@ pub enum ReadExactError { /// Future produced by [`RecvStream::read_chunk()`]. /// -/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +/// [`RecvStream::read_chunk()`]: crate::quic::driver::RecvStream::read_chunk struct ReadChunk<'a> { stream: &'a mut RecvStream, max_length: usize, @@ -732,7 +719,7 @@ impl Future for ReadChunk<'_> { /// Future produced by [`RecvStream::read_chunks()`]. /// -/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +/// [`RecvStream::read_chunks()`]: crate::quic::driver::RecvStream::read_chunks struct ReadChunks<'a> { stream: &'a mut RecvStream, bufs: &'a mut [Bytes], diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs index 9898a0e879..cc89f2f90e 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/mod.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -1,5 +1,3 @@ -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -use std::sync::Arc; use std::{ fmt::{self, Debug}, future::Future, @@ -10,35 +8,9 @@ use std::{ time::Duration, }; -use udp::{RecvMeta, Transmit}; - -use crate::Instant; - -/// Abstracts I/O and timer operations for runtime independence -pub trait Runtime: Send + Sync + Debug + 'static { - /// Construct a timer that will expire at `i` - fn new_timer(&self, i: Instant) -> Pin>; - /// Drive `future` to completion in the background - #[track_caller] - fn spawn(&self, future: Pin + Send>>); - /// Convert `t` into the socket type used by this runtime - #[cfg(not(wasm_browser))] - fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; - /// Look up the current time - /// - /// Allows simulating the flow of time for testing. - fn now(&self) -> Instant { - Instant::now() - } -} +use moq_sock::udp::{RecvMeta, Transmit}; -/// Abstract implementation of an async timer for runtime independence -pub trait AsyncTimer: Send + Debug + 'static { - /// Update the timer to expire at `i` - fn reset(self: Pin<&mut Self>, i: Instant); - /// Check whether the timer has expired, and register to be woken if not - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; -} +use crate::quic::driver::Instant; /// Abstract implementation of a UDP socket for runtime independence pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { @@ -143,7 +115,6 @@ impl /// that resolves once the socket is write-ready. /// /// See also the bounds on this struct's [`UdpSender`] implementation. - #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { Self { socket: inner, @@ -215,7 +186,7 @@ fn log_sendmsg_error( ) { #[cfg(unix)] // Unix `EMSGSIZE` is expected for MTU probes. - if udp::is_msg_size_err(error) { + if moq_sock::udp::is_msg_size_err(error) { return; } @@ -253,41 +224,10 @@ trait UdpSenderHelperSocket: Send + Sync + 'static { fn max_transmit_segments(&self) -> usize; } -/// Automatically select an appropriate runtime from those enabled at compile time -/// -/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, -/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is -/// returned. Otherwise, `None` is returned. -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -#[allow(clippy::needless_return)] // Be sure we return the right thing -pub fn default_runtime() -> Option> { - #[cfg(feature = "runtime-tokio")] - { - if ::tokio::runtime::Handle::try_current().is_ok() { - return Some(Arc::new(TokioRuntime)); - } - } - - #[cfg(feature = "runtime-smol")] - { - return Some(Arc::new(SmolRuntime)); - } - - #[cfg(not(feature = "runtime-smol"))] - None -} - -#[cfg(feature = "runtime-tokio")] mod tokio; -#[cfg(feature = "runtime-tokio")] -pub use tokio::TokioRuntime; - -#[cfg(feature = "runtime-smol")] -mod smol; -#[cfg(feature = "runtime-smol")] -pub use smol::*; +pub(crate) use tokio::wrap_udp_socket; -#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +#[cfg(test)] mod tests { use std::{future, task::Waker}; diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs deleted file mode 100644 index b5ae3d0cf2..0000000000 --- a/rs/moq-tokio/src/quic/driver/runtime/smol.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::{ - future::Future, - pin::Pin, - task::{Context, Poll}, - time::Instant, -}; -use std::{io, sync::Arc, task::ready}; - -use async_io::Async; -use async_io::Timer; - -use super::AsyncTimer; -use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for smol -#[derive(Debug)] -pub struct SmolRuntime; - -impl Runtime for SmolRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(Timer::at(t)) - } - - fn spawn(&self, future: Pin + Send>>) { - ::smol::spawn(future).detach(); - } - - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket::new(sock)?)) - } -} - -impl AsyncTimer for Timer { - fn reset(mut self: Pin<&mut Self>, t: Instant) { - self.set_at(t) - } - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx).map(|_| ()) - } -} - -#[derive(Debug, Clone)] -struct UdpSocket { - io: Arc>, - inner: Arc, -} - -impl UdpSocket { - fn new(sock: std::net::UdpSocket) -> io::Result { - Ok(Self { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(Async::new_nonblocking(sock)?), - }) - } -} - -impl UdpSenderHelperSocket for UdpSocket { - fn max_transmit_segments(&self) -> usize { - self.inner.max_gso_segments() - } - - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { - self.inner.try_send((&self.io).into(), transmit)?; - - Ok(()) - } -} - -impl AsyncUdpSocket for UdpSocket { - fn create_sender(&self) -> Pin> { - Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { - let socket = socket.clone(); - async move { socket.io.writable().await } - })) - } - - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], - ) -> Poll> { - loop { - ready!(self.io.poll_readable(cx))?; - match self.inner.recv((&self.io).into(), bufs, meta) { - Ok(res) => return Poll::Ready(Ok(res)), - Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} - Err(e) => return Poll::Ready(Err(e)), - } - } - } - - fn local_addr(&self) -> io::Result { - self.io.as_ref().as_ref().local_addr() - } - - fn may_fragment(&self) -> bool { - self.inner.may_fragment() - } - - fn max_receive_segments(&self) -> usize { - self.inner.gro_segments() - } -} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs index ac5dfd6f20..750bff8ac6 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -1,57 +1,26 @@ use std::{ - future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll, ready}, - time::Instant, }; -use tokio::{ - io::Interest, - time::{Sleep, sleep_until}, -}; - -use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for Tokio -#[derive(Debug)] -pub struct TokioRuntime; - -impl Runtime for TokioRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(sleep_until(t.into())) - } - - fn spawn(&self, future: Pin + Send>>) { - tokio::spawn(future); - } +use tokio::io::Interest; - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), - })) - } +use super::{AsyncUdpSocket, UdpSenderHelper, UdpSenderHelperSocket}; - fn now(&self) -> Instant { - tokio::time::Instant::now().into_std() - } -} - -impl AsyncTimer for Sleep { - fn reset(self: Pin<&mut Self>, t: Instant) { - Self::reset(self, t.into()) - } - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx) - } +/// Register `sock` with the current tokio runtime. +pub(crate) fn wrap_udp_socket(sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(moq_sock::udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) } #[derive(Debug, Clone)] struct UdpSocket { io: Arc, - inner: Arc, + inner: Arc, } impl UdpSenderHelperSocket for UdpSocket { @@ -59,7 +28,7 @@ impl UdpSenderHelperSocket for UdpSocket { self.inner.max_gso_segments() } - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + fn try_send(&self, transmit: &moq_sock::udp::Transmit<'_>) -> io::Result<()> { self.io.try_io(Interest::WRITABLE, || { self.inner.try_send((&self.io).into(), transmit) })?; @@ -80,7 +49,7 @@ impl AsyncUdpSocket for UdpSocket { &mut self, cx: &mut Context<'_>, bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { loop { ready!(self.io.poll_recv_ready(cx))?; diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 570db9ac81..e51af7cf25 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -6,10 +6,10 @@ use std::{ }; use bytes::Bytes; -use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use moq_quic::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ VarInt, connection::{ConnectionRef, State}, }; @@ -140,9 +140,9 @@ impl SendStream { write_fn: F, ) -> Poll> where - F: FnOnce(&mut proto::SendStream<'_>) -> Result, + F: FnOnce(&mut moq_quic::SendStream<'_>) -> Result, { - use proto::WriteError::*; + use moq_quic::WriteError::*; let mut conn = self.conn.state.lock("SendStream::poll_write"); if self.is_0rtt { conn.check_0rtt() @@ -315,25 +315,6 @@ fn send_stream_stopped( } } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncWrite for SendStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.poll_write(cx, buf).map_err(Into::into) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().finish().map_err(Into::into)) - } -} - impl tokio::io::AsyncWrite for SendStream { fn poll_write( self: Pin<&mut Self>, @@ -394,7 +375,7 @@ pub enum WriteError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -437,7 +418,7 @@ pub enum StoppedError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index e0c6f3ef9e..0b284f1770 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1,8 +1,8 @@ -#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +#![cfg(any(feature = "aws-lc-rs", feature = "ring"))] -#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +#[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] use rustls::crypto::aws_lc_rs::default_provider; -#[cfg(feature = "rustls-ring")] +#[cfg(feature = "ring")] use rustls::crypto::ring::default_provider; use std::{ @@ -19,15 +19,16 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; -use crate::{Duration, Instant}; +use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}; +use crate::quic::driver::{Duration, Instant}; use bytes::Bytes; -use proto::{ +use moq_quic::{ ConnectionId, RandomConnectionIdGenerator, crypto::rustls::{QuicClientConfig, QuicServerConfig}, crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, transport_parameters::TransportParameters, }; +use moq_sock::udp::RecvMeta; use rand::{Rng, SeedableRng, rngs::StdRng}; use rustls::{ RootCertStore, @@ -41,7 +42,6 @@ use tokio::{ use tracing::instrument::Instrument as _; use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; -use udp::RecvMeta; use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; @@ -64,6 +64,7 @@ fn handshake_timeout() { let mut transport_config = TransportConfig::default(); transport_config .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .handshake_idle_timeout(IDLE_TIMEOUT) .initial_rtt(Duration::from_millis(10)); client_config.transport_config(Arc::new(transport_config)); @@ -78,7 +79,7 @@ fn handshake_timeout() { .unwrap() .await { - Err(crate::ConnectionError::TimedOut) => {} + Err(crate::quic::driver::ConnectionError::TimedOut) => {} Err(e) => panic!("unexpected error: {e:?}"), Ok(_) => panic!("unexpected success"), } @@ -119,7 +120,7 @@ async fn close_endpoint() { .unwrap(); endpoint.close(0u32.into(), &[]); match conn.await { - Err(crate::ConnectionError::LocallyClosed) => (), + Err(crate::quic::driver::ConnectionError::LocallyClosed) => (), Err(e) => panic!("unexpected error: {e}"), Ok(_) => { panic!("unexpected success"); @@ -134,7 +135,7 @@ fn local_addr() { let runtime = rt_basic(); let ep = { let _guard = runtime.enter(); - Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + Endpoint::new(Default::default(), None, socket).unwrap() }; assert_eq!( addr, @@ -254,7 +255,7 @@ async fn ip_blocking() { .await .expect_err("server should have blocked this"); assert!( - matches!(e, crate::ConnectionError::ConnectionClosed(_)), + matches!(e, crate::quic::driver::ConnectionError::ConnectionClosed(_)), "wrong error" ); }, @@ -299,8 +300,11 @@ impl EndpointFactory { fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); let transport_config = Arc::new(transport_config); - let mut server_config = - crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + let mut server_config = crate::quic::driver::ServerConfig::with_single_cert( + vec![self.cert.cert.der().clone()], + key, + ) + .unwrap(); server_config.transport_config(transport_config.clone()); let mut roots = RootCertStore::empty(); @@ -309,7 +313,6 @@ impl EndpointFactory { self.endpoint_config.clone(), Some(server_config), UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - Arc::new(TokioRuntime), ) .unwrap(); let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); @@ -448,43 +451,8 @@ impl AsyncUdpSocket for FailingSocket { } } -/// Poll a connection driver before its registration returns, once the endpoint has failed. -#[derive(Debug)] -struct DriverLossRuntime { - failure: Arc, - stopped_on_first_poll: AtomicBool, -} - -impl crate::runtime::Runtime for DriverLossRuntime { - fn new_timer(&self, deadline: Instant) -> Pin> { - TokioRuntime.new_timer(deadline) - } - - fn spawn(&self, mut future: Pin + Send>>) { - if self.failure.armed.load(Ordering::SeqCst) { - let mut cx = Context::from_waker(Waker::noop()); - if future.as_mut().poll(&mut cx).is_ready() { - self.stopped_on_first_poll.store(true, Ordering::SeqCst); - return; - } - } - TokioRuntime.spawn(future); - } - - fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { - TokioRuntime.wrap_udp_socket(socket) - } - - fn now(&self) -> Instant { - TokioRuntime.now() - } -} - fn localhost_socket() -> Box { - TokioRuntime - .wrap_udp_socket( - UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - ) + wrap_udp_socket(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) .unwrap() } @@ -492,7 +460,6 @@ fn blocking_server_pair( server_alpn: &[u8], client_alpn: &[u8], server_socket: Box, - server_runtime: Arc, ) -> (Endpoint, Endpoint, Arc) { let factory = EndpointFactory::new(); let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); @@ -503,7 +470,7 @@ fn blocking_server_pair( .with_single_cert(vec![factory.cert.cert.der().clone()], key) .unwrap(); server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; - let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + let mut server_config = crate::quic::driver::ServerConfig::with_crypto(Arc::new( QuicServerConfig::try_from(server_crypto).unwrap(), )); let blocker = Arc::new(HandshakeBlocker::default()); @@ -516,14 +483,20 @@ fn blocking_server_pair( factory.endpoint_config.clone(), Some(server_config), server_socket, - server_runtime, ) .unwrap(); let mut roots = RootCertStore::empty(); roots.add(factory.cert.cert.der().clone()).unwrap(); let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); - let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + // One classical key share keeps the ClientHello in the first Initial, so a handshake failure + // surfaces from `accept` itself. aws-lc-rs's default post-quantum share spills it into a + // second packet, which only reaches the connection after it is registered. + let mut provider = default_provider(); + provider + .kx_groups + .retain(|group| group.name() == rustls::NamedGroup::X25519); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(provider.into()) .with_protocol_versions(&[&rustls::version::TLS13]) .unwrap() .with_root_certificates(roots) @@ -544,7 +517,7 @@ async fn wait_for_blocked_handshake(blocker: Arc) { async fn accept_with_blocked_start_session( server: Endpoint, -) -> Result { +) -> Result { let incoming = server.accept().await.unwrap(); // The test crypto provider intentionally blocks in `start_session`, so run // accept on the blocking pool rather than parking a Tokio worker thread. @@ -559,8 +532,12 @@ struct BlockedHandshake { server: Endpoint, blocker: Arc, release_guard: HandshakeReleaseGuard, - accept: tokio::task::JoinHandle>, - connect: tokio::task::JoinHandle>, + accept: tokio::task::JoinHandle< + Result, + >, + connect: tokio::task::JoinHandle< + Result, + >, } fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { @@ -572,22 +549,7 @@ fn blocked_handshake_with_socket( client_alpn: &[u8], server_socket: Box, ) -> BlockedHandshake { - blocked_handshake_with_runtime( - server_alpn, - client_alpn, - server_socket, - Arc::new(TokioRuntime), - ) -} - -fn blocked_handshake_with_runtime( - server_alpn: &[u8], - client_alpn: &[u8], - server_socket: Box, - server_runtime: Arc, -) -> BlockedHandshake { - let (client, server, blocker) = - blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let (client, server, blocker) = blocking_server_pair(server_alpn, client_alpn, server_socket); let server_addr = server.local_addr().unwrap(); let accept = tokio::spawn({ @@ -610,8 +572,10 @@ fn blocked_handshake_with_runtime( } async fn await_connection( - task: tokio::task::JoinHandle>, -) -> Result { + task: tokio::task::JoinHandle< + Result, + >, +) -> Result { timeout(Duration::from_secs(5), task) .await .unwrap() @@ -713,7 +677,9 @@ fn wait_idle_returns_when_the_driver_is_dropped() { let _ = waiter.join(); } -async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { +async fn wait_closed( + conn: &crate::quic::driver::Connection, +) -> crate::quic::driver::ConnectionError { timeout(Duration::from_secs(5), conn.closed()) .await .unwrap() @@ -914,7 +880,8 @@ fn run_echo(args: EchoArgs) { let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); let cert = CertificateDer::from(cert.cert); let mut server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); server_config.transport = transport_config.clone(); let server_sock = UdpSocket::bind(args.server_addr).unwrap(); @@ -922,13 +889,7 @@ fn run_echo(args: EchoArgs) { let server = { let _guard = runtime.enter(); let _guard = error_span!("server").entered(); - Endpoint::new( - Default::default(), - Some(server_config), - server_sock, - Arc::new(TokioRuntime), - ) - .unwrap() + Endpoint::new(Default::default(), Some(server_config), server_sock).unwrap() }; let mut roots = RootCertStore::empty(); @@ -1144,7 +1105,8 @@ async fn rebind_recv() { client.set_default_client_config(client_config); let server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); let server = { let _guard = tracing::error_span!("server").entered(); Endpoint::server( @@ -1254,8 +1216,8 @@ async fn split_accept_failure_wakes_idle_waiter() { let error = await_connection(accept).await.unwrap_err(); assert!(matches!( error, - crate::ConnectionError::TransportError(error) - if error.code == proto::TransportErrorCode::crypto(0x78) + crate::quic::driver::ConnectionError::TransportError(error) + if error.code == moq_quic::TransportErrorCode::crypto(0x78) )); // Repolling wait_idle alone would hide a missing notification now that the endpoint is // idle. Verify that the waiter registered before the failure was actually woken. @@ -1291,18 +1253,30 @@ async fn split_accept_close_during_pending_accept() { match await_connection(accept).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )); } - Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + Err(err) => assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )), } match await_connection(connect).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } Err(err) => { - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } } @@ -1364,51 +1338,6 @@ async fn split_accept_driver_lost_during_handshake() { wait_idle(&client).await; } -#[tokio::test] -async fn split_accept_driver_lost_before_connection_spawn() { - let _guard = subscribe(); - let failure = Arc::new(SocketFailure::default()); - let runtime = Arc::new(DriverLossRuntime { - failure: failure.clone(), - stopped_on_first_poll: AtomicBool::new(false), - }); - let socket = Box::new(FailingSocket { - inner: localhost_socket(), - failure: failure.clone(), - }); - let BlockedHandshake { - client, - server, - blocker, - release_guard: _release_guard, - accept, - connect, - } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); - - wait_for_blocked_handshake(blocker.clone()).await; - failure.trigger(); - assert!( - timeout(Duration::from_secs(5), server.accept()) - .await - .unwrap() - .is_none() - ); - - blocker.release(); - // The runtime polls the connection driver before registration returns. It must observe the - // closed event channel immediately, before transmitting any handshake packets. - assert!(await_connection(accept).await.is_err()); - assert!( - runtime.stopped_on_first_poll.load(Ordering::SeqCst), - "connection driver ran before its event channel was closed" - ); - wait_idle(&server).await; - - client.close(0u32.into(), b"done"); - assert!(await_connection(connect).await.is_err()); - wait_idle(&client).await; -} - #[tokio::test] async fn remote_address_monitoring() { let _guard = subscribe(); @@ -1782,8 +1711,8 @@ async fn dropped_endpoint_cleans_up() { let _guard = subscribe(); let mut endpoint_factory = EndpointFactory::new(); - let cid_generator = Arc::new(|| -> Box { - Box::::default() + let cid_generator = Arc::new(|| -> Box { + Box::::default() }); endpoint_factory .endpoint_config diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs index c3c3d3551d..eda6e92d6c 100644 --- a/rs/moq-tokio/src/quic/driver/work_limiter.rs +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -1,4 +1,4 @@ -use crate::{Duration, Instant}; +use crate::quic::driver::{Duration, Instant}; /// Limits the amount of time spent on a certain type of work in a cycle /// diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs index c3dcee477b..e38cbd62e9 100644 --- a/rs/moq-tokio/src/quic/web/client.rs +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -1,28 +1,28 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; -use crate::proto::ConnectRequest; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use noq::crypto::rustls::QuicClientConfig; +use crate::quic::driver::crypto::rustls::QuicClientConfig; +use crate::quic::web::proto::ConnectRequest; use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; use tokio::net::lookup_host; use url::Host; -use crate::crypto; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::ALPN; -use crate::{ClientError, Session}; +use crate::quic::web::ALPN; +use crate::quic::web::crypto; +use crate::quic::web::{ClientError, Session}; /// Congestion control algorithm to use for the connection. /// /// Different algorithms make different tradeoffs between throughput and latency. pub enum CongestionControl { - /// Use the default congestion control algorithm (typically CUBIC). - Default, - /// Optimize for throughput (typically CUBIC). - Throughput, - /// Optimize for low latency (typically BBR). - LowLatency, + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -31,136 +31,124 @@ pub enum CongestionControl { /// This is optional; advanced users may use [Client::new] directly. #[derive(Clone)] pub struct ClientBuilder { - provider: crypto::Provider, - congestion_controller: - Option>, + provider: crypto::Provider, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ClientBuilder { - /// Create a Client builder, which can be used to establish multiple [Session]s. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - congestion_controller: None, - } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Accept any certificate from the server if it uses a known root CA. - pub fn with_system_roots(self) -> Result { - let mut roots = rustls::RootCertStore::empty(); - - let native = rustls_native_certs::load_native_certs(); - - // Log any errors that occurred while loading the native root certificates. - for err in native.errors { - tracing::warn!(?err, "failed to load root cert"); - } - - // Add the platform's native root certificates. - for cert in native.certs { - if let Err(err) = roots.add(cert) { - tracing::warn!(?err, "failed to add root cert"); - } - } - - let crypto = self - .builder() - .with_root_certificates(roots) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Supply certificates for accepted servers instead of using root CAs. - pub fn with_server_certificates( - self, - certs: Vec, - ) -> Result { - let hashes = certs.iter().map({ - let provider = self.provider.clone(); - move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() - }); - - self.with_server_certificate_hashes(hashes.collect()) - } - - /// Supply sha256 hashes for accepted certificates instead of using root CAs. - pub fn with_server_certificate_hashes( - self, - hashes: Vec>, - ) -> Result { - // Use a custom fingerprint verifier. - let fingerprints = Arc::new(ServerFingerprints { - provider: self.provider.clone(), - fingerprints: hashes, - }); - - // Configure the crypto client. - let crypto = self - .builder() - .dangerous() - .with_custom_certificate_verifier(fingerprints.clone()) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Access dangerous configuration options. - /// - /// This method returns a builder that provides access to potentially insecure - /// TLS configurations. These options are opt-in and require explicit acknowledgment - /// through the builder pattern, making the security implications clear at the call site. - pub fn dangerous(self) -> DangerousClientBuilder { - DangerousClientBuilder { inner: self } - } - - fn builder(&self) -> rustls::ConfigBuilder { - rustls::ClientConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13]) - .unwrap() - } - - fn build(self, mut crypto: rustls::ClientConfig) -> Result { - crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; - - let client_config = QuicClientConfig::try_from(crypto).unwrap(); - let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); - - let mut transport = noq::TransportConfig::default(); - if let Some(cc) = &self.congestion_controller { - transport.congestion_controller_factory(cc.clone()); - } - - client_config.transport_config(transport.into()); - - let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); - Ok(Client { - endpoint: client, - config: client_config, - }) - } + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self.builder().with_root_certificates(roots).with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates(self, certs: Vec) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes(self, hashes: Vec>) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = crate::quic::driver::ClientConfig::new(Arc::new(client_config)); + + let mut transport = crate::quic::driver::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = crate::quic::driver::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ClientBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -170,209 +158,182 @@ impl Default for ClientBuilder { /// These options should only be used when you understand the security implications, /// such as in local development or over a secure VPN connection. pub struct DangerousClientBuilder { - inner: ClientBuilder, + inner: ClientBuilder, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl DangerousClientBuilder { - /// Disable certificate verification entirely. - /// - /// This makes the connection vulnerable to man-in-the-middle attacks. - /// Only use this in secure environments, such as in local development or over a VPN connection. - /// - /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the - /// security sense, hence the explicit `dangerous()` builder requirement. - pub fn with_no_certificate_verification(self) -> Result { - let noop = NoCertificateVerification(self.inner.provider.clone()); - - let crypto = self - .inner - .builder() - .dangerous() - .with_custom_certificate_verifier(Arc::new(noop)) - .with_no_client_auth(); - - self.inner.build(crypto) - } + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } } /// A client for connecting to a WebTransport server. #[derive(Clone, Debug)] pub struct Client { - endpoint: noq::Endpoint, - config: noq::ClientConfig, + endpoint: crate::quic::driver::Endpoint, + config: crate::quic::driver::ClientConfig, } impl Client { - /// Manually create a client via a Noq endpoint and config. - /// - /// The ALPN MUST be set to [ALPN]. - pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { - Self { endpoint, config } - } - - /// Connect to the server. - pub async fn connect( - &self, - request: impl Into, - ) -> Result { - let request = request.into(); - - let port = request.url.port().unwrap_or(443); - - // TODO error on username:password in host - let (host, remote) = match request - .url - .host() - .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? - { - Host::Domain(domain) => { - let domain = domain.to_string(); - // Look up the DNS entry. - let mut remotes = match lookup_host((domain.clone(), port)).await { - Ok(remotes) => remotes, - Err(_) => return Err(ClientError::InvalidDnsName(domain)), - }; - - // Return the first entry. - let remote = match remotes.next() { - Some(remote) => remote, - None => return Err(ClientError::InvalidDnsName(domain)), - }; - - (domain, remote) - } - Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), - Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), - }; - - // Connect to the server using the addr we just resolved. - let conn = self - .endpoint - .connect_with(self.config.clone(), remote, &host)?; - let conn = conn.await?; - - // Connect with the connection we established. - Session::connect(conn, request).await - } + /// Manually create a client via a QUIC endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: crate::quic::driver::Endpoint, config: crate::quic::driver::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect(&self, request: impl Into) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self.endpoint.connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for Client { - fn default() -> Self { - ClientBuilder::new().with_system_roots().unwrap() - } + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } } #[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] #[derive(Debug)] struct ServerFingerprints { - provider: crypto::Provider, - fingerprints: Vec>, + provider: crypto::Provider, + fingerprints: Vec>, } impl ServerCertVerifier for ServerFingerprints { - fn verify_server_cert( - &self, - end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - let cert_hash = crypto::sha256(&self.provider, end_entity); - if self - .fingerprints - .iter() - .any(|fingerprint| fingerprint == cert_hash.as_ref()) - { - return Ok(rustls::client::danger::ServerCertVerified::assertion()); - } - - Err(rustls::Error::InvalidCertificate( - rustls::CertificateError::UnknownIssuer, - )) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.provider - .signature_verification_algorithms - .supported_schemes() - } + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider.signature_verification_algorithms.supported_schemes() + } } #[derive(Debug)] pub struct NoCertificateVerification(Arc); impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer<'_>, - _intermediates: &[CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.0.signature_verification_algorithms.supported_schemes() - } + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } } diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 4307355d3f..471f5e416c 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -6,153 +6,145 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum ConnectError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::ConnectError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("http error status: {0}")] - ErrorStatus(http::StatusCode), + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), - #[error("server returned protocol not in request: {0}")] - ProtocolMismatch(String), + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), } /// An HTTP/3 CONNECT request/response for establishing a WebTransport session. pub struct Connecting { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connecting { - pub async fn accept(conn: &noq::Connection) -> Result { - // Accept the stream that will be used to send the HTTP CONNECT request. - // If they try to send any other type of HTTP request, we will error out. - let (send, mut recv) = conn.accept_bi().await?; - - let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; - tracing::debug!(?request, "received CONNECT request"); - - // The request was successfully decoded, so we can send a response. - Ok(Self { - request, - send, - recv, - }) - } - - // Called by the server to send a response to the client and establish the session. - pub async fn respond( - mut self, - response: impl Into, - ) -> Result { - let response = response.into(); - - // Validate that our protocol was in the client's request. - if let Some(protocol) = &response.protocol { - if !self.request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - tracing::debug!(?response, "sending CONNECT response"); - response.write(&mut self.send).await?; - - Ok(Connected { - request: self.request, - response, - send: self.send, - recv: self.recv, - }) - } - - pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { - let mut connect = self.respond(status).await?; - connect.send.finish().ok(); - Ok(()) - } + pub async fn accept(conn: &crate::quic::driver::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { request, send, recv }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond(mut self, response: impl Into) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol + && !self.request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } } impl Deref for Connecting { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.request - } + fn deref(&self) -> &Self::Target { + &self.request + } } pub struct Connected { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // The response sent by the server. - pub response: ConnectResponse, + // The response sent by the server. + pub response: ConnectResponse, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connected { - /// Open a new WebTransport session on the given connection for the given URL. - /// - /// You may add any number of subprotocols allowing the server to select from. - /// If the list is empty the field will be omitted in the request header. - pub async fn open( - conn: &noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Create a new stream that will be used to send the CONNECT frame. - let (mut send, mut recv) = conn.open_bi().await?; - - tracing::debug!(?request, "sending CONNECT request"); - request.write(&mut send).await?; - - let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; - tracing::debug!(?response, "received CONNECT response"); - - // Throw an error if we didn't get a 200 OK. - if response.status != http::StatusCode::OK { - return Err(ConnectError::ErrorStatus(response.status)); - } - - // Validate that the server's protocol was in our request. - if let Some(protocol) = &response.protocol { - if !request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - Ok(Self { - request, - response, - send, - recv, - }) - } - - // The session ID is the stream ID of the CONNECT request. - pub fn session_id(&self) -> VarInt { - // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. - // We don't use the noq::VarInt because that would mean a noq dependency in - // web-transport-proto - let stream_id = noq::VarInt::from(self.send.id()); - VarInt::try_from(stream_id.into_inner()).unwrap() - } + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol + && !request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // Convert from the QUIC VarInt to the WebTransport VarInt, since web-transport-proto + // doesn't depend on moq-quic. + let stream_id = crate::quic::driver::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } } diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs index 1b8bb8ecf1..a1581975c9 100644 --- a/rs/moq-tokio/src/quic/web/crypto.rs +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -5,8 +5,8 @@ use std::sync::Arc; -use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::crypto::CryptoProvider; +use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::pki_types::CertificateDer; /// A shared reference to a crypto provider. @@ -22,25 +22,23 @@ pub type Provider = Arc; /// Panics if no provider is available. Either call `CryptoProvider::set_default()` /// or enable exactly one of the `ring` or `aws-lc-rs` features. pub fn default_provider() -> Provider { - // See - if let Some(provider) = CryptoProvider::get_default().cloned() { - return provider; - } + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } - #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] - { - return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); - } - #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] - { - return Arc::new(rustls::crypto::ring::default_provider()); - } - #[allow(unreachable_code)] - { - panic!( - "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." - ); - } + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!("CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature."); + } } /// Computes the SHA-256 hash of a certificate using the provided crypto provider. @@ -49,19 +47,19 @@ pub fn default_provider() -> Provider { /// /// Panics if the provider doesn't expose a SHA-256 hash algorithm. pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { - let hash_provider = provider.cipher_suites.iter().find_map(|suite| { - let hash_provider = suite.tls13()?.common.hash_provider; - if hash_provider.algorithm() == HashAlgorithm::SHA256 { - Some(hash_provider) - } else { - None - } - }); - if let Some(hash_provider) = hash_provider { - return hash_provider.hash(cert); - } + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } - panic!( - "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." - ); + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); } diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 654260fc4c..5697bf684e 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -2,70 +2,68 @@ use std::sync::{Arc, OnceLock}; use thiserror::Error; -use crate::{ConnectError, SettingsError}; +use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] pub enum ClientError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error: {0}")] - Connection(#[from] noq::ConnectionError), + #[error("connection error: {0}")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write: {0}")] - WriteError(#[from] noq::WriteError), + #[error("failed to write: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read: {0}")] - ReadError(#[from] noq::ReadError), + #[error("failed to read: {0}")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings: {0}")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect: {0}")] - HttpError(#[from] ConnectError), + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), - #[error("quic error: {0}")] - NoqError(#[from] noq::ConnectError), + #[error("quic error: {0}")] + ConnectError(#[from] crate::quic::driver::ConnectError), - #[error("invalid DNS name: {0}")] - InvalidDnsName(String), + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } -/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] pub enum SessionError { - #[error("connection error: {0}")] - ConnectionError(noq::ConnectionError), + #[error("connection error: {0}")] + ConnectionError(crate::quic::driver::ConnectionError), - #[error("webtransport error: {0}")] - WebTransportError(#[from] WebTransportError), + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), - #[error("send datagram error: {0}")] - SendDatagramError(#[from] noq::SendDatagramError), + #[error("send datagram error: {0}")] + SendDatagramError(#[from] crate::quic::driver::SendDatagramError), } -impl From for SessionError { - fn from(e: noq::ConnectionError) -> Self { - match &e { - noq::ConnectionError::ApplicationClosed(close) => { - match web_transport_proto::error_from_http3(close.error_code.into_inner()) { - Some(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - None => SessionError::ConnectionError(e), - } - } - _ => SessionError::ConnectionError(e), - } - } +impl From for SessionError { + fn from(e: crate::quic::driver::ConnectionError) -> Self { + match &e { + crate::quic::driver::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } } /// Why a session closed, shared by the session and its streams. The first close wins. @@ -74,192 +72,191 @@ impl From for SessionError { /// codes are encoded. #[derive(Debug)] pub(crate) struct CloseReason { - // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. - raw: bool, - reason: OnceLock, + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, } impl CloseReason { - pub(crate) fn new(raw: bool) -> Self { - Self { - raw, - reason: OnceLock::new(), - } - } - - /// The QUIC code to reset or stop a stream with. - pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { - if self.raw { - return code.into(); - } - noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() - } - - /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. - pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { - let code = code.into_inner(); - // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that - // range, so the mapped form cannot be mistaken for a raw code. - match web_transport_proto::error_from_http3(code) { - Some(code) => Some(code), - None if self.raw => u32::try_from(code).ok(), - None => None, - } - } - - /// Record the close reason, returning false if one was already recorded. - pub(crate) fn set(&self, err: SessionError) -> bool { - self.reason.set(err).is_ok() - } - - /// Replace an error caused by the connection closing with the recorded reason, or - /// decode the peer's raw QUIC close code when none was recorded. - pub(crate) fn map(&self, err: SessionError) -> SessionError { - let conn = match &err { - SessionError::ConnectionError(conn) - | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { - Some(conn) - } - SessionError::WebTransportError(WebTransportError::Closed(..)) => None, - _ => return err, - }; - - if let Some(reason) = self.reason.get() { - return reason.clone(); - } - - match conn { - Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { - match u32::try_from(close.error_code.into_inner()) { - Ok(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - Err(_) => err, - } - } - _ => err, - } - } + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> crate::quic::driver::VarInt { + if self.raw { + return code.into(); + } + crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: crate::quic::driver::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(crate::quic::driver::SendDatagramError::ConnectionLost(conn)) => Some(conn), + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(crate::quic::driver::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + Err(_) => err, + } + } + _ => err, + } + } } /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] pub enum WebTransportError { - #[error("closed: code={0} reason={1}")] - Closed(u32, String), + #[error("closed: code={0} reason={1}")] + Closed(u32, String), - #[error("unknown session")] - UnknownSession, + #[error("unknown session")] + UnknownSession, - #[error("read error: {0}")] - ReadError(#[from] noq::ReadExactError), + #[error("read error: {0}")] + ReadError(#[from] crate::quic::driver::ReadExactError), - #[error("write error: {0}")] - WriteError(#[from] noq::WriteError), + #[error("write error: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), } -/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +/// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] pub enum WriteError { - #[error("STOP_SENDING: {0}")] - Stopped(u32), + #[error("STOP_SENDING: {0}")] + Stopped(u32), - #[error("invalid STOP_SENDING: {0}")] - InvalidStopped(noq::VarInt), + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(crate::quic::driver::VarInt), - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("stream closed")] - ClosedStream, + #[error("stream closed")] + ClosedStream, } -impl From for WriteError { - fn from(e: noq::WriteError) -> Self { - match e { - noq::WriteError::Stopped(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - } - } - noq::WriteError::ClosedStream => WriteError::ClosedStream, - noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), - noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for WriteError { + fn from(e: crate::quic::driver::WriteError) -> Self { + match e { + crate::quic::driver::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + crate::quic::driver::WriteError::ClosedStream => WriteError::ClosedStream, + crate::quic::driver::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + crate::quic::driver::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } } -/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +/// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] pub enum ReadError { - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("RESET_STREAM: {0}")] - Reset(u32), + #[error("RESET_STREAM: {0}")] + Reset(u32), - #[error("invalid RESET_STREAM: {0}")] - InvalidReset(noq::VarInt), + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(crate::quic::driver::VarInt), - #[error("stream already closed")] - ClosedStream, + #[error("stream already closed")] + ClosedStream, } -impl From for ReadError { - fn from(value: noq::ReadError) -> Self { - match value { - noq::ReadError::Reset(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - } - } - noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), - noq::ReadError::ClosedStream => ReadError::ClosedStream, - noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for ReadError { + fn from(value: crate::quic::driver::ReadError) -> Self { + match value { + crate::quic::driver::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + crate::quic::driver::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + crate::quic::driver::ReadError::ClosedStream => ReadError::ClosedStream, + crate::quic::driver::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + crate::quic::driver::ReadError::IllegalOrderedRead => { + unreachable!("WebTransport streams are only read in order") + } + } + } } -/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] pub enum ReadExactError { - #[error("finished early")] - FinishedEarly(usize), + #[error("finished early")] + FinishedEarly(usize), - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadExactError { - fn from(e: noq::ReadExactError) -> Self { - match e { - noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), - noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), - } - } +impl From for ReadExactError { + fn from(e: crate::quic::driver::ReadExactError) -> Self { + match e { + crate::quic::driver::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + crate::quic::driver::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } } -/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] pub enum ReadToEndError { - #[error("too long")] - TooLong, + #[error("too long")] + TooLong, - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadToEndError { - fn from(e: noq::ReadToEndError) -> Self { - match e { - noq::ReadToEndError::TooLong => ReadToEndError::TooLong, - noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), - } - } +impl From for ReadToEndError { + fn from(e: crate::quic::driver::ReadToEndError) -> Self { + match e { + crate::quic::driver::ReadToEndError::TooLong => ReadToEndError::TooLong, + crate::quic::driver::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } } /// An error indicating the stream was already closed. @@ -267,39 +264,39 @@ impl From for ReadToEndError { #[error("stream closed")] pub struct ClosedStream; -impl From for ClosedStream { - fn from(_: noq::ClosedStream) -> Self { - ClosedStream - } +impl From for ClosedStream { + fn from(_: crate::quic::driver::ClosedStream) -> Self { + ClosedStream + } } /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] pub enum ServerError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error")] - Connection(#[from] noq::ConnectionError), + #[error("connection error")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write")] - WriteError(#[from] noq::WriteError), + #[error("failed to write")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read")] - ReadError(#[from] noq::ReadError), + #[error("failed to read")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect")] - ConnectError(#[from] ConnectError), + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), - #[error("io error: {0}")] - IoError(Arc), + #[error("io error: {0}")] + IoError(Arc), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } // #[derive(Clone, Error, Debug)] @@ -317,89 +314,89 @@ pub enum ServerError { // SessionError(#[from] SessionError), // } -// impl From for SendDatagramError { -// fn from(value: noq::SendDatagramError) -> Self { +// impl From for SendDatagramError { +// fn from(value: crate::quic::driver::SendDatagramError) -> Self { // match value { -// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, -// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, -// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, -// noq::SendDatagramError::ConnectionLost(e) => +// crate::quic::driver::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// crate::quic::driver::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// crate::quic::driver::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// crate::quic::driver::SendDatagramError::ConnectionLost(e) => // SendDatagramError::SessionError(e.into()), } // } // } impl web_transport_trait::Error for SessionError { - fn session_error(&self) -> Option<(u32, String)> { - if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { - return Some((*code, reason.to_string())); - } + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } - None - } + None + } } impl web_transport_trait::Error for WriteError { - fn session_error(&self) -> Option<(u32, String)> { - if let WriteError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - WriteError::Stopped(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } } impl web_transport_trait::Error for ReadError { - fn session_error(&self) -> Option<(u32, String)> { - if let ReadError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - ReadError::Reset(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } } #[cfg(test)] mod tests { - use super::*; - - /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its - /// range is not a WebTransport code. - #[test] - fn http3_stream_codes_stay_mapped() { - let close = CloseReason::new(false); - let mapped = close.encode_stream_code(5); - assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); - assert_eq!(close.decode_stream_code(mapped), Some(5)); - assert_eq!(close.decode_stream_code(5u32.into()), None); - } - - /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. - #[test] - fn raw_stream_codes_pass_through() { - let close = CloseReason::new(true); - assert_eq!(close.encode_stream_code(5).into_inner(), 5); - assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); - assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); - - let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); - assert_eq!(close.decode_stream_code(legacy), Some(5)); - - // Past u32 and outside the HTTP/3 range: neither form. - let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); - assert_eq!(close.decode_stream_code(invalid), None); - } + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = crate::quic::driver::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } } diff --git a/rs/moq-tokio/src/quic/web/fan.rs b/rs/moq-tokio/src/quic/web/fan.rs new file mode 100644 index 0000000000..0962877fcd --- /dev/null +++ b/rs/moq-tokio/src/quic/web/fan.rs @@ -0,0 +1,140 @@ +use std::sync::{Arc, Mutex}; +use std::task::{Wake, Waker}; + +use kio::{Waiter, WaiterList}; + +/// A shared [`WaiterList`] that a [`Waker`] can wake. +/// +/// The accept futures below take one `Waker` and keep it. Handing them a caller's waker +/// would make the latest caller own the wakeup for everyone, so they are polled with +/// [`Fan::waker`] instead, which outlives every caller and wakes the whole list. +#[derive(Clone, Default)] +pub(crate) struct Fan(Arc); + +#[derive(Default)] +struct Inner(Mutex); + +#[derive(Default)] +struct State { + list: WaiterList, + /// Outstanding [`Hold`]s; a wake while any is alive is owed instead of delivered. + held: usize, + owed: bool, +} + +impl Fan { + /// Park `waiter` until the next wake. + pub(crate) fn register(&self, waiter: &Waiter) { + waiter.register(&mut self.0.0.lock().unwrap().list); + } + + /// A waker that wakes every parked waiter. + pub(crate) fn waker(&self) -> Waker { + Waker::from(self.0.clone()) + } + + /// Wake every parked waiter, or owe the wake until the last [`Hold`] drops. + pub(crate) fn wake(&self) { + self.0.notify(); + } + + /// Defer wakes until the guard drops, so none is delivered under the caller's lock. + pub(crate) fn hold(&self) -> Hold { + self.0.0.lock().unwrap().held += 1; + Hold(self.0.clone()) + } +} + +impl Inner { + fn notify(&self) { + let mut list = { + let mut state = self.0.lock().unwrap(); + if state.held > 0 { + state.owed = true; + return; + } + state.list.take() + }; + // Outside the lock: a waker may resume its task inline. + list.wake(); + } +} + +impl Wake for Inner { + fn wake(self: Arc) { + self.notify(); + } + + fn wake_by_ref(self: &Arc) { + self.notify(); + } +} + +/// Defers a [`Fan`]'s wakes while alive, delivering any owed one on drop. +pub(crate) struct Hold(Arc); + +impl Drop for Hold { + fn drop(&mut self) { + let owed = { + let mut state = self.0.0.lock().unwrap(); + state.held -= 1; + state.held == 0 && std::mem::take(&mut state.owed) + }; + if owed { + self.0.notify(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hold_defers_the_wake_until_dropped() { + let fan = Fan::default(); + let (waker, count) = counting(); + let waiter = Waiter::new(waker); + fan.register(&waiter); + + let hold = fan.hold(); + fan.waker().wake_by_ref(); + assert_eq!(count.load(), 0, "held back"); + + drop(hold); + assert_eq!(count.load(), 1, "delivered on drop"); + } + + #[test] + fn waker_wakes_every_waiter() { + let fan = Fan::default(); + let (a, count_a) = counting(); + let (b, count_b) = counting(); + // The list holds waiters weakly, so they have to outlive the wake. + let (a, b) = (Waiter::new(a), Waiter::new(b)); + fan.register(&a); + fan.register(&b); + + fan.wake(); + assert_eq!((count_a.load(), count_b.load()), (1, 1)); + } + + struct Count(std::sync::atomic::AtomicUsize); + + impl Count { + fn load(&self) -> usize { + self.0.load(std::sync::atomic::Ordering::SeqCst) + } + } + + impl Wake for Count { + fn wake(self: Arc) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + fn counting() -> (Waker, Arc) { + let count = Arc::new(Count(Default::default())); + (Waker::from(count.clone()), count) + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs index e5f3706492..47bf299eae 100644 --- a/rs/moq-tokio/src/quic/web/mod.rs +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -3,7 +3,7 @@ //! //! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. //! This library hides that detail and tries to expose only the QUIC API, delegating as much as -//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! possible to the underlying implementation. See [`crate::quic::driver`] for more documentation. //! //! QUIC provides two primary APIs: //! @@ -43,12 +43,13 @@ pub use session::*; // Internal mod connect; +mod fan; mod settings; use connect::*; use settings::*; -// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +// Required to access proto::ConnectError wrapped in ClientError pub use connect::ConnectError; /// The HTTP/3 ALPN is required when negotiating a QUIC connection. @@ -57,15 +58,8 @@ pub const ALPN: &str = "h3"; /// Export our simple crypto provider. pub mod crypto; -/// Re-export the underlying QUIC implementation. -pub use noq; - /// Re-export the http crate because it's in the public API. pub use http; -/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of -/// [`SessionAccept`]'s poll methods holds its place in the accept queue. -pub use kio; - /// Re-export the WebTransport protocol implementation. pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 4c3d188172..1e46bac784 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -1,159 +1,131 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::Bytes; -use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; +use crate::quic::web::{CloseReason, ReadError, ReadExactError, ReadToEndError, SessionError}; -/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +/// A stream that can be used to receive bytes. See [`crate::quic::driver::RecvStream`]. #[derive(Debug)] pub struct RecvStream { - inner: noq::RecvStream, - close: Arc, + inner: crate::quic::driver::RecvStream, + close: Arc, } impl RecvStream { - pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { - Self { - inner: stream, - close, - } - } - - /// Decode the peer's reset code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::ReadError) -> ReadError { - match e { - noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - }, - e => match ReadError::from(e) { - ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Tell the other end to stop sending data with the given error code. See - /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error - /// space with HTTP/3. A raw QUIC session sends the code as is. - pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { - let code = self.close.encode_stream_code(code); - self.inner.stop(code) - } - - // Unfortunately, we have to wrap ReadError for a bunch of functions. - - /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - self.inner.read(buf).await.map_err(|e| self.map_error(e)) - } - - /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. - pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { - self.inner.read_exact(buf).await.map_err(|e| match e { - noq::ReadExactError::ReadError(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - self.inner - .read_chunk(max_length) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. - pub async fn read_many_chunks( - &mut self, - bufs: &mut [Bytes], - ) -> Result, ReadError> { - self.inner - .read_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. - pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { - self.inner - .read_to_end(size_limit) - .await - .map_err(|e| match e { - noq::ReadToEndError::Read(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Block until the stream has been reset and return the error code. See - /// [`noq::RecvStream::received_reset`]. - /// - /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. - pub async fn received_reset(&mut self) -> Result, SessionError> { - match self.inner.received_reset().await { - Ok(None) => Ok(None), - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), - Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.inner.id() - } - - /// Returns the number of bytes read from this stream. - /// - /// This is the offset of the next byte to be read, i.e. the length of the contiguous - /// prefix of the stream consumed by the application. - pub fn bytes_read(&self) -> Result { - self.inner.bytes_read().map_err(|_| ClosedStream) - } - - // We purposely don't expose the 0RTT because it's not valid with WebTransport + pub(crate) fn new(stream: crate::quic::driver::RecvStream, close: Arc) -> Self { + Self { inner: stream, close } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::ReadError) -> ReadError { + match e { + crate::quic::driver::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`crate::quic::driver::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), crate::quic::driver::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`crate::quic::driver::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`crate::quic::driver::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + crate::quic::driver::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`crate::quic::driver::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length, true) + .await + .map(|chunk| chunk.map(|chunk| chunk.bytes)) + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`crate::quic::driver::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner.read_to_end(size_limit).await.map_err(|e| match e { + crate::quic::driver::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`crate::quic::driver::RecvStream::received_reset`]. + /// + /// Unlike the driver, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(crate::quic::driver::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.inner.id() + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport } impl tokio::io::AsyncRead for RecvStream { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut tokio::io::ReadBuf, - ) -> Poll> { - Pin::new(&mut self.inner).poll_read(cx, buf) - } + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } } impl web_transport_trait::RecvStream for RecvStream { - type Error = ReadError; + type Error = ReadError; - fn stop(&mut self, code: u32) { - Self::stop(self, code).ok(); - } + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } - async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { - self.read(dst).await - } + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } - async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { - self.read_chunk(max).await - } + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } - async fn closed(&mut self) -> Result<(), Self::Error> { - self.received_reset().await?; - Ok(()) - } + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } } diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 39af0d1fc7..7f87718527 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -1,197 +1,168 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::{Buf, Bytes}; -use crate::{CloseReason, ClosedStream, SessionError, WriteError}; +use crate::quic::web::{CloseReason, ClosedStream, SessionError, WriteError}; -/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// A stream that can be used to send bytes. See [`crate::quic::driver::SendStream`]. /// /// This wrapper is mainly needed for error codes, which is unfortunate. /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. #[derive(Debug)] pub struct SendStream { - stream: noq::SendStream, - close: Arc, + stream: crate::quic::driver::SendStream, + close: Arc, } impl SendStream { - pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { - Self { stream, close } - } - - /// Decode the peer's stop code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::WriteError) -> WriteError { - match e { - noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - }, - e => match WriteError::from(e) { - WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. - /// This is a u32 with WebTransport because we share the error space with HTTP/3. - /// A raw QUIC session sends the code as is. - pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { - let code = self.close.encode_stream_code(code); - self.stream.reset(code).map_err(Into::into) - } - - /// Wait until the stream has been stopped and return the error code. See - /// [`noq::SendStream::stopped`]. - /// - /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. - /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not - /// supported. - pub async fn stopped(&self) -> Result, SessionError> { - match self.stream.stopped().await { - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Ok(None) => Ok(None), - Err(noq::StoppedError::ConnectionLost(conn_err)) => { - Err(self.close.map(conn_err.into())) - } - Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - // Unfortunately, we have to wrap WriteError for a bunch of functions. - - /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. - pub async fn write(&mut self, buf: &[u8]) -> Result { - self.stream.write(buf).await.map_err(|e| self.map_error(e)) - } - - /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. - pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { - self.stream - .write_all(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write chunks of data to the stream, returning the number of bytes written. - /// - /// See [`noq::SendStream::write_many_chunks`]. - pub async fn write_many_chunks( - &mut self, - bufs: &mut &mut [Bytes], - ) -> Result { - self.stream - .write_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. - pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { - self.stream - .write_chunk(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. - pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { - self.stream - .write_all_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Mark the stream as finished, such that no more data can be written. See - /// [`noq::SendStream::finish`]. - /// - /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. - /// If you cancel futures by dropping them you'll get incomplete writes. - pub fn finish(&mut self) -> Result<(), ClosedStream> { - self.stream.finish().map_err(Into::into) - } - - pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { - self.stream.set_priority(order).map_err(Into::into) - } - - pub fn priority(&self) -> Result { - self.stream.priority().map_err(Into::into) - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.stream.id() - } + pub(crate) fn new(stream: crate::quic::driver::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::WriteError) -> WriteError { + match e { + crate::quic::driver::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`crate::quic::driver::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`crate::quic::driver::SendStream::stopped`]. + /// + /// Unlike the driver, this returns None if the code is not a valid WebTransport error code. + /// Also unlike the driver, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(crate::quic::driver::StoppedError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`crate::quic::driver::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`crate::quic::driver::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream.write_all(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`crate::quic::driver::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream.write_chunk(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`crate::quic::driver::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream.write_all_chunks(bufs).await.map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`crate::quic::driver::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in QUIC libraries. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.stream.id() + } } impl tokio::io::AsyncWrite for SendStream { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - // We have to use this syntax because noq added its own poll_write method. - tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_shutdown(cx) - } + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + // We have to use this syntax because the driver has its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } } impl web_transport_trait::SendStream for SendStream { - type Error = WriteError; - - fn set_priority(&mut self, order: i32) { - Self::set_priority(self, order).ok(); - } - - fn reset(&mut self, code: u32) { - Self::reset(self, code).ok(); - } - - fn finish(&mut self) -> Result<(), Self::Error> { - Self::finish(self).map_err(|_| WriteError::ClosedStream) - } - - async fn write(&mut self, buf: &[u8]) -> Result { - Self::write(self, buf).await - } - - async fn write_buf(&mut self, buf: &mut B) -> Result { - // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. - let size = buf.chunk().len(); - let chunk = buf.copy_to_bytes(size); - self.write_chunk(chunk).await?; - Ok(size) - } - - async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { - self.write_chunk(chunk).await - } - - async fn closed(&mut self) -> Result<(), Self::Error> { - // NOTE: This used to require &mut in an older version of Noq. - match self.stopped().await? { - Some(code) => Err(WriteError::Stopped(code)), - None => Ok(()), - } - } + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as the driver will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } } diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs index 8af3e81c98..381ec11934 100644 --- a/rs/moq-tokio/src/quic/web/server.rs +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -1,15 +1,15 @@ #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use std::sync::Arc; -use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use rustls::pki_types::{CertificateDer, PrivateKeyDer}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::{crypto, CongestionControl}; -use crate::{ - proto::{ConnectRequest, ConnectResponse}, - Connecting, ServerError, Session, Settings, +use crate::quic::web::{CongestionControl, crypto}; +use crate::quic::web::{ + Connecting, ServerError, Session, Settings, + proto::{ConnectRequest, ConnectResponse}, }; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -17,184 +17,178 @@ use crate::{ /// /// This is optional; advanced users may use [Server::new] directly. pub struct ServerBuilder { - provider: crypto::Provider, - addr: std::net::SocketAddr, - congestion_controller: - Option>, + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ServerBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ServerBuilder { - /// Create a server builder with sane defaults. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - addr: "[::]:443".parse().unwrap(), - congestion_controller: None, - } - } - - /// Listen on the specified address. - pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { - Self { addr, ..self } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Supply a certificate used for TLS. - // TODO support multiple certs based on...? - pub fn with_certificate( - self, - chain: Vec>, - key: PrivateKeyDer<'static>, - ) -> Result { - // Standard Noq setup - let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13])? - .with_no_client_auth() - .with_single_cert(chain, key)?; - - config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important - - let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); - let config = noq::ServerConfig::with_crypto(Arc::new(config)); - - let server = - noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; - - Ok(Server::new(server)) - } + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard QUIC endpoint setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::quic::web::ALPN.as_bytes().to_vec()]; // this one is important + + let config: crate::quic::driver::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = crate::quic::driver::ServerConfig::with_crypto(Arc::new(config)); + + let server = + crate::quic::driver::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } } /// A WebTransport server that accepts new sessions. pub struct Server { - endpoint: noq::Endpoint, - accept: FuturesUnordered>>, + endpoint: crate::quic::driver::Endpoint, + accept: FuturesUnordered>>, } impl core::ops::Deref for Server { - type Target = noq::Endpoint; + type Target = crate::quic::driver::Endpoint; - fn deref(&self) -> &Self::Target { - &self.endpoint - } + fn deref(&self) -> &Self::Target { + &self.endpoint + } } impl Server { - /// Manually create a new server with a manually constructed Endpoint. - /// - /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. - pub fn new(endpoint: noq::Endpoint) -> Self { - Self { - endpoint, - accept: Default::default(), - } - } - - /// Accept a new WebTransport session Request from a client. - pub async fn accept(&mut self) -> Option { - loop { - tokio::select! { - res = self.endpoint.accept() => { - let conn = res?; - self.accept.push(Box::pin(async move { - let conn = conn.await?; - Request::accept(conn).await - })); - } - Some(res) = self.accept.next() => { - if let Ok(session) = res { - return Some(session) - } - } - } - } - } + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::quic::web::ALPN` for WebTransport to work. + pub fn new(endpoint: crate::quic::driver::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } } /// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to /// accept or reject the session based on the URL. pub struct Request { - conn: noq::Connection, - settings: Settings, - connect: Connecting, + conn: crate::quic::driver::Connection, + settings: Settings, + connect: Connecting, } impl Request { - /// Accept a new WebTransport session from a client. - pub async fn accept(conn: noq::Connection) -> Result { - // Perform the H3 handshake by sending/reciving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Accept the CONNECT request but don't send a response yet. - let connect = Connecting::accept(&conn).await?; - - // Return the resulting request with a reference to the settings/connect streams. - Ok(Self { - conn, - settings, - connect, - }) - } - - pub async fn ok(self) -> Result { - self.respond(ConnectResponse::OK).await - } - - /// Reply to the session with the given response, usually 200 OK. - /// - /// [ConnectResponse::with_protocol] can be used to select a subprotocol. - pub async fn respond( - self, - response: impl Into, - ) -> Result { - let response = response.into(); - let connect = self.connect.respond(response).await?; - Ok(Session::new(self.conn, self.settings, connect)) - } - - /// Reject the session with the given status code. - pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { - self.connect.reject(status).await?; - Ok(()) - } - - /// Returns the underlying QUIC connection. - pub fn conn(&self) -> &noq::Connection { - &self.conn - } - - /// Returns the CONNECT request that was sent by the client. - /// - /// DEPRECATED: You can access this via the Deref impl. - pub fn connect(&self) -> &ConnectRequest { - &self.connect - } + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: crate::quic::driver::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond(self, response: impl Into) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &crate::quic::driver::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } } impl core::ops::Deref for Request { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.connect - } + fn deref(&self) -> &Self::Target { + &self.connect + } } diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index 82a9149d84..ad0fc8c2d6 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1,639 +1,609 @@ use std::{ - fmt, - future::Future, - io::Cursor, - ops::Deref, - pin::Pin, - sync::{Arc, Mutex, OnceLock}, - task::{Context, Poll, Waker}, - time::Duration, + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, }; use bytes::{Bytes, BytesMut}; use futures::stream::{FuturesUnordered, Stream, StreamExt}; -use kio::{Fan, Waiter}; +use kio::Waiter; -use crate::{ - proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, - ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, - WebTransportError, +use crate::quic::web::fan::Fan; +use crate::quic::web::{ + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, WebTransportError, + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, }; /// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake /// has not completed, or it is not valid UTF-8. /// -/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// The driver only exposes the ALPN via a downcast of the boxed handshake data, so the result /// is owned rather than borrowed from the connection. #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -fn negotiated_alpn(conn: &noq::Connection) -> Option { - let data = conn.handshake_data()?; - let data = data.downcast::().ok()?; +fn negotiated_alpn(conn: &crate::quic::driver::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data + .downcast::() + .ok()?; - String::from_utf8(data.protocol?).ok() + String::from_utf8(data.protocol?).ok() } -/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// `crate::quic::driver::crypto::rustls` only exists once a TLS backend is enabled, so there is no /// handshake data type to name here. /// /// This crate's own builders are gated the same way, so normally there is no connection -/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// to read an ALPN from either. A caller that enables a provider on `moq-quic` directly and /// brings its own endpoint can still establish one, and [`Session::protocol`] reports /// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to /// get the ALPN. #[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] -fn negotiated_alpn(_conn: &noq::Connection) -> Option { - None +fn negotiated_alpn(_conn: &crate::quic::driver::Connection) -> Option { + None } /// An established WebTransport session, acting like a full QUIC connection. See -/// [`noq::Connection`]. +/// [`crate::quic::driver::Connection`]. /// /// It is important to remember that WebTransport is layered on top of QUIC: /// 1. Each stream starts with a few bytes identifying the stream type and session ID. /// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. /// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. /// -/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// Deref is used to expose non-overloaded methods on [`crate::quic::driver::Connection`]. /// These should be safe to use with WebTransport, but file a PR if you find one that isn't. #[derive(Clone)] pub struct Session { - conn: noq::Connection, - - // The session ID, as determined by the stream ID of the connect request. - session_id: Option, - - // The accept logic is stateful, so use an Arc to share it. - accept: Option>>, - - // Cache the headers in front of each stream we open. - header_uni: Vec, - header_bi: Vec, - header_datagram: Vec, - - // Keep a reference to the settings and connect stream to avoid closing them until dropped. - #[allow(dead_code)] - settings: Option>, - - // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. - // Wrapped in Arc>> so close() can take it exactly once. - connect_send: Arc>>, - - // Why the session closed: set once by a local close() or the background task - // when a remote CloseWebTransportSession capsule is received. - close: Arc, - - // The request sent by the client, or None for a raw QUIC session. - request: Option, - - // The response sent by the server, or None for a raw QUIC session. - response: Option, - - // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only - // exposes it behind an allocating downcast, so `protocol()` has nothing in the - // connection to borrow from and needs somewhere to keep the result. - // - // Resolved on first read rather than at construction: `raw()` accepts any - // connection, including one whose handshake has not completed and so has no ALPN - // yet. Only a successful read is latched, so an early call cannot pin `None` for - // the life of the session. Shared across clones. - // - // Unused by HTTP/3 sessions, which read the subprotocol out of the response. - alpn: Arc>, + conn: crate::quic::driver::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. The driver only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, } impl Session { - pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { - // The session ID is the stream ID of the CONNECT request. - let session_id = connect.session_id(); - - // Cache the tiny header we write in front of each stream we open. - let mut header_uni = Vec::new(); - StreamUni::WEBTRANSPORT.encode(&mut header_uni); - session_id.encode(&mut header_uni); - - let mut header_bi = Vec::new(); - Frame::WEBTRANSPORT.encode(&mut header_bi); - session_id.encode(&mut header_bi); - - let mut header_datagram = Vec::new(); - session_id.encode(&mut header_datagram); - - let close = Arc::new(CloseReason::new(false)); - - // Accept logic is stateful, so use an Arc to share it. - let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); - - let this = Self { - conn, - accept: Some(Arc::new(Mutex::new(accept))), - session_id: Some(session_id), - header_uni, - header_bi, - header_datagram, - settings: Some(Arc::new(settings)), - connect_send: Arc::new(Mutex::new(Some(connect.send))), - close: close.clone(), - request: Some(connect.request.clone()), - response: Some(connect.response.clone()), - alpn: Default::default(), - }; - - // Run a background task to read capsules from the CONNECT recv stream. - let conn2 = this.conn.clone(); - tokio::spawn(Self::run_recv(conn2, connect.recv, close)); - - this - } - - // Read capsules from the CONNECT recv stream until it's closed, - // then record the close error and tear down the connection. - async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { - let close_info = Self::read_capsules(recv).await; - let code = close_info.as_ref().map_or(0, |(c, _)| *c); - - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Try to record the remote close error. If close() already set - // the error, it owns the connection teardown, so we bail out. - match close_info { - Some((code, reason)) => { - let err = WebTransportError::Closed(code, reason.clone()); - if !close.set(err.into()) { - return; - } - conn.close(http3_code, reason.as_bytes()); - } - None => { - let err = noq::ConnectionError::LocallyClosed.into(); - if !close.set(err) { - return; - } - conn.close(http3_code, b""); - } - }; - } - - // Keep reading capsules from the CONNECT recv stream until it's closed. - // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, - // or None if the stream closed without a capsule. - async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { - let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); - loop { - match reader.read().await { - Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { - code, - reason, - })) => return Some((code, reason)), - Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} - Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { - tracing::warn!(%typ, size = payload.len(), "unknown capsule"); - } - Ok(None) => return None, - Err(e) => { - tracing::warn!(?e, "failed to read capsule"); - return None; - } - } - } - } - - /// Connect using an established QUIC connection if you want to create the connection yourself. - /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. - pub async fn connect( - conn: noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Perform the H3 handshake by sending/receiving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Send the HTTP/3 CONNECT request. - let connect = Connected::open(&conn, request).await?; - - // Return the resulting session with a reference to the control/connect streams. - // If either stream is closed, then the session will be closed, so we need to keep them - // around. - let session = Session::new(conn, settings, connect); - - Ok(session) - } - - /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. - pub async fn accept_uni(&self) -> Result { - if let Some(accept) = &self.accept { - // `kio::wait` owns the waiter, so dropping this future — a `timeout` that - // expires, say — also drops its registration in `SessionAccept`. - kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let recv = self - .conn - .accept_uni() - .await - .map_err(|e| self.map_error(e))?; - Ok(RecvStream::new(recv, self.close.clone())) - } - } - - /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. - pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - if let Some(accept) = &self.accept { - kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - } - - /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. - pub async fn open_uni(&self) -> Result { - let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_uni) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(SendStream::new(send, self.close.clone())) - } - - /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. - pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_bi) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - - /// Asynchronously receives an application datagram from the remote peer. - /// - /// This method is used to receive an application datagram sent by the remote - /// peer over the connection. - /// It waits for a datagram to become available and returns the received bytes. - pub async fn read_datagram(&self) -> Result { - let mut datagram = self - .conn - .read_datagram() - .await - .map_err(|e| self.map_error(e))?; - - let mut cursor = Cursor::new(&datagram); - - if let Some(session_id) = self.session_id { - // We have to check and strip the session ID from the datagram. - let actual_id = - VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; - if actual_id != session_id { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // Return the datagram without the session ID. - let datagram = datagram.split_off(cursor.position() as usize); - - Ok(datagram) - } - - /// Sends an application datagram to the remote peer. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram(buf.into()) - } else { - self.conn.send_datagram(data) - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Sends an application datagram to the remote peer, waiting for buffer space if needed. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram_wait(buf.into()).await - } else { - self.conn.send_datagram_wait(data).await - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Computes the maximum size of datagrams that may be passed to - /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. - pub fn max_datagram_size(&self) -> usize { - self.conn - .max_datagram_size() - .unwrap_or(0) - .saturating_sub(self.header_datagram.len()) - } - - /// Returns the available buffer space for sending datagrams. - /// - /// This value is adjusted to account for the WebTransport session header. - pub fn datagram_send_buffer_space(&self) -> usize { - self.conn - .datagram_send_buffer_space() - .saturating_sub(self.header_datagram.len()) - } - - /// Close the session with an error code and reason. - /// - /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` - /// capsule is written on the CONNECT stream before the QUIC connection is closed. - /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. - /// - /// The capsule write and connection close happen asynchronously in a spawned task - /// that keeps the session alive until then, so the caller may drop it right away. - /// `await` [`Session::closed()`] to learn when the connection is gone. Session - /// operations will fail once the QUIC connection is closed. - pub fn close(&self, code: u32, reason: &[u8]) { - // First close wins: a no-op if the connection already closed, the background - // task recorded a remote close, or close() was already called. - if let Some(err) = self.conn.close_reason() { - self.close.set(self.map_error(err)); - return; - } - let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); - if !self.close.set(err) { - return; - } - - if self.session_id.is_some() { - // Take the send stream for the capsule write. - let send = self.connect_send.lock().unwrap().take(); - - if let Some(send) = send { - let reason = String::from_utf8_lossy(reason).into_owned(); - let capsule = - web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .unwrap_or(Duration::from_millis(100)); - let timeout = (rtt * 3).max(Duration::from_millis(100)); - - // The task owns a clone of the whole session, not just the connection: - // dropping the caller's last handle would otherwise end the HTTP/3 - // control and QPACK streams before the capsule is delivered, which a - // browser treats as a connection error, losing the code and reason. - let session = self.clone(); - tokio::spawn(async move { - session - .close_with_capsule(send, capsule, code, timeout) - .await; - }); - } - } else { - // Raw QUIC mode: no capsule needed. - self.conn.close(code.into(), reason); - } - } - - /// Deliver the close capsule and FIN, then give the peer its teardown grace. - async fn close_with_capsule( - &self, - mut send: noq::SendStream, - capsule: web_transport_proto::Capsule, - code: u32, - timeout: std::time::Duration, - ) { - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Encode the capsule, then wrap it in an HTTP/3 DATA frame. - // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT - // stream (RFC 9297 Section 3.2). - let mut capsule_bytes = Vec::new(); - capsule.encode(&mut capsule_bytes); - - let mut frame = Vec::new(); - Frame::DATA.encode(&mut frame); - let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { - tracing::warn!("capsule too large to encode as DATA frame"); - self.conn.close(http3_code, b""); - return; - }; - len.encode(&mut frame); - frame.extend_from_slice(&capsule_bytes); - - // Write the DATA frame to the CONNECT send stream. - if let Err(e) = send.write_all(&frame).await { - tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); - self.conn.close(http3_code, b""); - return; - } - - // FIN the send stream so the peer knows no more capsules are coming. - if let Err(e) = send.finish() { - tracing::warn!(?e, "failed to finish CONNECT send stream"); - self.conn.close(http3_code, b""); - return; - } - - // A queued write is not delivery: tearing down QUIC before the DATA and FIN - // are acknowledged can discard the capsule during loss recovery. Transport - // failure (including endpoint shutdown) also completes this wait. - match send.stopped().await { - Ok(None) => {} - Ok(Some(code)) => { - tracing::debug!(?code, "peer stopped the CONNECT stream during close"); - self.conn.close(http3_code, b""); - return; - } - Err(e) => { - tracing::debug!( - ?e, - "connection ended before the close capsule was acknowledged" - ); - return; - } - } - - // Wait for the peer to close the CONNECT stream after receiving the capsule. - if tokio::time::timeout(timeout, self.conn.closed()) - .await - .is_err() - { - tracing::debug!("timeout waiting for peer to close; force-closing connection"); - self.conn.close(http3_code, b""); - } - } - - /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. - /// - /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be - /// [`WebTransportError::Closed`] with the code and reason from the capsule. - /// - /// Unlike [`noq::Connection::closed`], this does **not** return early when - /// [`close()`](Self::close) has been called. It waits for the underlying QUIC - /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has - /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. - pub async fn closed(&self) -> SessionError { - self.map_error(self.conn.closed().await) - } - - /// Return why the session was closed, or None if it's not closed. See - /// [`noq::Connection::close_reason`]. - pub fn close_reason(&self) -> Option { - self.conn.close_reason().map(|e| self.map_error(e)) - } - - /// Report an error caused by the connection closing as the session's close reason. - fn map_error(&self, e: impl Into) -> SessionError { - self.close.map(e.into()) - } - - async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { - match send.write_all(buf).await { - Ok(_) => Ok(()), - Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), - Err(err) => Err(WebTransportError::WriteError(err).into()), - } - } - - /// Create a new session from a raw QUIC connection. - /// - /// This is used to pretend like a QUIC connection is a WebTransport session, - /// making it easier to support WebTransport and raw QUIC simultaneously. - /// - /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both - /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC - /// handshake, read from `conn` on demand — a connection that is still handshaking - /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. - pub fn raw(conn: noq::Connection) -> Self { - Self { - conn, - session_id: None, - header_uni: Default::default(), - header_bi: Default::default(), - header_datagram: Default::default(), - accept: None, - settings: None, - connect_send: Arc::new(Mutex::new(None)), - close: Arc::new(CloseReason::new(true)), - request: None, - response: None, - alpn: Default::default(), - } - } - - /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn request(&self) -> Option<&ConnectRequest> { - self.request.as_ref() - } - - /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn response(&self) -> Option<&ConnectResponse> { - self.response.as_ref() - } - - /// Returns the application protocol negotiated for this session. - /// - /// For an HTTP/3 session this is the subprotocol the server selected via - /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. - /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw - /// connection is still handshaking. - pub fn protocol(&self) -> Option<&str> { - if let Some(response) = &self.response { - return response.protocol.as_deref(); - } - - if let Some(alpn) = self.alpn.get() { - return Some(alpn); - } - - // Latch only a successful read, so a call made while the connection is still - // handshaking cannot pin `None` for the life of the session. - let alpn = negotiated_alpn(&self.conn)?; - Some(self.alpn.get_or_init(|| alpn)) - } - - /// Return connection-level statistics. - pub fn stats(&self) -> SessionStats { - let path = self.conn.path_stats(noq::PathId::ZERO); - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .or_else(|| path.map(|p| p.rtt)); - SessionStats { - stats: self.conn.stats(), - path, - rtt, - } - } + pub(crate) fn new(conn: crate::quic::driver::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv( + conn: crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + close: Arc, + ) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = crate::quic::driver::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { code, reason })) => { + return Some((code, reason)); + } + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`crate::quic::driver::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self.conn.accept_uni().await.map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`crate::quic::driver::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`crate::quic::driver::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`crate::quic::driver::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self.conn.read_datagram().await.map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(crate::quic::driver::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let timeout = (self.conn.rtt() * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session.close_with_capsule(send, capsule, code, timeout).await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: crate::quic::driver::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!(?e, "connection ended before the close capsule was acknowledged"); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()).await.is_err() { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`crate::quic::driver::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`crate::quic::driver::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`crate::quic::driver::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut crate::quic::driver::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(crate::quic::driver::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: crate::quic::driver::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + SessionStats(self.conn.stats()) + } } impl Deref for Session { - type Target = noq::Connection; + type Target = crate::quic::driver::Connection; - fn deref(&self) -> &Self::Target { - &self.conn - } + fn deref(&self) -> &Self::Target { + &self.conn + } } impl fmt::Debug for Session { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.conn.fmt(f) - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } } impl PartialEq for Session { - fn eq(&self, other: &Self) -> bool { - self.conn.stable_id() == other.conn.stable_id() - } + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } } impl Eq for Session {} @@ -645,687 +615,656 @@ impl Eq for Session {} // inline, and the first thing a resumed accepter does is take this same lock. An // arrival or a failure is exactly what the others are parked waiting to retry // after, so `Ready` is the signal. -fn poll_accept_uni_shared( - accept: &Mutex, - waiter: &Waiter, -) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.uni_waiters.clone(); - - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_uni(waiter); - - (result, waiters, hold) - }; - - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); - - if result.is_ready() { - waiters.wake(); - } - - result +fn poll_accept_uni_shared(accept: &Mutex, waiter: &Waiter) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result } fn poll_accept_bi_shared( - accept: &Mutex, - waiter: &Waiter, + accept: &Mutex, + waiter: &Waiter, ) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.bi_waiters.clone(); + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_bi(waiter); + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); - (result, waiters, hold) - }; + (result, waiters, hold) + }; - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); - if result.is_ready() { - waiters.wake(); - } + if result.is_ready() { + waiters.wake(); + } - result + result } // Type aliases just so clippy doesn't complain about the complexity. -type AcceptUni = dyn Stream> + Send; -type AcceptBi = - dyn Stream> + Send; -type PendingUni = dyn Future> + Send; -type PendingBi = - dyn Future, SessionError>> + Send; +type AcceptUni = + dyn Stream> + Send; +type AcceptBi = dyn Stream< + Item = Result< + (crate::quic::driver::SendStream, crate::quic::driver::RecvStream), + crate::quic::driver::ConnectionError, + >, + > + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = dyn Future, SessionError>> + + Send; // Logic just for accepting streams, which is annoying because of the stream header. pub struct SessionAccept { - session_id: VarInt, - - // Shared close reason for propagation to accepted streams. - close: Arc, - - // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them. Again, this is just so they don't get closed until we drop the session. - qpack_encoder: Option, - qpack_decoder: Option, - - accept_uni: Pin>, - accept_bi: Pin>, - - // Keep track of work being done to read/write the WebTransport stream header. - pending_uni: FuturesUnordered>>, - pending_bi: FuturesUnordered>>, - - // Waiters from concurrent callers of accept_bi / accept_uni. - // Every clone of the session polls this one struct, so an arrival has to be fanned - // out: each caller registers here and all of them are woken when a stream lands — - // by the caller that saw it, once it has released the lock on this struct. - bi_waiters: Fan, - uni_waiters: Fan, - - // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with - // the same waker every time. That waker outlives every caller, so an accepter that - // drops its future cannot take the wakeup path with it, and the layer below holds - // one registration rather than one per caller. - bi_waker: Waker, - uni_waker: Waker, + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, } impl SessionAccept { - pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { - // Create a stream that just outputs new streams, so it's easy to call from poll. - let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { - Some((conn.accept_uni().await, conn)) - })); - - let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { - Some((conn.accept_bi().await, conn)) - })); - - let bi_waiters = Fan::new(); - let uni_waiters = Fan::new(); - let bi_waker = bi_waiters.waker(); - let uni_waker = uni_waiters.waker(); - - Self { - session_id, - close, - - qpack_decoder: None, - qpack_encoder: None, - - accept_uni, - accept_bi, - - pending_uni: FuturesUnordered::new(), - pending_bi: FuturesUnordered::new(), - - bi_waiters, - uni_waiters, - bi_waker, - uni_waker, - } - } - - /// Poll for the next unidirectional WebTransport stream. - /// - /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops - /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive - /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this - /// with [`kio::wait`], which holds the waiter inside the future it builds. - /// - /// A `Ready` here means every *other* parked accepter should be woken so it can - /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes - /// them once the lock on this struct is released. - // - // Poll-based because we accept and decode streams in parallel. In async land this - // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is - // runtime-agnostic. - pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { - // Register before polling, not on the way out: the shared waker can fire from the - // layer below at any point here, and a wake that lands before the caller is on - // the list would be lost. - self.uni_waiters.register(waiter); - - let waker = self.uni_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let recv = match res { - Ok(recv) => recv, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_uni(recv, self.session_id); - self.pending_uni.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "unidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - // Decide if we keep looping based on the type. - match typ { - StreamUni::WEBTRANSPORT => { - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok(recv)); - } - StreamUni::QPACK_DECODER => { - self.qpack_decoder = Some(recv); - } - StreamUni::QPACK_ENCODER => { - self.qpack_encoder = Some(recv); - } - _ => { - // ignore unknown streams - tracing::debug!(?typ, "ignoring unknown unidirectional stream"); - } - } - } - } - - // Reads the stream header, returning the stream type. - async fn decode_uni( - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result<(StreamUni, noq::RecvStream), SessionError> { - // Read the VarInt at the start of the stream. - let typ = StreamUni(read_varint(&mut recv).await?); - - if typ == StreamUni::WEBTRANSPORT { - // Read the session_id and validate it - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them, so return everything. - Ok((typ, recv)) - } - - /// Poll for the next bidirectional WebTransport stream. - /// - /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` - /// registration is weak and owned by the caller, and a `Ready` is what the other - /// parked accepters need to be woken for. - pub fn poll_accept_bi( - &mut self, - waiter: &Waiter, - ) -> Poll> { - // Register before polling; see `poll_accept_uni`. - self.bi_waiters.register(waiter); - - let waker = self.bi_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let (send, recv) = match res { - Ok(pair) => pair, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_bi(send, recv, self.session_id); - self.pending_bi.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let res = match self.pending_bi.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "bidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - if let Some((send, recv)) = res { - // Wrap the streams in our own types for correct error codes. - let send = SendStream::new(send, self.close.clone()); - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok((send, recv))); - } - - // Keep looping if it's a stream we want to ignore. - } - } - - // Reads the stream header, returning Some if it's a WebTransport stream. - async fn decode_bi( - send: noq::SendStream, - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result, SessionError> { - let typ = read_varint(&mut recv).await?; - if Frame(typ) != Frame::WEBTRANSPORT { - tracing::debug!(?typ, "ignoring unknown bidirectional stream"); - return Ok(None); - } - - // Read the session ID and validate it. - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - - Ok(Some((send, recv))) - } + pub(crate) fn new(conn: crate::quic::driver::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::default(); + let uni_waiters = Fan::default(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, crate::quic::driver::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: crate::quic::driver::SendStream, + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } } // Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any // failure as a truncation, which would hide a stream the peer reset before its header // arrived. -async fn read_varint(recv: &mut noq::RecvStream) -> Result { - let mut buf = [0u8; 8]; - recv.read_exact(&mut buf[..1]).await?; +async fn read_varint(recv: &mut crate::quic::driver::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; - // The first two bits encode the length. - let size = 1 << (buf[0] >> 6); - recv.read_exact(&mut buf[1..size]).await?; + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; - Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) } // Without reliable reset, a stream the peer resets early loses its header with it, which // moq does routinely. Anything else is a misbehaving peer. fn log_header_error(err: SessionError, direction: &'static str) { - match err { - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError( - noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), - ), - )) => tracing::debug!(?err, direction, "stream closed before its header"), - _ => tracing::warn!(?err, direction, "failed to decode stream header"), - } + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError( + crate::quic::driver::ReadError::Reset(_) | crate::quic::driver::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } } -pub struct SessionStats { - stats: noq::ConnectionStats, - path: Option, - rtt: Option, -} +pub struct SessionStats(crate::quic::driver::ConnectionStats); impl web_transport_trait::Stats for SessionStats { - fn bytes_sent(&self) -> Option { - Some(self.stats.udp_tx.bytes) - } - - fn bytes_received(&self) -> Option { - Some(self.stats.udp_rx.bytes) - } - - fn bytes_lost(&self) -> Option { - self.path.map(|path| path.lost_bytes) - } - - fn packets_sent(&self) -> Option { - Some(self.stats.udp_tx.datagrams) - } - - fn packets_received(&self) -> Option { - Some(self.stats.udp_rx.datagrams) - } - - fn packets_lost(&self) -> Option { - self.path.map(|path| path.lost_packets) - } - - fn rtt(&self) -> Option { - self.rtt - } - - fn estimated_send_rate(&self) -> Option { - let path = self.path?; - let rtt = self.rtt?; - let rtt_secs = rtt.as_secs_f64(); - if path.cwnd > 0 && rtt_secs > 0.0 { - Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) - } else { - None - } - } + fn bytes_sent(&self) -> Option { + Some(self.0.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.0.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + Some(self.0.path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.0.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.0.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + Some(self.0.path.lost_packets) + } + + fn rtt(&self) -> Option { + Some(self.0.path.rtt) + } + + /// The congestion controller's bandwidth estimate, in bits per second. + fn estimated_send_rate(&self) -> Option { + self.0.path.bandwidth_estimate.map(|bytes| bytes.saturating_mul(8)) + } } impl web_transport_trait::Session for Session { - type SendStream = SendStream; - type RecvStream = RecvStream; - type Error = SessionError; - - async fn accept_uni(&self) -> Result { - Self::accept_uni(self).await - } - - async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::accept_bi(self).await - } - - async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::open_bi(self).await - } - - async fn open_uni(&self) -> Result { - Self::open_uni(self).await - } - - fn close(&self, code: u32, reason: &str) { - Self::close(self, code, reason.as_bytes()); - } - - async fn closed(&self) -> Self::Error { - Self::closed(self).await - } - - fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { - Self::send_datagram(self, data) - } - - async fn recv_datagram(&self) -> Result { - Self::read_datagram(self).await - } - - fn max_datagram_size(&self) -> usize { - Self::max_datagram_size(self) - } - - fn protocol(&self) -> Option<&str> { - Self::protocol(self) - } - - #[allow(refining_impl_trait)] - fn stats(&self) -> SessionStats { - Self::stats(self) - } + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } } #[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] mod tests { - use std::{ - io::{self, IoSliceMut}, - net::{Ipv4Addr, SocketAddr}, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - }; - - use rcgen::{CertifiedKey, KeyPair}; - use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; - - use super::*; - - const SESSION: VarInt = VarInt::from_u32(0); - - #[derive(Debug, Default)] - struct Gate { - blocked: AtomicBool, - attempted: tokio::sync::Notify, - wakers: Mutex>, - } - - #[derive(Debug)] - struct Socket { - inner: Box, - gate: Arc, - } - - #[derive(Debug)] - struct Sender { - inner: Pin>, - gate: Arc, - } - - impl noq::UdpSender for Sender { - fn poll_send( - self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, - cx: &mut Context<'_>, - ) -> Poll> { - let this = self.get_mut(); - let mut wakers = this.gate.wakers.lock().unwrap(); - if this.gate.blocked.load(Ordering::SeqCst) { - if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { - wakers.push(cx.waker().clone()); - } - this.gate.attempted.notify_one(); - return Poll::Pending; - } - drop(wakers); - this.inner.as_mut().poll_send(transmit, cx) - } - } - - impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { - Box::pin(Sender { - inner: self.inner.create_sender(), - gate: self.gate.clone(), - }) - } - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], - ) -> Poll> { - self.inner.poll_recv(cx, bufs, meta) - } - fn local_addr(&self) -> io::Result { - self.inner.local_addr() - } - } - - /// A connected client and server QUIC connection, plus the endpoints that drive them. - async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { - let (client, server, endpoints, _) = gated_pair().await; - (client, server, endpoints) - } - - async fn gated_pair() -> ( - noq::Connection, - noq::Connection, - [noq::Endpoint; 2], - Arc, - ) { - #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - - let CertifiedKey { cert, signing_key } = - rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); - let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( - &signing_key, - ))); - - let server_config = - noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); - let runtime: Arc = Arc::new(noq::TokioRuntime); - let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - let gate = Arc::new(Gate::default()); - let socket = Socket { - inner: runtime.wrap_udp_socket(udp).unwrap(), - gate: gate.clone(), - }; - let server = noq::Endpoint::new_with_abstract_socket( - Default::default(), - Some(server_config), - Box::new(socket), - runtime, - ) - .unwrap(); - - let mut roots = rustls::RootCertStore::empty(); - roots.add(cert.der().clone()).unwrap(); - let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); - let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); - - let connecting = client - .connect_with(client_config, server.local_addr().unwrap(), "localhost") - .unwrap(); - let accept = async { server.accept().await.unwrap().await.unwrap() }; - let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); - - (client_conn, server_conn, [client, server], gate) - } - - /// A queued capsule must survive a stalled socket beyond the teardown grace. - #[tokio::test] - async fn close_waits_for_capsule_ack() { - close_with_stalled_socket(false).await; - } - - /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. - #[tokio::test] - async fn close_ack_wait_ends_with_connection() { - close_with_stalled_socket(true).await; - } - - async fn close_with_stalled_socket(cancel: bool) { - let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; - let (client, server) = tokio::join!( - Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), - async { - crate::Request::accept(server_conn) - .await - .unwrap() - .ok() - .await - }, - ); - let client = client.unwrap(); - let server = server.unwrap(); - let conn = server.conn.clone(); - let settings = Arc::downgrade(server.settings.as_ref().unwrap()); - tokio::time::pause(); - gate.blocked.store(true, Ordering::SeqCst); - server.close(2, b"unauthorized"); - drop(server); - gate.attempted.notified().await; - tokio::time::advance(Duration::from_secs(1)).await; - tokio::task::yield_now().await; - assert!( - conn.close_reason().is_none(), - "closed before the capsule was acknowledged" - ); - - if cancel { - conn.close(0u32.into(), b"drain deadline"); - conn.closed().await; - tokio::task::yield_now().await; - assert_eq!( - settings.strong_count(), - 0, - "close task retained the HTTP/3 streams after cancellation" - ); - return; - } - - gate.blocked.store(false, Ordering::SeqCst); - for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { - waker.wake(); - } - tokio::time::resume(); - let err = client.closed().await; - assert!( - matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), - "{err:?}" - ); - conn.closed().await; - } - - fn assert_reset(err: SessionError) { - assert!( - matches!( - &err, - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) - )) if code.into_inner() == 7 - ), - "{err:?}" - ); - } - - /// A uni stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn uni_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - // The stream type alone, so the reset also cuts off the session ID. - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54]).await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A bi stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn bi_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - let (mut send, _recv) = client.open_bi().await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let (send, recv) = server.accept_bi().await.unwrap(); - let err = SessionAccept::decode_bi(send, recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A stream that names another session is still reported as `UnknownSession`. - #[tokio::test] - async fn uni_other_session() { - let (client, server, _endpoints) = pair().await; - - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); - send.finish().unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert!( - matches!( - err, - SessionError::WebTransportError(WebTransportError::UnknownSession) - ), - "{err:?}" - ); - } + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl crate::quic::driver::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &moq_sock::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl crate::quic::driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [moq_sock::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + ) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der(&signing_key))); + + let server_config = crate::quic::driver::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: crate::quic::driver::wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = crate::quic::driver::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = crate::quic::driver::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = crate::quic::driver::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { crate::quic::web::Request::accept(server_conn).await.unwrap().ok().await }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError(crate::quic::driver::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::UnknownSession)), + "{err:?}" + ); + } } diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index 08ea9a53a1..fdee95f23c 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -4,67 +4,67 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum SettingsError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::SettingsError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), - #[error("WebTransport is not supported")] - WebTransportUnsupported, + #[error("WebTransport is not supported")] + WebTransportUnsupported, - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), } pub struct Settings { - // A reference to the send/recv stream, so we don't close it until dropped. - #[allow(dead_code)] - send: noq::SendStream, + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: crate::quic::driver::SendStream, - #[allow(dead_code)] - recv: noq::RecvStream, + #[allow(dead_code)] + recv: crate::quic::driver::RecvStream, } impl Settings { - // Establish the H3 connection. - pub async fn connect(conn: &noq::Connection) -> Result { - let recv = Self::accept(conn); - let send = Self::open(conn); + // Establish the H3 connection. + pub async fn connect(conn: &crate::quic::driver::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); - // Run both tasks concurrently until one errors or they both complete. - let (send, recv) = try_join!(send, recv)?; - Ok(Self { send, recv }) - } + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } - async fn accept(conn: &noq::Connection) -> Result { - let mut recv = conn.accept_uni().await?; - let settings = web_transport_proto::Settings::read(&mut recv).await?; + async fn accept(conn: &crate::quic::driver::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; - tracing::debug!(?settings, "received SETTINGS frame"); + tracing::debug!(?settings, "received SETTINGS frame"); - if settings.supports_webtransport() == 0 { - return Err(SettingsError::WebTransportUnsupported); - } + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } - Ok(recv) - } + Ok(recv) + } - async fn open(conn: &noq::Connection) -> Result { - let mut settings = web_transport_proto::Settings::default(); - settings.enable_webtransport(1); + async fn open(conn: &crate::quic::driver::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); - tracing::debug!(?settings, "sending SETTINGS frame"); + tracing::debug!(?settings, "sending SETTINGS frame"); - let mut send = conn.open_uni().await?; - settings.write(&mut send).await?; + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; - Ok(send) - } + Ok(send) + } } diff --git a/rs/moq-tokio/src/resolve.rs b/rs/moq-tokio/src/resolve.rs index 3f58d119fa..2ce9408e11 100644 --- a/rs/moq-tokio/src/resolve.rs +++ b/rs/moq-tokio/src/resolve.rs @@ -289,7 +289,7 @@ impl Candidates { /// are handed out anyway so the dial surfaces the OS error instead of a /// confusing "no DNS entries". See /// for the Windows failure this family matching originally fixed. - #[cfg(any(feature = "noq", test))] + #[cfg(any(feature = "quic", test))] pub(crate) fn with_local(mut self, local: SocketAddr, dual_stack: bool) -> Self { self.local = Some((local, dual_stack)); self diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 3c57e4b287..754d40ff57 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -16,7 +16,7 @@ use url::Url; // Only the transports that finish their handshake in a spawned future need `.boxed()`; // the stream listeners hand back an already-built `Request`. -#[cfg(any(feature = "noq", feature = "iroh", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "iroh", feature = "websocket"))] use futures::FutureExt; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; @@ -86,7 +86,7 @@ impl crate::listen::Config { } /// Default bind address used when [`crate::listen::Config::bind`] is not set. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) const DEFAULT_BIND: net::SocketAddr = net::SocketAddr::V6(net::SocketAddrV6::new(net::Ipv6Addr::UNSPECIFIED, 443, 0, 0)); @@ -109,7 +109,7 @@ pub(crate) enum Parts { /// The QUIC listener only, as one member of a `SO_REUSEPORT` group. Folding /// the member in here is what stops a group member and a stream-only server /// from being asked for at once. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] Member(crate::listen::Socket), } @@ -130,7 +130,7 @@ impl Parts { /// This server's socket in a complete group, when it is a member of one. /// Consuming, because only one backend may own the serving handle. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] fn member(self) -> Option { match self { Self::Member(member) => Some(member), @@ -197,7 +197,7 @@ impl Config { } /// Copy the settings one QUIC worker owns. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn worker(&self) -> Self { Self { listen: self.listen.clone(), @@ -232,8 +232,8 @@ pub struct Server { streams: StreamListeners, #[cfg(feature = "iroh")] iroh: Option, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "websocket")] websocket: Option, } @@ -250,10 +250,10 @@ pub struct Server { /// only caller. #[derive(Clone, Debug)] #[allow(dead_code)] -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) struct SocketRetainer { - #[cfg(feature = "noq")] - noq: Option>, + #[cfg(feature = "quic")] + quic: Option>, } impl Server { @@ -314,9 +314,9 @@ impl Server { // `parts.quic()` and not `build_quic`: a caller that asked for streams only // is not asking for a backend, while leaving everything else configured is // still the error it always was. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] if config.bind.is_some() && parts.quic() { - return Err(Error::NoBackend("--listen requires the noq feature")); + return Err(Error::NoBackend("--listen requires the quic feature")); } // Only the QUIC backend verifies client certificates; the qmux listeners @@ -327,8 +327,8 @@ impl Server { return Err(Error::MtlsUnsupported); } if parts.quic() && !config.tls.root.is_empty() { - #[cfg(not(feature = "noq"))] - return Err(Error::NoBackend("--listen-tls-root requires the noq feature")); + #[cfg(not(feature = "quic"))] + return Err(Error::NoBackend("--listen-tls-root requires the quic feature")); } #[cfg(feature = "_certs")] @@ -337,7 +337,7 @@ impl Server { return Err(crate::tls::Error::ConflictingServe.into()); } #[allow(unused_mut)] - let mut needed = cfg!(feature = "noq") && build_quic; + let mut needed = cfg!(feature = "quic") && build_quic; #[cfg(feature = "tcp")] { needed |= build_streams && config.tcp.bind.is_some() && config.tcp.tls == Some(true); @@ -362,13 +362,13 @@ impl Server { // The member is a serving handle released by a complete reuseport group, // so the backend may take it only once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let member = parts.member(); - #[cfg(feature = "noq")] - let noq = build_quic + #[cfg(feature = "quic")] + let quic = build_quic .then(|| { - crate::noq::NoqServer::new(crate::noq::Options { + crate::quic::Server::new(crate::quic::Options { config: config.clone(), quic: quic.clone(), member, @@ -437,8 +437,8 @@ impl Server { streams, #[cfg(feature = "iroh")] iroh, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "websocket")] websocket, }) @@ -510,9 +510,9 @@ impl Server { /// /// Empty when no TLS-bearing backend is configured (e.g. a stream-only server). pub fn certificates(&self) -> crate::tls::Certificates { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return noq.certificates(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return quic.certificates(); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`): no certificates. crate::tls::Certificates::empty() @@ -522,17 +522,17 @@ impl Server { /// group after this server is gone. /// /// Crate-private: only the worker group retains sockets this way. A worker - /// member serves the same noq backend as the tokio listener. - #[cfg(feature = "noq")] + /// member serves the same QUIC backend as the tokio listener. + #[cfg(feature = "quic")] pub(crate) fn retain(&self) -> SocketRetainer { SocketRetainer { - #[cfg(feature = "noq")] - noq: self.noq.as_ref().and_then(|server| server.retain()), + #[cfg(feature = "quic")] + quic: self.quic.as_ref().and_then(|server| server.retain()), } } #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -624,7 +624,7 @@ impl Server { /// The body of [`Listener::accept`]. Stream accept loops start here if /// [`Server::bind`] left them stopped; [`Server::listen`] already started them. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -641,22 +641,22 @@ impl Server { } loop { // The QUIC endpoint address, reported as a QUIC session's local side. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let local = self.local_addr().ok(); // tokio::select! does not support cfg directives on arms, so we need to create the futures here. // `Some(None)` once a configured endpoint stops, which only losing its // driver does while we accept. - #[cfg(feature = "noq")] - let noq_accept = async { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_mut() { - return Some(noq.accept().await); + #[cfg(feature = "quic")] + let quic_accept = async { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_mut() { + return Some(quic.accept().await); } None }; - #[cfg(not(feature = "noq"))] - let noq_accept = async { None::<()> }; + #[cfg(not(feature = "quic"))] + let quic_accept = async { None::<()> }; #[cfg(feature = "iroh")] let iroh_accept = async { @@ -714,8 +714,8 @@ impl Server { #[cfg(not(any(feature = "tcp", all(feature = "uds", unix))))] let _ = request; } - Some(_conn) = noq_accept => { - #[cfg(feature = "noq")] + Some(_conn) = quic_accept => { + #[cfg(feature = "quic")] { // The process would keep its other listeners and silently drop // QUIC, so stop accepting at all and let the owner fail loudly. @@ -734,13 +734,13 @@ impl Server { // Accept the transport (capturing url + mTLS identity) and exchange the // MoQ SETUP up front, so path/role are known before the caller authorizes // (like the stream bindings). - let Accepted { session, url, identity, authority, mut link } = within(deadline, super::noq::accept(_conn, alpns)).await?; + let Accepted { session, url, identity, authority, mut link } = within(deadline, crate::quic::accept(_conn, alpns)).await?; link.local = local; let binding = match url { Some(_) => moq_net::transport::Binding::Uri, None => moq_net::transport::Binding::InBand }; let (request, deadline) = accept_request(&setup.server, session, binding, deadline).await?; // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; - Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline, active }) + Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Quic(Box::new(request)), deadline, active }) }.boxed()); } } @@ -809,9 +809,9 @@ impl Server { /// Errors with [`Error::NoBackend`] on a stream-only server, which has no /// QUIC listener. pub fn local_addr(&self) -> crate::Result { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return Ok(noq.local_addr()?); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return Ok(quic.local_addr()?); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`). Err(Error::NoBackend("no QUIC listener configured")) @@ -828,9 +828,9 @@ impl Server { #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] self.streams.shutdown().await; - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.take() { - noq.shutdown().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.take() { + quic.shutdown().await; } #[cfg(feature = "iroh")] if let Some(iroh) = self.iroh.take() { @@ -844,9 +844,9 @@ impl Server { /// Close the QUIC connections without waiting for the socket, as a drop must. fn close(&mut self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - noq.close(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + quic.close(); } } } @@ -943,7 +943,7 @@ async fn serve_session(request: Request) -> crate::Result<()> { /// The MoQ server an accepted connection handshakes against, and how long it has. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -957,7 +957,7 @@ struct Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -972,7 +972,7 @@ impl Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1035,7 +1035,7 @@ struct Deadline { impl Deadline { #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1078,7 +1078,7 @@ impl Deadline { /// Nothing is established yet that could carry a close code, so an expired /// handshake is dropped, which closes its connection. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1526,8 +1526,8 @@ async fn stream_request( type PendingRequest = moq_net::server::Handshake; pub(crate) enum RequestKind { - #[cfg(feature = "noq")] - Noq(Box>>), + #[cfg(feature = "quic")] + Quic(Box>>), #[cfg(feature = "iroh")] Iroh(Box>>), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1537,7 +1537,7 @@ pub(crate) enum RequestKind { /// The transport-level facts a backend captures while accepting a connection, before the /// MoQ SETUP. Grouped so the shared accept loop builds a [`Request`] from named fields /// rather than a wide tuple. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) struct Accepted { pub session: S, pub url: Option, @@ -1606,8 +1606,8 @@ pub enum Reject { macro_rules! request_ref { ($self:expr, $r:ident => $body:expr) => { match &$self.kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1620,8 +1620,8 @@ macro_rules! request_ref { macro_rules! request_into { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1634,8 +1634,8 @@ macro_rules! request_into { macro_rules! request_map { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => RequestKind::Noq(Box::new($body)), + #[cfg(feature = "quic")] + RequestKind::Quic($r) => RequestKind::Quic(Box::new($body)), #[cfg(feature = "iroh")] RequestKind::Iroh($r) => RequestKind::Iroh(Box::new($body)), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1729,7 +1729,7 @@ impl Request { /// transport carries no host (iroh, stream bindings). /// /// Reported as offered: rustls sends no SNI for an IP-literal dial (RFC 6066), - /// so noq reports `None`. + /// so the QUIC backend reports `None`. /// /// Not the moq-net IETF SETUP `Authority` parameter. Client-asserted and not authenticated, /// so authorize on the token or [`Self::peer_identity`] rather than on this value. @@ -1824,7 +1824,7 @@ impl Request { /// The client certificate chain the peer presented, if any, validated /// against a configured [`crate::tls::Listen::root`] during the handshake. /// - /// Captured at the transport handshake (before the SETUP). Noq supports mTLS; + /// Captured at the transport handshake (before the SETUP). The QUIC backend supports mTLS; /// other transports always return `None`. Use it to grant /// elevated access or to close the session once the certificate expires (see /// [`crate::tls::PeerIdentity::expiry`]). @@ -1859,7 +1859,7 @@ mod tests { assert!(matches!(died, Some(Err(err)) if err.is_panic())); } - #[cfg(all(not(feature = "noq"), feature = "_certs"))] + #[cfg(all(not(feature = "quic"), feature = "_certs"))] #[test] fn no_quic_backend_needs_no_served_identity() { Server::new(Config::default()).expect("an absent QUIC backend must not load a TLS identity"); @@ -2654,7 +2654,7 @@ mod tests { /// Closing a listener must release its UDP socket before returning, even while /// an accepted session still holds its QUIC connection, so a restart can rebind /// the port at once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn close_releases_quic_socket() { let config = crate::listen::Config { @@ -2709,7 +2709,7 @@ mod tests { } /// An explicit QUIC bind cannot be honored without a QUIC backend. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn quic_bind_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2729,7 +2729,7 @@ mod tests { /// A client CA on the default QUIC listener names the missing backend, not a /// stream-only server the caller never configured. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn client_ca_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2752,7 +2752,7 @@ mod tests { /// A QUIC-only server reports nothing. It multiplexes over one UDP socket and /// never calls `accept`, so a zero counter would be a watch that cannot fire. - #[cfg(all(feature = "noq", not(feature = "tcp")))] + #[cfg(all(feature = "quic", not(feature = "tcp")))] #[test] fn accept_health_is_empty_without_a_stream_listener() { let server = crate::listen::Config::default() @@ -2773,7 +2773,7 @@ mod tests { /// Building the endpoint needs a runtime, and `certificates()` must stay /// readable without one (no guard escapes to the caller). - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn certificates_expose_generated_fingerprints() { let mut config = crate::listen::Config { diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 1c19e4310f..2ff15e6778 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -376,7 +376,7 @@ mod tests { listen.tcp.tls = Some(true); listen.tls.generate = vec!["localhost".into()]; // Client-certificate policy is refused without a QUIC listener to read it. - if !cfg!(feature = "noq") { + if !cfg!(feature = "quic") { continue; } listen.bind = Some("127.0.0.1:0".parse().unwrap()); diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index 79f56ecef6..a28dcfcd7c 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -24,7 +24,7 @@ use rustls::pki_types::PrivatePkcs8KeyDer; /// Errors loading or generating TLS certificates and keys. /// -/// Shared by the client TLS config and noq server. +/// Shared by the client TLS config and QUIC server. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -791,7 +791,7 @@ pub(crate) enum Verification { Fingerprints(Vec<[u8; 32]>), /// Standard CA verification. When `system` is set the platform/default trust - /// store is trusted too; noq uses the OS platform verifier. + /// store is trusted too; the QUIC backend uses the OS platform verifier. /// `custom` are extra PEM roots trusted in addition. Roots { custom: CustomRoots, system: bool }, } @@ -955,7 +955,7 @@ impl Connect { /// plaintext fetch, and there is nothing to bootstrap when verification is /// disabled. With CA roots (the default), `http://` is the deliberate /// per-connection way to pin a self-signed relay, so it is allowed. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn allows_http_bootstrap(&self) -> bool { self.fingerprint.is_empty() && !self.insecure.unwrap_or_default() } @@ -1634,7 +1634,7 @@ impl PeerIdentity { /// Wrap the type-erased identity from the QUIC connection. /// Returns `None` if the peer presented no certificate or the identity is /// not a certificate chain. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn from_any(identity: Option>) -> Option { let chain = identity?.downcast::>>().ok()?; Some(Self { chain: *chain }) @@ -2092,7 +2092,7 @@ impl rustls::client::danger::ServerCertVerifier for FingerprintVerifier { } #[cfg(test)] -#[cfg(all(feature = "noq", feature = "aws-lc-rs"))] +#[cfg(all(feature = "quic", feature = "aws-lc-rs"))] mod tests { #[tokio::test] async fn resolved_config_keeps_reload_owner_alive() { @@ -2266,7 +2266,7 @@ mod tests { params.self_signed(&key).unwrap().into() } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_expiry_reads_not_after() { // notAfter at a whole second so the round-trip is exact. @@ -2277,7 +2277,7 @@ mod tests { params.not_after = not_after; let cert: CertificateDer<'static> = params.self_signed(&key).unwrap().into(); - // noq hand back the chain as a boxed Vec. + // The QUIC backend hands back the chain as a boxed Vec. let identity: Box = Box::new(vec![cert]); let parsed = PeerIdentity::from_any(Some(identity)).expect("chain parsed"); let expiry = parsed.expiry().expect("expiry parsed"); @@ -2287,7 +2287,7 @@ mod tests { ); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_none_without_chain() { assert!(PeerIdentity::from_any(None).is_none()); @@ -2390,7 +2390,7 @@ mod tests { /// keeps its task, an OS directory watch, and the certificate keys alive for the /// rest of the process. The `Arc` is the observable half: while the task lives it /// holds one, so a `Weak` that still upgrades is a watcher that never stopped. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[tokio::test] async fn dropping_the_reload_guard_stops_the_watcher() { let key = rcgen::KeyPair::generate().unwrap(); @@ -2525,7 +2525,7 @@ mod tests { /// A resumed handshake skips certificate verification, so a listener with a /// live peer set must disable resumption for removals to take effect. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn removed_peers_cannot_resume() { let server_identity = Identity::generate(["localhost"]).unwrap(); @@ -2559,7 +2559,7 @@ mod tests { } /// A plain-TLS embedder receives a working config from the shared identity. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn served_config_handshakes() { let server = Listen { @@ -2685,7 +2685,7 @@ mod tests { (file, path) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn signed_certificates() -> ( String, CertificateDer<'static>, @@ -2722,7 +2722,7 @@ mod tests { ) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn handshake_kinds( client: Arc, server: Arc, @@ -2754,7 +2754,7 @@ mod tests { panic!("TLS handshake did not settle"); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_roots_disable_session_resumption() { use std::io::Write; @@ -2805,7 +2805,7 @@ mod tests { ); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_client_roots_disable_server_resumption() { let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates(); @@ -2884,7 +2884,7 @@ mod tests { assert!(handshake_kinds(reloadable_client, reloadable).is_err()); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn custom_roots_reload_for_new_client_and_server_handshakes() { let (ca_a, server_a, _, client_a, _) = signed_certificates(); @@ -3296,7 +3296,7 @@ mod legacy_tests { /// The accept side, where a dropped `--server-tls-root` takes the mTLS client /// CAs with it and leaves the listener accepting unauthenticated peers. /// moq-cli's `export hls` flattens this type directly, so it is a live path. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn the_server_builders_refuse_a_released_spelling() { let tls = parse(&["--server-tls-root", "/tmp/ca.pem"]).listen; diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index a87ec2f1c1..5ab97f52a9 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -1,7 +1,7 @@ //! Adapts the async transport interface to the poll one moq-net requires. //! //! moq-net accepts its own `moq_net::transport::poll` traits. -//! A backend that offers only the async interface (qmux, iroh, noq) is wrapped +//! A backend that offers only the async interface (qmux, iroh, quic) is wrapped //! in [`Session`] here, at the edge where async already lives. The wrapping //! costs one allocation per operation and one copy per write, and its //! closed-watch emulation is weaker than a native implementation (see diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 609de2be9f..90b2e67fd1 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -184,7 +184,7 @@ fn won_key(url: &Url) -> Result<(String, u16)> { } /// Forget that WebSocket won for `url`, giving QUIC its head start again. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) fn forget(url: &Url) { if let Ok(key) = won_key(url) { WEBSOCKET_WON.lock().unwrap().remove(&key); @@ -192,13 +192,13 @@ pub(crate) fn forget(url: &Url) { } /// Record that WebSocket won for `url`, as a fallback dial that lands does. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn remember(url: &Url) { WEBSOCKET_WON.lock().unwrap().insert(won_key(url).unwrap()); } /// Whether the next dial to `url` skips QUIC's head start. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn won(url: &Url) -> bool { won_key(url).is_ok_and(|key| WEBSOCKET_WON.lock().unwrap().contains(&key)) } @@ -339,7 +339,7 @@ pub struct Resolved { /// The fallback arm of the QUIC-vs-WebSocket race, so only compiled when there is a /// QUIC dial to race against. A WebSocket-only build calls [`connect`] directly. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) async fn race_handle( config: &Config, tls: &rustls::ClientConfig, diff --git a/rs/moq-tokio/src/worker/group.rs b/rs/moq-tokio/src/worker/group.rs index 282a1db116..c98642d752 100644 --- a/rs/moq-tokio/src/worker/group.rs +++ b/rs/moq-tokio/src/worker/group.rs @@ -94,7 +94,7 @@ impl Workers { // hands out one member per slot in the order the kernel numbers them by. let mut forming = moq_sock::shard::Group::acquire(requested, config.count).map_err(|err| match err { moq_sock::shard::Error::Count { count, max } => Error::WorkerCount { count, max }, - moq_sock::shard::Error::Resolve(err) => crate::noq::Error::BindSocket(err).into(), + moq_sock::shard::Error::Resolve(err) => crate::quic::Error::BindSocket(err).into(), // The port lock is the only other way to lose the address, and it is // held by exactly one thing: another group of this UID. _ => Error::WorkerOverlap { addr: requested }, @@ -102,16 +102,16 @@ impl Workers { let count = forming.count(); let mut claims = Vec::with_capacity(count as usize); while let Some(member) = forming.member() { - claims.push(member.bind().map_err(crate::noq::Error::BindSocket)?); + claims.push(member.bind().map_err(crate::quic::Error::BindSocket)?); } - let mut group = forming.complete(claims).map_err(crate::noq::Error::BindSocket)?; + let mut group = forming.complete(claims).map_err(crate::quic::Error::BindSocket)?; let shared = Arc::new(Shared::default()); let mut workers = Vec::with_capacity(count as usize); let mut certificates = None; - while let Some(member) = group.member().map_err(crate::noq::Error::BindSocket)? { + while let Some(member) = group.member().map_err(crate::quic::Error::BindSocket)? { let index = member.shard().index(); // `max(1)` because an empty core list means pinning is off, not that // there are no workers. diff --git a/rs/moq-tokio/src/worker/mod.rs b/rs/moq-tokio/src/worker/mod.rs index 403835a4c0..1cb9991806 100644 --- a/rs/moq-tokio/src/worker/mod.rs +++ b/rs/moq-tokio/src/worker/mod.rs @@ -20,10 +20,10 @@ // Everything but the knobs: a group binds and serves QUIC, so it needs a backend // to bind with. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod group; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub use group::{Group, Member, Spawner, Workers}; /// How many QUIC workers to run, and whether to pin them. diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 6097344197..161ebce324 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -1,4 +1,4 @@ -//! Integration tests for noq QUIC and the iroh transport. +//! Integration tests for QUIC and the iroh transport. use std::time::Duration; @@ -10,7 +10,7 @@ fn timed() -> moq_net::track::Info { const TIMEOUT: Duration = Duration::from_secs(10); /// Inputs for [`connect_test`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct ConnectTest<'a> { /// URL scheme to dial (`moqt` for raw QUIC, `https` for WebTransport). scheme: &'a str, @@ -39,7 +39,7 @@ struct ConnectTest<'a> { /// /// Dials `localhost`, so the client sends an SNI. Use [`no_sni_test`] to cover /// the SNI-less path. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn backend_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -60,7 +60,7 @@ async fn backend_test(scheme: &str) { /// Raw QUIC (`moqt`/`moql`) has no request URI, so the whole request target has to /// ride the SETUP; WebTransport carries it in the CONNECT URL instead. Either way the /// server reports the same route and query through [`moq_tokio::server::Request`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn path_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -80,7 +80,7 @@ async fn path_test(scheme: &str) { /// in the server name). Raw QUIC has no in-band request URL, so this exercises /// the accept path with an empty server name, which must still establish rather /// than reject. Binds the loopback IP directly to avoid dual-stack flakiness. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn no_sni_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -98,7 +98,7 @@ async fn no_sni_test(scheme: &str) { /// Publish a broadcast on the server bound to `bind`, subscribe on a client that /// dials `authority`, and verify the data arrives over the requested scheme. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn connect_test(config: ConnectTest<'_>) { let ConnectTest { scheme, @@ -225,7 +225,7 @@ async fn connect_test(config: ConnectTest<'_>) { /// Write a self-signed PEM cert + key for `name` to `dir`, prefixed by `stem`. /// /// Returns the two paths, in the order the `cert` and `key` lists want them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::path::PathBuf, std::path::PathBuf) { use std::io::Write; @@ -253,7 +253,7 @@ fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::pat /// when the server picked the certificate the SNI asked for. Without SNI /// selection every client would get the first configured certificate, so the /// `alt.localhost` pin below would never match. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn sni_test() { let dir = tempfile::tempdir().expect("tempdir"); let (first_cert, first_key) = write_self_signed(dir.path(), "first", "localhost"); @@ -310,7 +310,7 @@ async fn sni_test() { } /// A generated certificate joins the file-backed ones rather than replacing them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn cert_sources_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -335,7 +335,7 @@ async fn cert_sources_test() { /// Rotate the certificate files under a running listener and assert the served /// set follows, without the listener being rebuilt. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn reload_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -383,7 +383,7 @@ async fn reload_test() { /// Generate a CA, a server cert + key, and a client cert + key (all PEM, the /// leaf certs signed by the CA) written to a tempdir. Returns the dir plus the /// five paths so the caller can wire them into the TLS configs. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { use rcgen::{ BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, KeyUsagePurpose, @@ -436,7 +436,7 @@ fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { } /// Filesystem paths to the PEM material produced by [`generate_mtls_certs`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct MtlsPaths { ca: std::path::PathBuf, server_cert: std::path::PathBuf, @@ -447,7 +447,7 @@ struct MtlsPaths { /// Connect with a client certificate signed by a CA the server trusts, and /// assert the server observes the validated peer certificate via mTLS. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn mtls_test(scheme: &str, reject: bool) { let (_dir, paths) = generate_mtls_certs(); @@ -736,14 +736,14 @@ async fn iroh_connect_test(version: Option<&str>) { .expect("server task failed"); } -// ── Noq backend ───────────────────────────────────────────────────── +// ── QUIC backend ──────────────────────────────────────────────────── /// A client that closes before its runtime stops tells the server at once, instead of /// leaving it to the idle timeout, which is what a process exiting on a signal does. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_reaches_server() { +async fn quic_client_close_reaches_server() { // An idle timeout inside TIMEOUT would hide a lost close. let mut quic = moq_tokio::quic::Config::default(); quic.idle_timeout = Some(TIMEOUT * 3); @@ -802,10 +802,10 @@ async fn noq_client_close_reaches_server() { /// A client that finishes its track and closes before its runtime stops still delivers /// the queued group and the track's finish, instead of the close discarding them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_finished_track() { +async fn quic_client_close_drains_finished_track() { // Small, since the debug build logging every packet is slow and the drain has // one second. Queued right before the close, it is still unacknowledged then. let payload: Vec = (0..1024).map(|i| i as u8).collect(); @@ -916,10 +916,10 @@ async fn noq_client_close_drains_finished_track() { /// A GOAWAY leaves the old session serving its subscriptions after the replacement /// connects, and a close then drains that predecessor too rather than dropping it. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_migrated_predecessor() { +async fn quic_client_close_drains_migrated_predecessor() { // Several congestion windows, so it is still in flight when the close starts. let payload: Vec = (0..64 * 1024).map(|i| i as u8).collect(); @@ -1064,10 +1064,10 @@ async fn noq_client_close_drains_migrated_predecessor() { /// A close cuts a predecessor that cannot drain off at its handover deadline, /// rather than holding it for the full one second close window. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_keeps_predecessor_handover() { +async fn quic_client_close_keeps_predecessor_handover() { let quic = moq_tokio::quic::Config::default(); let mut server_config = moq_tokio::listen::Config::default(); server_config.bind = Some("127.0.0.1:0".parse().unwrap()); @@ -1173,73 +1173,73 @@ async fn noq_client_close_keeps_predecessor_handover() { ); } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic() { +async fn quic_raw_quic() { backend_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_no_sni() { +async fn quic_raw_quic_no_sni() { no_sni_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_path() { +async fn quic_raw_quic_path() { path_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_moql_path() { +async fn quic_raw_quic_moql_path() { path_test("moql").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport_path() { +async fn quic_webtransport_path() { path_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport() { +async fn quic_webtransport() { backend_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_mtls() { +async fn quic_mtls() { mtls_test("https", false).await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_sni_certificate() { +async fn quic_sni_certificate() { sni_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_sources() { +async fn quic_cert_sources() { cert_sources_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_reload() { +async fn quic_cert_reload() { reload_test().await; } @@ -1249,8 +1249,8 @@ async fn noq_cert_reload() { /// files it left behind. /// /// Both ends write into one directory, so this covers the client and server paths -/// at once. Noq writes one file per connection. -#[cfg(all(feature = "qlog", feature = "noq"))] +/// at once. Each endpoint writes one file. +#[cfg(all(feature = "qlog", feature = "quic"))] async fn qlog_test(scheme: &str) -> Vec { let dir = tempfile::tempdir().expect("failed to create tempdir"); @@ -1276,7 +1276,7 @@ async fn qlog_test(scheme: &str) -> Vec { .collect(); for trace in &traces { - // Noq writes JSON-SEQ (RFC 7464): each record is a 0x1e separator then + // qlog streams JSON-SEQ (RFC 7464): each record is a 0x1e separator then // JSON, the first being the qlog header. Checking the bytes rather than just the // length catches a writer that was buffered and never flushed. let raw = std::fs::read(trace).expect("failed to read trace"); @@ -1303,9 +1303,9 @@ async fn qlog_test(scheme: &str) -> Vec { traces } -#[cfg(all(feature = "qlog", feature = "noq"))] +#[cfg(all(feature = "qlog", feature = "quic"))] #[tokio::test] -async fn noq_qlog() { +async fn quic_qlog() { let traces = qlog_test("moqt").await; assert!(!traces.is_empty(), "expected at least one trace"); } @@ -1334,7 +1334,7 @@ async fn connect_once( /// The size is the point: a frame that fits inside one window would pass whether or /// not the setting ever reached the backend. /// -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn window_test(scheme: &str) { let mut quic = moq_tokio::quic::Config::default(); quic.receive_window = Some(64 * 1024); @@ -1357,9 +1357,9 @@ async fn window_test(scheme: &str) { .await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] -async fn noq_windows() { +async fn quic_windows() { window_test("moqt").await; } diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 7cb70653e9..4ee4044de5 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -2481,7 +2481,7 @@ async fn quic_driver_task_inherits_connection_span() { .expect("group closed prematurely"); assert_eq!(&frame.payload[..], b"hello"); - // Other tasks (e.g. noq's own internal per-connection IO loop) legitimately + // Other tasks (e.g. the QUIC driver's per-connection IO loop) legitimately // carry the "conn" span too, since they're spawned while it's entered. So it's // not enough for the marker to appear *somewhere* in the logs; it must be on // the exact line moq-net emits from inside the driver task the runtime diff --git a/rs/moq-tokio/tests/close_code.rs b/rs/moq-tokio/tests/close_code.rs index 9d5f287e97..c3149c1f77 100644 --- a/rs/moq-tokio/tests/close_code.rs +++ b/rs/moq-tokio/tests/close_code.rs @@ -3,7 +3,7 @@ //! Each case runs several fresh connections, since the losses this guards against were //! races between the peer's close and whatever the transport reported next. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-tokio/tests/stream_code.rs b/rs/moq-tokio/tests/stream_code.rs index 922fe36488..508a56db8d 100644 --- a/rs/moq-tokio/tests/stream_code.rs +++ b/rs/moq-tokio/tests/stream_code.rs @@ -6,11 +6,11 @@ //! subscription it never made with STOP_SENDING, and the client reads that code off the //! wire. moq-transport codes take the same raw session. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::{sync::Arc, time::Duration}; -use web_transport_moq::noq; +use moq_tokio::quic::driver; /// moq-lite sends CANCEL (0x1) for a group it has no subscription for. const CANCEL: u32 = 0x1; @@ -68,7 +68,7 @@ async fn serve() -> Server { /// Dial the server over raw QUIC with `alpn`, send an empty SETUP and a GROUP for an /// unknown subscription, and return the code the server stopped the GROUP with. -async fn refused_code(server: &Server, alpn: &str) -> Option { +async fn refused_code(server: &Server, alpn: &str) -> Option { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let mut roots = rustls::RootCertStore::empty(); @@ -77,12 +77,12 @@ async fn refused_code(server: &Server, alpn: &str) -> Option { .with_root_certificates(roots) .with_no_client_auth(); tls.alpn_protocols = vec![alpn.as_bytes().to_vec()]; - let quic = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + let quic = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); - let endpoint = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); + let endpoint = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); let conn = endpoint .connect_with( - noq::ClientConfig::new(Arc::new(quic)), + driver::ClientConfig::new(Arc::new(quic)), ([127, 0, 0, 1], server.port).into(), "localhost", ) diff --git a/rs/moq-tokio/tests/worker.rs b/rs/moq-tokio/tests/worker.rs index c31196628b..bce5b6607e 100644 --- a/rs/moq-tokio/tests/worker.rs +++ b/rs/moq-tokio/tests/worker.rs @@ -3,7 +3,7 @@ //! Linux-only, because the mode is: every worker binds the listen address with //! `SO_REUSEPORT`, and no other platform load-balances a unicast UDP port across //! the group. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] use std::net::UdpSocket; diff --git a/rs/moq-transcode/Cargo.toml b/rs/moq-transcode/Cargo.toml index 1f5a900982..39ef2b9d42 100644 --- a/rs/moq-transcode/Cargo.toml +++ b/rs/moq-transcode/Cargo.toml @@ -44,7 +44,7 @@ tracing = { workspace = true } [dev-dependencies] anyhow = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } # test-util unlocks tokio::time::pause for the deterministic live-feed tests. tokio = { workspace = true, features = ["full", "test-util"] } url = { workspace = true } diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index e373fe2e65..a0e1f2513e 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -13,13 +13,13 @@ keywords = ["quic", "io-uring", "networking", "linux"] categories = ["asynchronous", "network-programming", "os::linux-apis"] [features] -default = ["noq"] -# moq-noq-proto, with TLS through rustls. -noq = ["dep:moq-noq-proto", "dep:rustls", "dep:rustls-platform-verifier"] +default = ["quic"] +# QUIC and WebTransport on moq-quic, with TLS through rustls. +quic = ["dep:moq-quic", "dep:moq-clock", "dep:rustls", "dep:rustls-platform-verifier"] # Capture qlog traces (`quic::Transport::qlog`). Off by default: it compiles the -# qlog crate and its serde_json encoder into noq, which a production build has +# qlog crate and its serde_json encoder into moq-quic, which a production build has # no use for. -qlog = ["moq-noq-proto?/qlog"] +qlog = ["moq-quic?/qlog"] [target.'cfg(target_os = "linux")'.dependencies] bytes = "1" @@ -27,9 +27,10 @@ http = "1" io-uring = { workspace = true } kio = { workspace = true } libc = { workspace = true } +# The QUIC driver hands moq-quic its clock's instants. +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } -# default-features off so the crypto provider is ours to pick. -moq-noq-proto = { workspace = true, features = ["bloom", "tracing-log", "rustls-aws-lc-rs"], optional = true } +moq-quic = { workspace = true, features = ["bloom", "rustls-aws-lc-rs"], optional = true } moq-sock = { workspace = true } rand = "0.10" rustc-hash = { workspace = true } @@ -56,7 +57,7 @@ web-transport-trait = { workspace = true } [target.'cfg(target_os = "linux")'.dev-dependencies] io-uring = { workspace = true } libc = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } nix = { workspace = true } rcgen = { version = "0.14", default-features = false, features = ["pem", "aws_lc_rs"] } rustls = "0.23" @@ -65,7 +66,6 @@ rustls = "0.23" serde_json = { workspace = true } tempfile = { workspace = true } tracing = "0.1" -web-transport-moq = { workspace = true } [[bench]] name = "udp_tokio" @@ -76,10 +76,9 @@ name = "udp_uring" harness = false [[bench]] -name = "echo_noq" -path = "benches/echo_noq.rs" +name = "echo_quic" harness = false -required-features = ["noq"] +required-features = ["quic"] [[bench]] name = "session_lite" diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index af817d5e93..db5a5a8aec 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -71,35 +71,35 @@ the tokio stack. ## Backends -The `quic` module uses the sans-IO [moq-noq-proto](https://github.com/moq-dev/noq) -stack with rustls. The `noq` feature is enabled by default and remains optional -so the worker, timers, and UDP socket can be built without QUIC. +The `quic` module uses the sans-IO [moq-quic](../moq-quic) stack with rustls. +The `quic` feature is enabled by default and remains optional so the worker, +timers, and UDP socket can be built without QUIC. | Feature | Stack | TLS | |---|---|---| -| `noq` (default) | [moq-noq-proto](https://github.com/moq-dev/noq) | rustls | +| `quic` (default) | [moq-quic](../moq-quic) | rustls | Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features noq +cargo test -p moq-uring --features quic ``` `moq-relay` enables it with `--features io-uring`. -The `qlog` feature turns on noq's capture support. It writes one file per -connection. +The `qlog` feature turns on moq-quic's capture support. It writes one file per +endpoint on the accept side and per dial on the connect side. ## Validation -`tests/echo.rs` runs an echo against a tokio noq peer over the worker: -handshake, half a megabyte each way, and timers driven by noq's timeout. +`tests/echo.rs` runs an echo against a tokio QUIC peer over the worker: +handshake, half a megabyte each way, and timers driven by moq-quic's timeout. `tests/session.rs` runs full moq-lite sessions through `quic::Endpoint` (including two clients demuxed on one server socket), and `tests/endpoint.rs` covers the endpoint mechanics (dial+accept on one socket, version negotiation, the dial-only refusal), `tests/workers.rs` runs a steered two-worker reuseport group serving one port across threads, and -`tests/web.rs` is WebTransport interop against `web-transport-moq`: stream and +`tests/web.rs` is WebTransport interop against moq-tokio's: stream and datagram echo through the H3 framing, close codes through the capsule, and a full moq-lite session over WebTransport. All of them skip (loudly) below the kernel floor, which @@ -109,8 +109,8 @@ includes GitHub-hosted CI runners. `udp_tokio` and `udp_uring` are the disposable syscall-level matrices from the first spike (recv batching x GRO x GSO, epoll vs io\_uring); see git history -for their methodology. `echo_noq` is the ablation matrix over the real -worker: the same noq echo with receive batching, GRO, and GSO toggled one +for their methodology. `echo_quic` is the ablation matrix over the real +worker: the same QUIC echo with receive batching, GRO, and GSO toggled one at a time. `worker_turns` sweeps socket counts and datagrams per burst independently to measure drive-loop batching without QUIC. Its smoke cases run in the Rust nightly benchmark group. diff --git a/rs/moq-uring/benches/echo_noq.rs b/rs/moq-uring/benches/echo_quic.rs similarity index 93% rename from rs/moq-uring/benches/echo_noq.rs rename to rs/moq-uring/benches/echo_quic.rs index e9c0866f1c..a7fbc1008d 100644 --- a/rs/moq-uring/benches/echo_noq.rs +++ b/rs/moq-uring/benches/echo_quic.rs @@ -1,4 +1,4 @@ -//! Echo a tokio noq peer through the io_uring worker while ablating UDP features. +//! Echo a tokio QUIC peer through the io_uring worker while ablating UDP features. use criterion::{criterion_group, criterion_main}; @@ -50,7 +50,7 @@ mod linux { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -72,11 +72,11 @@ mod linux { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); @@ -154,7 +154,7 @@ mod linux { ("all-off", none), ]; - let mut group = c.benchmark_group("echo_noq"); + let mut group = c.benchmark_group("echo_quic"); group.throughput(Throughput::Bytes(PAYLOAD as u64)); for (name, udp_config) in ablations { diff --git a/rs/moq-uring/benches/session_lite.rs b/rs/moq-uring/benches/session_lite.rs index cc629a98b6..1b4bd616e0 100644 --- a/rs/moq-uring/benches/session_lite.rs +++ b/rs/moq-uring/benches/session_lite.rs @@ -1,7 +1,7 @@ //! The PR-4 ablation matrix at session level: a full moq-lite session pair on //! one worker, publisher to subscriber over loopback QUIC, toggling receive //! batching, GRO, and GSO exactly like the raw echo. Each iteration delivers -//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_noq`'s unit), so +//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_quic`'s unit), so //! the two matrices are directly comparable: the difference is the moq-net //! machine and container framing on top of the same wire path. //! @@ -9,11 +9,11 @@ use criterion::{criterion_group, criterion_main}; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] #[path = "../tests/support.rs"] mod support; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] mod linux { use std::net::UdpSocket; use std::time::Instant; @@ -235,10 +235,10 @@ mod linux { } } -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] use linux::benchmark; -#[cfg(not(all(target_os = "linux", feature = "noq")))] +#[cfg(not(all(target_os = "linux", feature = "quic")))] fn benchmark(_: &mut criterion::Criterion) {} criterion_group!(benches, benchmark); diff --git a/rs/moq-uring/src/lib.rs b/rs/moq-uring/src/lib.rs index da3c2be544..09f75226f6 100644 --- a/rs/moq-uring/src/lib.rs +++ b/rs/moq-uring/src/lib.rs @@ -21,7 +21,7 @@ //! member of a steered reuseport group ([`udp::Bound`]) issues connection ids //! that steer back to it. [`Handle::run`] supplies time and schedules driver //! wakeups; callers run the returned drivers with [`Handle::spawn`]. The stack -//! underneath is enabled by the `noq` feature; a build without it leaves the +//! underneath is enabled by the `quic` feature; a build without it leaves the //! module out. //! //! [`metrics::Metrics`] is how the worker's own health leaves its thread: @@ -39,7 +39,7 @@ mod error; pub mod metrics; mod park; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub mod quic; mod shared; mod timer; diff --git a/rs/moq-uring/src/quic/noq/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs similarity index 87% rename from rs/moq-uring/src/quic/noq/connection.rs rename to rs/moq-uring/src/quic/driver/connection.rs index 9def53c5ee..fb70a5a112 100644 --- a/rs/moq-uring/src/quic/noq/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -7,7 +7,7 @@ use std::task::{Context, Poll}; use std::time::Instant; use bytes::Bytes; -use moq_noq_proto::{ConnectionHandle, Dir, StreamId, VarInt}; +use moq_quic::{ConnectionHandle, Dir, StreamId, VarInt}; use rustc_hash::FxHashMap; use super::super::{Error, SEGMENT}; @@ -28,7 +28,7 @@ const TRAIN_SEGMENTS: usize = 63; /// operation can mutate the connection, drop that borrow, and then register a /// waiter without ever holding both. pub(crate) struct Inner { - pub(crate) conn: RefCell, + pub(crate) conn: RefCell, pub(crate) state: RefCell, /// The worker driving this connection, which anything layered on it /// (the WebTransport handshake, say) runs on too. @@ -44,7 +44,7 @@ pub(crate) struct State { established: bool, establish_waiters: kio::WaiterList, - /// Whoever is waiting for a peer-initiated stream. noq-proto hands the + /// Whoever is waiting for a peer-initiated stream. moq-quic hands the /// ids out in order, so the queue is its own. accept_bi_waiters: kio::WaiterList, accept_uni_waiters: kio::WaiterList, @@ -55,7 +55,7 @@ pub(crate) struct State { /// Per-stream read/write parking, keyed by stream id. /// /// FxHash rather than SipHash on all four of these: they sit on the - /// driver's per-event path, and noq-proto assigns the ids. + /// driver's per-event path, and moq-quic assigns the ids. readable: FxHashMap, writable: FxHashMap, /// Send streams waiting for their end: a FIN the peer acknowledged, or a @@ -83,7 +83,7 @@ pub(crate) struct State { dead: bool, /// The close this side asked for, until the driver has put it on the wire. - /// noq raises no event for it, so this is what the terminal error is + /// moq-quic raises no event for it, so this is what the terminal error is /// built from. local_close: Option<(u64, String)>, } @@ -191,13 +191,13 @@ impl Inner { /// Start tracking send stream `id`, which a handle now owns. /// - /// Seeded from noq rather than empty: a peer can open a bidirectional + /// Seeded from moq-quic rather than empty: a peer can open a bidirectional /// stream and stop it before the application ever accepts it, and that - /// event arrives with no handle to record it against. noq still knows, + /// event arrives with no handle to record it against. moq-quic still knows, /// so a `poll_closed` on the fresh handle reports the stop instead of /// waiting for an event that has already been and gone. pub(crate) fn track(&self, id: StreamId) { - // Err means noq has no send half for the id, which is nothing to + // Err means moq-quic has no send half for the id, which is nothing to // report either way. let stopped = self.conn.borrow_mut().send_stream(id).stopped().ok().flatten(); self.state @@ -242,11 +242,11 @@ impl Inner { /// space, which needs the whole varint range. pub(crate) fn close_code(&self, code: u64, reason: &str) { self.conn.borrow_mut().close( - Instant::now(), + moq_clock::Instant::now(), VarInt::from_u64(code).unwrap_or(VarInt::MAX), Bytes::copy_from_slice(reason.as_bytes()), ); - // noq raises no event for a close the application asked for, so the + // moq-quic raises no event for a close the application asked for, so the // terminal error is ours to publish. Not here though: the driver // publishes it once it has staged the CONNECTION_CLOSE, so a caller // that stops driving the worker the moment `poll_closed` resolves has @@ -355,7 +355,7 @@ impl Clone for Connection { /// Build a connection's shared state and its driver. /// /// The driver future does timers, event sweeps, and egress; ingress arrives -/// from the endpoint's demux task, which feeds noq-proto directly and +/// from the endpoint's demux task, which feeds moq-quic directly and /// [kicks](Inner::kick) the driver. The caller spawns the future and reclaims /// the connection's bookkeeping once it resolves. pub(crate) fn launch( @@ -363,7 +363,7 @@ pub(crate) fn launch( socket: Rc, endpoint: Weak, key: ConnectionHandle, - conn: moq_noq_proto::Connection, + conn: moq_quic::Connection, ) -> (Shared, impl Future + use<>) { let shared = Rc::new(Inner { conn: RefCell::new(conn), @@ -443,7 +443,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu let handshake = conn .crypto_session() .handshake_data() - .and_then(|data| data.downcast::().ok()); + .and_then(|data| data.downcast::().ok()); let (alpn, server_name) = match handshake { Some(data) => ( data.protocol.map(|proto| String::from_utf8_lossy(&proto).into_owned()), @@ -451,11 +451,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu ), None => (None, None), }; - // Multipath is never negotiated here, so the first path is the only one. - let remote = conn - .network_path(moq_noq_proto::PathId::ZERO) - .expect("an established connection has its first path") - .remote(); + let remote = conn.remote_address(); (alpn, server_name, remote) }; @@ -563,7 +559,7 @@ impl web_transport_trait::poll::Session for Connection { Poll::Ready(Ok(())) } // The send queue is full; a flush frees space. - Err(moq_noq_proto::SendDatagramError::Blocked(_)) => { + Err(moq_quic::SendDatagramError::Blocked(_)) => { let mut state = self.shared.state.borrow_mut(); waiter.register(&mut state.datagram_send_waiters); Poll::Pending @@ -608,20 +604,16 @@ impl web_transport_trait::poll::Session for Connection { } fn stats(&self) -> impl web_transport_trait::Stats { - let (stats, path) = { - let mut conn = self.shared.conn.borrow_mut(); - let stats = conn.stats(); - let path = conn.path_stats(moq_noq_proto::PathId::ZERO).unwrap_or_default(); - (stats, path) - }; + let stats = self.shared.conn.borrow().stats(); Stats { bytes_sent: stats.udp_tx.bytes, bytes_received: stats.udp_rx.bytes, - bytes_lost: stats.lost_bytes, + bytes_lost: stats.path.lost_bytes, packets_sent: stats.udp_tx.datagrams, packets_received: stats.udp_rx.datagrams, - packets_lost: stats.lost_packets, - rtt: path.rtt, + packets_lost: stats.path.lost_packets, + rtt: stats.path.rtt, + bandwidth: stats.path.bandwidth_estimate, } } } @@ -632,7 +624,7 @@ impl std::fmt::Debug for Connection { } } -/// A snapshot of noq-proto's counters in [`web_transport_trait::Stats`] +/// A snapshot of moq-quic's counters in [`web_transport_trait::Stats`] /// shape. struct Stats { bytes_sent: u64, @@ -642,6 +634,8 @@ struct Stats { packets_received: u64, packets_lost: u64, rtt: std::time::Duration, + /// The congestion controller's bandwidth estimate in bytes per second. + bandwidth: Option, } impl web_transport_trait::Stats for Stats { @@ -673,24 +667,22 @@ impl web_transport_trait::Stats for Stats { Some(self.rtt) } - /// Nothing, because noq-proto exposes no delivery or pacing rate. + /// The congestion controller's bandwidth estimate, in bits per second. /// - /// The congestion window over the RTT is not a stand-in for one: BBR - /// deliberately holds about twice the bandwidth-delay product (nearly - /// three times it while starting up), so that number is a multiple of - /// what the path will carry, and moq-net feeds this straight into its - /// bandwidth allocator and PROBE. A missing sample is a state the model - /// already handles (`stats.estimated_send_rate` is an `Option`, and no - /// bandwidth producer is created without one); an inflated one is a rate - /// an encoder will chase. + /// BBR estimates the delivery rate; CUBIC offers none. The congestion + /// window over the RTT is not a stand-in: BBR deliberately holds about + /// twice the bandwidth-delay product, so that number is a multiple of what + /// the path will carry, and moq-net feeds this straight into its bandwidth + /// allocator and PROBE. A missing sample is a state the model already + /// handles; an inflated one is a rate an encoder will chase. fn estimated_send_rate(&self) -> Option { - None + self.bandwidth.map(|bytes| bytes.saturating_mul(8)) } } /// The per-connection task: endpoint events, application events, timers, and /// packets out. Packets in come from the endpoint's demux task, which feeds -/// noq-proto directly and kicks this driver. +/// moq-quic directly and kicks this driver. struct Driver { shared: Shared, socket: Rc, @@ -700,11 +692,11 @@ struct Driver { endpoint: Weak, key: ConnectionHandle, deadline: crate::Timer, - /// Egress staging: noq-proto writes into a `Vec`, so a train is built + /// Egress staging: moq-quic writes into a `Vec`, so a train is built /// here and copied into the socket's registered buffer. scratch: Vec, /// A CONNECTION_CLOSE has been handed to the socket. Not implied by a - /// flush that came back empty: noq paces the close like any other packet, + /// flush that came back empty: moq-quic paces the close like any other packet, /// so it can sit behind a pacing timer after the application asked for it. close_staged: bool, } @@ -732,9 +724,9 @@ impl Driver { } // A closed connection still owes the peer its CONNECTION_CLOSE // (and a retransmit for each packet that arrives after), so the - // driver runs until noq says the drain is over. + // driver runs until moq-quic says the drain is over. if self.shared.conn.borrow().is_drained() { - // A close noq never managed to send (the server's + // A close moq-quic never managed to send (the server's // anti-amplification limit can withhold it) ends here too. self.publish_close(); return Poll::Ready(()); @@ -752,11 +744,12 @@ impl Driver { // polling before the set would leave the firing to wake nobody // (fatal on a dial nobody answers, where no ingress ever re-polls // us). - self.deadline.set(self.shared.conn.borrow_mut().poll_timeout()); + self.deadline + .set(self.shared.conn.borrow_mut().poll_timeout().map(super::to_std)); if self.deadline.poll(waiter).is_pending() { return Poll::Pending; } - self.shared.conn.borrow_mut().handle_timeout(Instant::now()); + self.shared.conn.borrow_mut().handle_timeout(moq_clock::Instant::now()); } } @@ -791,25 +784,24 @@ impl Driver { let mut state = self.shared.state.borrow_mut(); match event { - moq_noq_proto::Event::Connected => { + moq_quic::Event::Connected => { state.established = true; state.establish_waiters.wake(); } - moq_noq_proto::Event::ConnectionLost { reason } => state.fail(reason.into()), - moq_noq_proto::Event::DatagramReceived => state.datagram_recv_waiters.wake(), - moq_noq_proto::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), - moq_noq_proto::Event::HandshakeDataReady => {} - moq_noq_proto::Event::Stream(event) => sweep_stream(&mut state, event), - moq_noq_proto::Event::HandshakeConfirmed - | moq_noq_proto::Event::Path(_) - | moq_noq_proto::Event::NatTraversal(_) => {} + moq_quic::Event::ConnectionLost { reason } => state.fail(reason.into()), + moq_quic::Event::DatagramReceived => state.datagram_recv_waiters.wake(), + moq_quic::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), + moq_quic::Event::HandshakeDataReady => {} + moq_quic::Event::Stream(event) => sweep_stream(&mut state, event), + // Non-exhaustive upstream; nothing else needs a wakeup. + _ => {} } } } /// Publish the terminal error for a close this side asked for. /// - /// noq raises no event for it, so the driver is what reports it, and + /// moq-quic raises no event for it, so the driver is what reports it, and /// only once the CONNECTION_CLOSE is staged (or the drain is over), since /// an application is free to stop driving the worker the moment /// `poll_closed` resolves. Staged is not delivered: the send is @@ -832,7 +824,7 @@ impl Driver { // Backpressure, or nothing left to stage. Poll::Pending => return Poll::Pending, } - // Requeue behind the other ready tasks. If noq is drained, the next + // Requeue behind the other ready tasks. If moq-quic is drained, the next // poll costs one empty acquire and then parks normally. waiter.waker().wake_by_ref(); Poll::Pending @@ -858,27 +850,26 @@ impl Driver { tx.len() )))); } - let segments = std::num::NonZeroUsize::new(segments).expect("segments was checked above"); self.scratch.clear(); let (transmit, closing) = { let mut conn = self.shared.conn.borrow_mut(); // Nothing to send, or paced; the buffer returns to the pool on drop. - let Some(transmit) = conn.poll_transmit(Instant::now(), segments, &mut self.scratch) else { + let Some(transmit) = conn.poll_transmit(moq_clock::Instant::now(), segments, &mut self.scratch) else { return Poll::Pending; }; - // Once closed, noq transmits nothing but the CONNECTION_CLOSE. + // Once closed, moq-quic transmits nothing but the CONNECTION_CLOSE. (transmit, conn.is_closed()) }; tx[..transmit.size].copy_from_slice(&self.scratch[..transmit.size]); // A lone datagram is its own segment size, and the socket's GSO - // stride has to match what noq actually packed. + // stride has to match what moq-quic actually packed. let transmit = udp::Transmit { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { return Poll::Ready(Err(Error::Io(err.to_string()))); @@ -913,25 +904,25 @@ fn end(state: &mut State, id: StreamId, end: End) { } /// Apply one stream event to the parking tables. -fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { +fn sweep_stream(state: &mut State, event: moq_quic::StreamEvent) { // Waking removes the entry: a still-interested poller re-registers on its // next poll, so the maps only hold streams somebody is parked on. match event { - moq_noq_proto::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), - moq_noq_proto::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), - moq_noq_proto::StreamEvent::Available { .. } => state.open_waiters.wake(), - moq_noq_proto::StreamEvent::Readable { id } => { + moq_quic::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), + moq_quic::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), + moq_quic::StreamEvent::Available { .. } => state.open_waiters.wake(), + moq_quic::StreamEvent::Readable { id } => { if let Some(mut waiters) = state.readable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Writable { id } => { + moq_quic::StreamEvent::Writable { id } => { if let Some(mut waiters) = state.writable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Finished { id } => end(state, id, End::Delivered), - moq_noq_proto::StreamEvent::Stopped { id, error_code } => { + moq_quic::StreamEvent::Finished { id } => end(state, id, End::Delivered), + moq_quic::StreamEvent::Stopped { id, error_code } => { end(state, id, End::Stopped(error_code.into_inner())); // A writer blocked on capacity has to learn it will never come. if let Some(mut waiters) = state.writable.remove(&id) { @@ -943,7 +934,7 @@ fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { #[cfg(test)] mod tests { - use moq_noq_proto::Side; + use moq_quic::Side; use super::*; diff --git a/rs/moq-uring/src/quic/noq/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs similarity index 92% rename from rs/moq-uring/src/quic/noq/endpoint.rs rename to rs/moq-uring/src/quic/driver/endpoint.rs index 2a7ff9d8e5..2246efed3e 100644 --- a/rs/moq-uring/src/quic/noq/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -1,6 +1,6 @@ -//! The noq endpoint: one socket, many noq-proto connections. +//! The endpoint: one socket, many moq-quic connections. //! -//! noq-proto's own [`moq_noq_proto::Endpoint`] is the routing table: it parses +//! moq-quic's own [`moq_quic::Endpoint`] is the routing table: it parses //! each datagram, hands it to the connection its destination id names, mints //! and retires ids as peers consume them, and answers an unsupported version //! itself. What is left for us is the socket, the accept backlog, and a driver @@ -17,7 +17,7 @@ use std::task::Poll; use std::time::Instant; use bytes::BytesMut; -use moq_noq_proto::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; +use moq_quic::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; use rustc_hash::FxHashMap; use super::super::{Error, endpoint::Config}; @@ -42,11 +42,11 @@ pub(crate) struct Inner { socket: Rc, local: SocketAddr, /// The routing table, and the server configuration it accepts with. - endpoint: RefCell, + endpoint: RefCell, accepting: RefCell>, /// Every live connection, looked up per received datagram. /// - /// FxHash rather than SipHash: noq-proto hands out the handle, and it + /// FxHash rather than SipHash: moq-quic hands out the handle, and it /// owns connection-id routing (and its hasher choices) itself, so nothing /// a peer picks reaches this map. conns: RefCell>, @@ -91,7 +91,7 @@ impl Endpoint { let server = match &config.server { Some(server) => { let mut server = super::server_config(server)?; - // noq buffers half-open handshakes itself, so it enforces + // moq-quic buffers half-open handshakes itself, so it enforces // the same bound the accept queue does. server.max_incoming(config.backlog); Some(Arc::new(server)) @@ -101,7 +101,7 @@ impl Endpoint { let accepting = server.is_some().then(|| Accepting { queue: VecDeque::new() }); // MTU discovery is off (the GSO pool sends fixed SEGMENT datagrams), // so the endpoint has no reason to allow it either. - let endpoint = moq_noq_proto::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); + let endpoint = moq_quic::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); let inner = Rc::new(Inner { activity: config.activity, @@ -165,7 +165,7 @@ impl Endpoint { .inner .endpoint .borrow_mut() - .connect(Instant::now(), client, config.peer, &config.server_name) + .connect(moq_clock::Instant::now(), client, config.peer, &config.server_name) .map_err(|err| Error::Quic(err.to_string()))?; let shared = self.inner.launch(key, conn); // Flush the Initial flight; the demux task takes it from here. @@ -246,7 +246,7 @@ impl Inner { /// Route every datagram in one receive, then kick the connections fed. fn demux(self: &Rc, packet: &mut udp::Packet) { let from = packet.from(); - let ecn = packet.ecn().map(super::ecn_to_noq); + let ecn = packet.ecn().map(super::ecn_to_quic); // Where the endpoint writes its own answers (version negotiation, // retry, a refusal), reused across the whole receive. let mut buf = Vec::new(); @@ -261,8 +261,9 @@ impl Inner { // arrived on, so the four-tuple stays half known, exactly as it // does on a platform without `IP_PKTINFO`. let event = self.endpoint.borrow_mut().handle( - Instant::now(), - from.into(), + moq_clock::Instant::now(), + from, + None, ecn, BytesMut::from(&segment[..]), &mut buf, @@ -339,9 +340,14 @@ impl Inner { return None; }; buf.clear(); - let now = Instant::now(); - let deadline = self.handshake.timeout.and_then(|timeout| now.checked_add(timeout)); - let accepted = self.endpoint.borrow_mut().accept(incoming, now, buf, None); + let deadline = self + .handshake + .timeout + .and_then(|timeout| Instant::now().checked_add(timeout)); + let accepted = self + .endpoint + .borrow_mut() + .accept(incoming, moq_clock::Instant::now(), buf, None); let (key, conn) = match accepted { Ok(accepted) => accepted, Err(err) => { @@ -402,7 +408,7 @@ impl Inner { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { tracing::debug!(%err, "failed to send an endpoint response"); @@ -410,7 +416,7 @@ impl Inner { } /// Register `conn`, spawn its driver, and arrange its teardown. - fn launch(self: &Rc, key: ConnectionHandle, conn: moq_noq_proto::Connection) -> connection::Shared { + fn launch(self: &Rc, key: ConnectionHandle, conn: moq_quic::Connection) -> connection::Shared { let (shared, driver) = connection::launch(&self.owner, self.socket.clone(), Rc::downgrade(self), key, conn); self.conns.borrow_mut().insert(key, shared.clone()); @@ -427,8 +433,8 @@ impl Inner { pub(crate) fn on_connection_event( &self, key: ConnectionHandle, - event: moq_noq_proto::EndpointEvent, - ) -> Option { + event: moq_quic::EndpointEvent, + ) -> Option { self.endpoint.borrow_mut().handle_event(key, event) } diff --git a/rs/moq-uring/src/quic/noq/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs similarity index 79% rename from rs/moq-uring/src/quic/noq/mod.rs rename to rs/moq-uring/src/quic/driver/mod.rs index 00d457bcdb..b1dc8fc86d 100644 --- a/rs/moq-uring/src/quic/noq/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -1,13 +1,13 @@ -//! The rustls-backed noq-proto stack on the worker's UDP path. +//! The rustls-backed moq-quic stack on the worker's UDP path. //! //! TLS goes through rustls, which is what the rest of this workspace speaks, //! so a build selecting this backend links one TLS stack instead of two. //! -//! [`moq_noq_proto::Endpoint`] holds -//! the connection-id routing table, mints and retires ids, answers unsupported -//! versions, and buffers half-open handshakes. So [`Endpoint`] here is mostly -//! the socket plumbing around it, and the parts that are ours (the accept -//! backlog, shard steering, and the driver task per connection. +//! [`moq_quic::Endpoint`] holds the connection-id routing table, mints and +//! retires ids, answers unsupported versions, and buffers half-open +//! handshakes. So [`Endpoint`] here is mostly the socket plumbing around it, +//! plus the parts that are ours: the accept backlog, shard steering, and the +//! driver task per connection. mod connection; mod endpoint; @@ -21,7 +21,7 @@ pub(crate) use connection::{End, Shared}; use std::sync::Arc; -use moq_noq_proto::crypto::rustls::{QuicClientConfig, QuicServerConfig}; +use moq_quic::crypto::rustls::{QuicClientConfig, QuicServerConfig}; use rustls::pki_types::pem::PemObject; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; @@ -37,25 +37,34 @@ const CONNECTION_WINDOW: u32 = 16 * 1024 * 1024; /// How many datagrams to buffer in each direction. const DATAGRAM_WINDOW: usize = 64 * SEGMENT; -fn ecn_to_noq(ecn: udp::Ecn) -> moq_noq_proto::EcnCodepoint { +fn ecn_to_quic(ecn: udp::Ecn) -> moq_quic::EcnCodepoint { match ecn { - udp::Ecn::Ect0 => moq_noq_proto::EcnCodepoint::Ect0, - udp::Ecn::Ect1 => moq_noq_proto::EcnCodepoint::Ect1, - udp::Ecn::Ce => moq_noq_proto::EcnCodepoint::Ce, + udp::Ecn::Ect0 => moq_quic::EcnCodepoint::Ect0, + udp::Ecn::Ect1 => moq_quic::EcnCodepoint::Ect1, + udp::Ecn::Ce => moq_quic::EcnCodepoint::Ce, } } -fn ecn_from_noq(ecn: moq_noq_proto::EcnCodepoint) -> udp::Ecn { +fn ecn_from_quic(ecn: moq_quic::EcnCodepoint) -> udp::Ecn { match ecn { - moq_noq_proto::EcnCodepoint::Ect0 => udp::Ecn::Ect0, - moq_noq_proto::EcnCodepoint::Ect1 => udp::Ecn::Ect1, - moq_noq_proto::EcnCodepoint::Ce => udp::Ecn::Ce, + moq_quic::EcnCodepoint::Ect0 => udp::Ecn::Ect0, + moq_quic::EcnCodepoint::Ect1 => udp::Ecn::Ect1, + moq_quic::EcnCodepoint::Ce => udp::Ecn::Ce, } } -impl From for Error { - fn from(err: moq_noq_proto::ConnectionError) -> Self { - use moq_noq_proto::ConnectionError; +/// Map a moq-quic deadline onto the worker's timers. +/// +/// moq-quic counts on moq-clock's instants while the worker's timers still take +/// std's, until the worker becomes a moq-clock backend. A deadline already past +/// maps to now, which fires at once either way. +fn to_std(at: moq_clock::Instant) -> std::time::Instant { + std::time::Instant::now() + at.saturating_duration_since(moq_clock::Instant::now()) +} + +impl From for Error { + fn from(err: moq_quic::ConnectionError) -> Self { + use moq_quic::ConnectionError; match err { ConnectionError::ApplicationClosed(close) => Self::App { code: close.error_code.into_inner(), @@ -91,10 +100,8 @@ fn provider() -> Arc { /// The endpoint-wide configuration: how connection ids are minted, and the /// largest datagram we tell peers we can receive. -pub(crate) fn endpoint_config( - shard: Option, -) -> Result, Error> { - let mut config = moq_noq_proto::EndpointConfig::default(); +pub(crate) fn endpoint_config(shard: Option) -> Result, Error> { + let mut config = moq_quic::EndpointConfig::default(); config.cid_generator(Arc::new(move || Box::new(Cids { shard }))); config .max_udp_payload_size(SEGMENT as u16) @@ -104,7 +111,7 @@ pub(crate) fn endpoint_config( /// Mints the endpoint's connection ids, steering prefix included. /// -/// noq-proto asks its generator for every id it issues, dials and rotations +/// moq-quic asks its generator for every id it issues, dials and rotations /// alike, so this is the one place the reuseport group's byte has to be /// stamped. #[derive(Debug)] @@ -112,9 +119,9 @@ struct Cids { shard: Option, } -impl moq_noq_proto::ConnectionIdGenerator for Cids { - fn generate_cid(&mut self) -> moq_noq_proto::ConnectionId { - moq_noq_proto::ConnectionId::new(&super::endpoint::cid(self.shard)) +impl moq_quic::ConnectionIdGenerator for Cids { + fn generate_cid(&mut self) -> moq_quic::ConnectionId { + moq_quic::ConnectionId::new(&super::endpoint::cid(self.shard)) } fn cid_len(&self) -> usize { @@ -127,7 +134,7 @@ impl moq_noq_proto::ConnectionIdGenerator for Cids { } /// Dial as `config` says. -pub(crate) fn client_config(config: &client::Config) -> Result { +pub(crate) fn client_config(config: &client::Config) -> Result { let provider = provider(); let builder = rustls::ClientConfig::builder_with_provider(provider.clone()) .with_protocol_versions(&[&rustls::version::TLS13]) @@ -170,21 +177,21 @@ pub(crate) fn client_config(config: &client::Config) -> Result Result { +pub(crate) fn server_config(config: &server::Config) -> Result { let crypto = QuicServerConfig::try_from(config.tls.clone()).map_err(|err| Error::Tls(err.to_string()))?; - let mut server = moq_noq_proto::ServerConfig::with_crypto(Arc::new(crypto)); + let mut server = moq_quic::ServerConfig::with_crypto(Arc::new(crypto)); let transport = transport_config(&config.transport)?; #[cfg(feature = "qlog")] - let transport = with_qlog(transport, &config.transport); + let transport = with_qlog(transport, &config.transport, qlog::Side::Server); server.transport_config(Arc::new(transport)); Ok(server) } @@ -233,53 +240,36 @@ pub(crate) fn identity_server_config( /// Attach the configured qlog sink, if any. /// -/// Noq asks a factory per connection, so each gets a file of its own. +/// moq-quic holds one sink per transport config, which is per endpoint on the +/// accept side and per dial on the connect side; its events carry the +/// connection's qlog `group_id` either way. #[cfg(feature = "qlog")] -fn with_qlog(mut transport: moq_noq_proto::TransportConfig, config: &Transport) -> moq_noq_proto::TransportConfig { +fn with_qlog( + mut transport: moq_quic::TransportConfig, + config: &Transport, + side: qlog::Side, +) -> moq_quic::TransportConfig { let Some(sink) = config.qlog.clone() else { return transport; }; - transport.qlog_factory(Arc::new(Traces { sink })); + let mut qlog = moq_quic::QlogConfig::default(); + qlog.writer(sink.trace(side)); + transport.qlog_stream(qlog.into_stream()); transport } -/// Opens one trace per connection, which is what noq-proto's factory hook is -/// for. -#[cfg(feature = "qlog")] -#[derive(Debug)] -struct Traces { - sink: qlog::Sink, -} - -#[cfg(feature = "qlog")] -impl moq_noq_proto::QlogFactory for Traces { - fn for_connection( - &self, - side: moq_noq_proto::Side, - _remote: std::net::SocketAddr, - initial_dst_cid: moq_noq_proto::ConnectionId, - _now: std::time::Instant, - ) -> Option { - let side = match side { - moq_noq_proto::Side::Client => qlog::Side::Client, - moq_noq_proto::Side::Server => qlog::Side::Server, - }; - Some(moq_noq_proto::QlogConfig::new(self.sink.trace(&initial_dst_cid, side))) - } -} - /// The per-connection knobs both roles share. -fn transport_config(config: &Transport) -> Result { - use moq_noq_proto::VarInt; +fn transport_config(config: &Transport) -> Result { + use moq_quic::VarInt; - let idle = moq_noq_proto::IdleTimeout::try_from(config.idle_timeout) + let idle = moq_quic::IdleTimeout::try_from(config.idle_timeout) .map_err(|_| Error::Quic(format!("idle timeout out of range: {:?}", config.idle_timeout)))?; let streams = VarInt::from_u64(config.max_streams) .map_err(|_| Error::Quic(format!("stream limit out of range: {}", config.max_streams)))?; - let mut transport = moq_noq_proto::TransportConfig::default(); + let mut transport = moq_quic::TransportConfig::default(); transport.max_idle_timeout(Some(idle)); transport.keep_alive_interval(config.keep_alive); transport.max_concurrent_bidi_streams(streams); @@ -301,22 +291,22 @@ fn transport_config(config: &Transport) -> Result Arc::new(moq_noq_proto::congestion::CubicConfig::default()) - as Arc, - Congestion::Delay => Arc::new(moq_noq_proto::congestion::Bbr3Config::default()), + Congestion::Loss => Arc::new(moq_quic::congestion::CubicConfig::default()) + as Arc, + Congestion::Delay => Arc::new(moq_quic::congestion::Bbr3Config::default()), }); Ok(transport) } /// A receive window on the wire is a QUIC varint. One that does not fit cannot /// be applied, so it is an error rather than a smaller window than was asked. -fn varint_window(name: &str, value: u64) -> Result { - moq_noq_proto::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) +fn varint_window(name: &str, value: u64) -> Result { + moq_quic::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) } /// ALPN protocols on the wire, which is a length-prefixed list of byte @@ -410,10 +400,10 @@ impl rustls::client::danger::ServerCertVerifier for NoVerify { mod tests { use super::{CONNECTION_WINDOW, STREAM_WINDOW, Transport, transport_config}; - /// noq keeps the windows private, but its `Debug` prints them. The leading + /// moq-quic keeps the windows private, but its `Debug` prints them. The leading /// space keeps `receive_window` from matching `stream_receive_window`, and /// the trailing comma keeps a shorter value from matching a longer one. - fn has_window(config: &moq_noq_proto::TransportConfig, name: &str, value: u64) -> bool { + fn has_window(config: &moq_quic::TransportConfig, name: &str, value: u64) -> bool { format!("{config:?}").contains(&format!(" {name}: {value},")) } diff --git a/rs/moq-uring/src/quic/noq/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs similarity index 90% rename from rs/moq-uring/src/quic/noq/stream.rs rename to rs/moq-uring/src/quic/driver/stream.rs index bb9d90110e..e415f57f30 100644 --- a/rs/moq-uring/src/quic/noq/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -1,6 +1,6 @@ -//! Stream handles: thin, direct calls into the shared noq-proto connection. +//! Stream handles: thin, direct calls into the shared moq-quic connection. //! -//! Single-threaded sans-IO means a write goes straight into noq's send +//! Single-threaded sans-IO means a write goes straight into moq-quic's send //! queue (no staging copy) and a read comes straight out of its reassembly //! buffer; the handles just kick the driver so egress reaches the wire and //! park on the per-stream waiter lists the driver wakes. @@ -8,7 +8,7 @@ use std::task::{Context, Poll}; use bytes::{Buf, Bytes, BytesMut}; -use moq_noq_proto::{StreamId, VarInt}; +use moq_quic::{StreamId, VarInt}; use super::super::Error; use super::{End, Shared}; @@ -51,7 +51,7 @@ impl SendStream { self.fin || self.reset } - /// Queue as much of `buf` as noq will take right now, without parking. + /// Queue as much of `buf` as moq-quic will take right now, without parking. /// Best-effort, for the close path where nobody is left to poll. pub(crate) fn try_write(&mut self, buf: &[u8]) -> usize { if self.fin || self.reset { @@ -105,14 +105,14 @@ impl web_transport_trait::poll::SendStream for SendStream { self.shared.kick(); Poll::Ready(Ok(n)) } - // No capacity right now; the driver wakes us when noq reports + // No capacity right now; the driver wakes us when moq-quic reports // the stream writable. - Err(moq_noq_proto::WriteError::Blocked) => { + Err(moq_quic::WriteError::Blocked) => { self.shared.park_writable(self.id, waiter); Poll::Pending } - Err(moq_noq_proto::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), - Err(moq_noq_proto::WriteError::ClosedStream) => { + Err(moq_quic::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), + Err(moq_quic::WriteError::ClosedStream) => { Poll::Ready(Err(Error::Quic("stream already finished".to_string()))) } } @@ -120,7 +120,7 @@ impl web_transport_trait::poll::SendStream for SendStream { fn set_priority(&mut self, order: i32) { // The trait (like W3C sendOrder) sends HIGHER values first, and so - // does noq. + // does moq-quic. let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); } @@ -133,12 +133,12 @@ impl web_transport_trait::poll::SendStream for SendStream { // A STOP_SENDING beat us here. Carry the code like `poll_write` // does, or `moq_net::Error::from_transport` cannot decode a // routine cancellation. - Err(moq_noq_proto::FinishError::Stopped(code)) => { + Err(moq_quic::FinishError::Stopped(code)) => { self.reset = true; return Err(Error::Stop(code.into_inner())); } // Already finished or reset, so the FIN it wanted is out. - Err(moq_noq_proto::FinishError::ClosedStream) => {} + Err(moq_quic::FinishError::ClosedStream) => {} } self.fin = true; self.shared.kick(); @@ -154,7 +154,7 @@ impl web_transport_trait::poll::SendStream for SendStream { if self.reset { return Poll::Ready(Ok(())); } - // noq reports a send stream's end as an event, which the driver + // moq-quic reports a send stream's end as an event, which the driver // records: an acknowledged FIN, or the peer's STOP_SENDING. match self.shared.ended(self.id) { Some(End::Stopped(code)) => Poll::Ready(Err(Error::Stop(code))), @@ -199,7 +199,7 @@ const READ_AHEAD: usize = 64 * 1024; /// How much to ask for per read-ahead chunk. const READ_CHUNK: usize = 8 * 1024; -/// One read out of noq's reassembly buffer. +/// One read out of moq-quic's reassembly buffer. enum Read { /// Bytes, at most as many as were asked for. Chunk(Bytes), @@ -216,12 +216,12 @@ pub struct RecvStream { shared: Shared, id: StreamId, park: kio::Park, - /// Every byte up to the FIN was read out of noq; reads report the end + /// Every byte up to the FIN was read out of moq-quic; reads report the end /// once `backlog` is drained too. finished: bool, /// We stopped the stream; no more reads matter. stopped: bool, - /// Bytes `poll_closed` read ahead, handed to `poll_read` before noq's. + /// Bytes `poll_closed` read ahead, handed to `poll_read` before moq-quic's. backlog: BytesMut, } @@ -261,7 +261,7 @@ impl RecvStream { self.shared.kick(); } - /// Take up to `max` bytes out of noq's reassembly buffer. + /// Take up to `max` bytes out of moq-quic's reassembly buffer. /// /// Reading is what returns the peer's flow control credit, so a read that /// owes it a frame kicks the driver. @@ -279,8 +279,8 @@ impl RecvStream { let read = match chunks.next(max) { Ok(Some(chunk)) => Read::Chunk(chunk.bytes), Ok(None) => Read::Finished, - Err(moq_noq_proto::ReadError::Blocked) => Read::Blocked, - Err(moq_noq_proto::ReadError::Reset(code)) => Read::Reset(code.into_inner()), + Err(moq_quic::ReadError::Blocked) => Read::Blocked, + Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), }; let transmit = chunks.finalize().should_transmit(); drop(conn); @@ -351,7 +351,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { if self.finished || self.stopped { return Poll::Ready(Ok(())); } - // The FIN sits behind whatever the peer sent before it, and noq only + // The FIN sits behind whatever the peer sent before it, and moq-quic only // reports the stream finished once that is read out. Waiting on // readability alone would park behind bytes nobody is reading, so read // ahead into the backlog `poll_read` serves first: this watch resolves diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index a21d821425..a312ab9584 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -25,7 +25,7 @@ use super::server; -pub use super::noq::Endpoint; +pub use super::driver::Endpoint; /// Every connection id this endpoint issues is this long: long enough to be /// unguessable per socket, and fixed because a short header does not encode diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index 9ed2b81116..f6e84c8959 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -18,7 +18,7 @@ //! [`web`] layer's HTTP/3 CONNECT handshake on top of the same adapter, with //! [`web::Session`] as the one transport type covering both. //! -//! Noq is the sans-IO QUIC stack underneath the worker. +//! moq-quic is the sans-IO QUIC stack underneath the worker. pub mod client; pub mod endpoint; @@ -28,10 +28,10 @@ pub mod qlog; pub mod server; pub mod web; -mod noq; +mod driver; +pub use driver::{Connection, RecvStream, SendStream}; pub use endpoint::Endpoint; -pub use noq::{Connection, RecvStream, SendStream}; /// The QUIC payload size every full datagram in a GSO train uses, and the /// stride GRO coalesces with. @@ -77,7 +77,7 @@ impl Identity { client_auth: server::ClientAuth, ) -> Result, Error> { let alpn = alpn.iter().map(|proto| proto.as_ref().as_bytes().to_vec()).collect(); - noq::identity_server_config(self, alpn, &client_auth) + driver::identity_server_config(self, alpn, &client_auth) } /// The PEM certificate chain being presented, for a caller that publishes @@ -135,8 +135,9 @@ pub struct Transport { pub keep_alive: Option, /// Where to write qlog traces, or `None` (the default) to write none. /// - /// Noq takes one sink per configuration and writes one file per connection, - /// named from that connection's Initial destination connection ID. + /// moq-quic takes one sink per transport config, so this writes one file + /// per endpoint on the accept side and per dial on the connect side, each + /// event tagged with the qlog `group_id` of the connection it belongs to. /// /// Only compiled with the `qlog` feature, so a build without it cannot ask /// for traces the backends would not produce. @@ -162,9 +163,9 @@ impl Default for Transport { /// The congestion control family a connection runs. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. /// -/// The default is [`Loss`](Self::Loss), which is what noq runs +/// The default is [`Loss`](Self::Loss), which is what moq-quic runs /// unasked. An application carrying live media wants [`Delay`](Self::Delay) /// and should say so; the relay does. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/rs/moq-uring/src/quic/qlog.rs b/rs/moq-uring/src/quic/qlog.rs index 0fe2d8250c..e3e412763c 100644 --- a/rs/moq-uring/src/quic/qlog.rs +++ b/rs/moq-uring/src/quic/qlog.rs @@ -8,7 +8,7 @@ //! the worker would stall every connection sharing its core for as long as //! the filesystem takes. //! -//! One file per connection, in the qlog JSON-SEQ format the tokio listener already +//! One file per trace, in the qlog JSON-SEQ format the tokio listener already //! writes, so an existing workflow reads both without knowing which runtime //! produced them. @@ -117,23 +117,14 @@ impl Sink { }) } - /// A writer for one connection's trace, named from its connection id. - pub(crate) fn trace(&self, cid: &[u8], side: Side) -> Box { - self.open(cid, side) - } - - fn open(&self, cid: &[u8], side: Side) -> Box { - use std::fmt::Write as _; - + /// A writer for one trace, which every connection sharing a transport + /// config writes into, each event tagged with its connection's qlog + /// `group_id`. + pub(crate) fn trace(&self, side: Side) -> Box { let inner = self.inner.clone(); + // The trace slot keeps each file unique within the sink. let file = inner.next.fetch_add(1, Ordering::Relaxed); - // The trace slot stays unique within the sink even when a peer reuses - // the same Initial destination connection id. - let cid = cid.iter().fold(String::with_capacity(cid.len() * 2), |mut id, byte| { - let _ = write!(id, "{byte:02x}"); - id - }); - let id = format!("connection{file}-{cid}"); + let id = format!("trace{file}"); let path = inner.dir.join(format!( "moq-{}-{}-{}-{id}-{}.qlog", inner.started, @@ -369,8 +360,8 @@ mod tests { let first = Sink::directory(dir.path()).expect("first sink"); let second = Sink::directory(dir.path()).expect("second sink"); - let mut first_trace = first.trace(b"first", Side::Server); - let mut second_trace = second.trace(b"second", Side::Server); + let mut first_trace = first.trace(Side::Server); + let mut second_trace = second.trace(Side::Server); first_trace.write_all(b"first").expect("write first trace"); second_trace.write_all(b"second").expect("write second trace"); drop(first_trace); @@ -387,14 +378,14 @@ mod tests { } #[test] - fn reused_connection_ids_use_distinct_files() { + fn traces_use_distinct_files() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let mut first = sink.trace(b"same", Side::Server); + let mut first = sink.trace(Side::Server); first.write_all(b"first").expect("write first trace"); drop(first); - let mut second = sink.trace(b"same", Side::Server); + let mut second = sink.trace(Side::Server); second.write_all(b"second").expect("write second trace"); drop(second); drop(sink); @@ -411,7 +402,7 @@ mod tests { fn an_open_trace_accounts_for_staging_and_close() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let trace = sink.trace(b"accounting", Side::Server); + let trace = sink.trace(Side::Server); let accounted = CHUNK + std::mem::size_of::(); for _ in 0..10_000 { @@ -433,7 +424,7 @@ mod tests { let held = MEMORY_MAX - CHUNK + 1; assert!(sink.inner.reserve(held)); - let mut trace = sink.trace(b"dropped", Side::Server); + let mut trace = sink.trace(Side::Server); trace.write_all(b"dropped").expect("write dropped trace"); drop(trace); sink.inner.release(held); diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 915724d195..192592f9b2 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -1012,7 +1012,7 @@ impl std::fmt::Debug for RecvStream { /// Unmap a raw-QUIC stream code that arrived in the HTTP/3 code space. /// -/// Older `web-transport-moq` raw sessions map RESET_STREAM and STOP_SENDING +/// Older moq-tokio raw sessions map RESET_STREAM and STOP_SENDING /// codes as if they were WebTransport. No `u32` application code reaches that /// range, so a code in it can only be the mapped form. Anything else is the /// peer's own code and passes through. diff --git a/rs/moq-uring/tests/echo.rs b/rs/moq-uring/tests/echo.rs index f50825b136..2f2fc623d9 100644 --- a/rs/moq-uring/tests/echo.rs +++ b/rs/moq-uring/tests/echo.rs @@ -1,6 +1,6 @@ -//! A tokio noq peer echoing through the io_uring WebTransport server. +//! A tokio QUIC peer echoing through the io_uring WebTransport server. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -48,7 +48,7 @@ async fn write_finish(send: &mut quic::web::SendStream, mut buf: &[u8]) { } #[test] -fn echo_noq_peer() { +fn echo_tokio_peer() { let Some(mut worker) = worker() else { return }; let handle = worker.handle(); let certs = support::certs().expect("certificates"); @@ -56,7 +56,7 @@ fn echo_noq_peer() { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -78,11 +78,11 @@ fn echo_noq_peer() { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); diff --git a/rs/moq-uring/tests/ecn.rs b/rs/moq-uring/tests/ecn.rs index 5f2c311796..1f89c4e7a7 100644 --- a/rs/moq-uring/tests/ecn.rs +++ b/rs/moq-uring/tests/ecn.rs @@ -2,7 +2,7 @@ //! the peer's `Packet::ecn` reads back, over IPv4 and IPv6, as a lone //! datagram and as a GSO train, with CE surviving too. Without the TOS and //! traffic-class control messages the peer sees no mark, its ACKs carry no -//! ECN counts, and noq disables ECN on the first one. +//! ECN counts, and moq-quic disables ECN on the first one. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. diff --git a/rs/moq-uring/tests/endpoint.rs b/rs/moq-uring/tests/endpoint.rs index 899bb2cd9b..a6e41b28e6 100644 --- a/rs/moq-uring/tests/endpoint.rs +++ b/rs/moq-uring/tests/endpoint.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -200,7 +200,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { // Junk first: it must not be fatal, and whatever answer it draws // must be smaller than it was, so the socket is never an - // amplifier. Noq answers a packet naming no connection with a + // amplifier. moq-quic answers a packet naming no connection with a // stateless reset, which is what tells a // peer holding stale state to give up. let junk = [0u8; 64]; @@ -276,7 +276,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { assert!( versions .iter() - .any(|version| moq_noq_proto::DEFAULT_SUPPORTED_VERSIONS.contains(version)), + .any(|version| moq_quic::DEFAULT_SUPPORTED_VERSIONS.contains(version)), "the supported version is offered: {versions:?}" ); drop((endpoint, dial_only)); @@ -382,7 +382,7 @@ fn an_unanswered_dial_times_out() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The dial ends at noq's handshake idle timeout (10s), not the idle timeout. + // The dial ends at moq-quic's handshake idle timeout (10s), not the idle timeout. let dial = dial_config(hole.local_addr().expect("addr")); let result = worker @@ -419,7 +419,7 @@ fn the_backlog_bounds_pending_handshakes() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The server never answers, so the dial ends at noq's handshake idle + // The server never answers, so the dial ends at moq-quic's handshake idle // timeout (10s). let dial = dial_config(server); diff --git a/rs/moq-uring/tests/identity.rs b/rs/moq-uring/tests/identity.rs index 4a1673e2bc..18bf14da98 100644 --- a/rs/moq-uring/tests/identity.rs +++ b/rs/moq-uring/tests/identity.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/qlog.rs b/rs/moq-uring/tests/qlog.rs index 9a1ff28ef6..c732b74bb5 100644 --- a/rs/moq-uring/tests/qlog.rs +++ b/rs/moq-uring/tests/qlog.rs @@ -2,14 +2,12 @@ //! tooling can read. //! //! The point of the test is the two halves the relay depends on: the sink -//! actually produces a file for the connection (per connection on noq and -//! per connection on noq, and -//! every record in it is JSON, so a reader is not handed a truncated or +//! actually produces a file for each end of the connection, and every record in it is JSON, so a reader is not handed a truncated or //! interleaved trace. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "qlog", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "qlog", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/session.rs b/rs/moq-uring/tests/session.rs index 22fef4fb5d..5571c8e587 100644 --- a/rs/moq-uring/tests/session.rs +++ b/rs/moq-uring/tests/session.rs @@ -8,7 +8,7 @@ //! Origin drivers run through Tokio's explicit-time adapter while session //! drivers use the worker's clock and timer. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -308,7 +308,7 @@ fn two_lite_sessions_share_the_server_socket() { } /// The client verifies for real, trusting only the certificate the server -/// presents: nothing handshakes unless the configured roots reach noq. +/// presents: nothing handshakes unless the configured roots reach moq-quic. #[test] fn configured_roots_verify_the_server() { let Some(mut worker) = worker() else { return }; diff --git a/rs/moq-uring/tests/teardown.rs b/rs/moq-uring/tests/teardown.rs index ad1383a5e9..26a8993b95 100644 --- a/rs/moq-uring/tests/teardown.rs +++ b/rs/moq-uring/tests/teardown.rs @@ -10,7 +10,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index 0cfef78d1c..2fbf2157ba 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -1,11 +1,11 @@ -//! WebTransport interop: the reference tokio stack (`web-transport-moq`, +//! WebTransport interop: the reference tokio stack (`moq_tokio::quic::web`, //! what browsers interop with) dials the uring server. One test hand-drives //! streams and datagrams through the H3 framing; the other runs a whole //! moq-lite session over it, which is exactly the browser-to-relay path. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -53,9 +53,9 @@ fn h3_endpoint(handle: &moq_uring::Handle, certs: &support::Certs) -> quic::Endp } /// The tokio-side client, in its own runtime on its own thread. -fn noq_client( +fn tokio_client( url: String, - body: impl FnOnce(web_transport_moq::Session) -> ClientFuture + Send + 'static, + body: impl FnOnce(moq_tokio::quic::web::Session) -> ClientFuture + Send + 'static, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { // May already be installed by a sibling test; either way one exists. @@ -65,12 +65,12 @@ fn noq_client( .build() .expect("tokio runtime"); rt.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = - web_transport_moq::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")).with_protocol(PROTO); + let request = moq_tokio::quic::web::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")) + .with_protocol(PROTO); let session = client.connect(request).await.expect("connect"); body(session).await; }); @@ -79,8 +79,8 @@ fn noq_client( type ClientFuture = std::pin::Pin + Send>>; -/// Read a noq-side stream to its end. -async fn read_all(recv: &mut web_transport_moq::RecvStream) -> Vec { +/// Read a tokio-side stream to its end. +async fn read_all(recv: &mut moq_tokio::quic::web::RecvStream) -> Vec { let mut out = Vec::new(); let mut buf = [0u8; 4096]; while let Some(n) = recv.read(&mut buf).await.expect("read") { @@ -126,7 +126,7 @@ fn webtransport_echo_end_to_end() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/echo?token=abc"), |session| { + let client = tokio_client(format!("https://{addr}/echo?token=abc"), |session| { Box::pin(async move { // Bidirectional echo. let (mut send, mut recv) = session.open_bi().await.expect("open_bi"); @@ -239,7 +239,7 @@ fn lite_session_over_webtransport() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/"), |session| { + let client = tokio_client(format!("https://{addr}/"), |session| { Box::pin(async move { assert_eq!(session.protocol(), Some(PROTO), "negotiated subprotocol"); let (sub_origin, sub_driver) = origin::Producer::new(origin::Config::default()); @@ -339,7 +339,7 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future std::thread::JoinHandle { +fn tokio_client_err(url: String) -> std::thread::JoinHandle { std::thread::spawn(move || { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let rt = tokio::runtime::Builder::new_current_thread() @@ -347,12 +347,12 @@ fn noq_client_err(url: String) -> std::thread::JoinHandle break err, } }; - assert!(matches!(err, web_transport_moq::WriteError::Stopped(0)), "got {err:?}"); + assert!( + matches!(err, moq_tokio::quic::web::WriteError::Stopped(0)), + "got {err:?}" + ); session.close(CLOSE_CODE, CLOSE_REASON.as_bytes()); session.closed().await; diff --git a/rs/moq-uring/tests/workers.rs b/rs/moq-uring/tests/workers.rs index d832d4c0a4..e427de24b4 100644 --- a/rs/moq-uring/tests/workers.rs +++ b/rs/moq-uring/tests/workers.rs @@ -9,7 +9,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index 8c6e634b14..3724bfd794 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -128,7 +128,7 @@ def test_media_keeps_dependency_contracts_and_vpx_tests(self): CARGO_STUB = r"""#!/bin/bash printf '%s\n' "$*" >> "$FEATURE_CALLS" -if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features noq' ]]; then +if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features quic' ]]; then echo 'a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' >&2 exit 1 fi diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh index 8727b4a233..555ef55272 100755 --- a/sh/rs/tokio-features.sh +++ b/sh/rs/tokio-features.sh @@ -19,8 +19,8 @@ cargo clippy --locked -p moq-tokio --all-features -- -D warnings # Only a feature-owner edit needs every isolated permutation on a PR. Shared # dependencies still compile at both extremes; Nightly runs every permutation. if ((all)); then - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,aws-lc-rs -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,ring -- -D warnings for feature in aws-lc-rs ring tcp uds websocket; do cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings @@ -29,12 +29,12 @@ fi # shellcheck disable=SC2016 # the backticks are literal, as rustc prints them want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' -if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 +if output=$(cargo check --locked -p moq-tokio --no-default-features --features quic 2>&1); then + echo "rs: moq-tokio with only quic should reject its missing crypto provider" >&2 exit 1 fi if ! grep -qF "$want" <<<"$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "rs: moq-tokio with only quic should fail on: $want" >&2 echo "$output" | grep -E '^error' >&2 || true exit 1 fi From 282197e2846d2c8f3fd20b4ae5b23c09c9e6aa36 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:46:50 -0700 Subject: [PATCH 04/37] claim quest/m1/quic/scheduler Co-Authored-By: Claude Opus 5.5 From 866b13d62b58895086916ae116c4d9c958eaf04a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:42 -0700 Subject: [PATCH 05/37] fix(moq-tokio): log a CONNECT stream cut off by the close at debug The capsule reader maps a lost stream to UnexpectedEnd, so every ordinary session close warned "failed to read capsule". A read error while the connection is closed is now debug; one on a live connection still warns. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/web/session.rs | 49 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index ad0fc8c2d6..38e01d6684 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -154,7 +154,7 @@ impl Session { recv: crate::quic::driver::RecvStream, close: Arc, ) { - let close_info = Self::read_capsules(recv).await; + let close_info = Self::read_capsules(&conn, recv).await; let code = close_info.as_ref().map_or(0, |(c, _)| *c); let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); @@ -182,7 +182,10 @@ impl Session { // Keep reading capsules from the CONNECT recv stream until it's closed. // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, // or None if the stream closed without a capsule. - async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + async fn read_capsules( + conn: &crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + ) -> Option<(u32, String)> { let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); loop { match reader.read().await { @@ -194,8 +197,13 @@ impl Session { tracing::warn!(%typ, size = payload.len(), "unknown capsule"); } Ok(None) => return None, + // The capsule reader reports a lost stream as `UnexpectedEnd`, so ask the + // connection: a read cut short by its close is the ordinary teardown. Err(e) => { - tracing::warn!(?e, "failed to read capsule"); + match conn.close_reason() { + Some(reason) => tracing::debug!(?e, %reason, "CONNECT stream ended with the connection"), + None => tracing::warn!(?e, "failed to read capsule"), + } return None; } } @@ -1223,6 +1231,41 @@ mod tests { ); } + /// A CONNECT stream cut off by the connection's close is ordinary teardown, not a warning. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_ends_quietly_with_connection() { + let (client, server, _endpoints) = pair().await; + + // A zero-length reserved frame, so the server sees the stream without a capsule. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x21, 0x00]).await.unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!(!logs_contain("failed to read capsule"), "an ordinary close warned"); + } + + /// A CONNECT stream that ends mid-frame on a live connection still warns. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_truncated_warns() { + let (client, server, _endpoints) = pair().await; + + // A DATA frame that promises 5 bytes and carries 1. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x00, 0x05, 0x01]).await.unwrap(); + send.finish().unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!( + logs_contain("failed to read capsule"), + "a truncated capsule stream was not reported" + ); + } + /// A uni stream reset before its header arrived reports the reset, not a foreign session. #[tokio::test] async fn uni_reset_before_header() { From 88a6fae64385c685bacc5af8d6a1bde783375932 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:46 -0700 Subject: [PATCH 06/37] docs: point fork and advisory guidance at moq-quic Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 7 +++---- rs/AGENTS.md | 2 +- rs/moq-quic/README.md | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbfcde1df5..d168b341ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,10 +74,9 @@ For non-trivial tasks, file an issue or offer to run `/quest-plan`. # Forks -[moq-dev/noq](https://github.com/moq-dev/noq) publishes `moq-noq*`, the QUIC stack every published MoQ crate builds on; iroh keeps upstream noq. -Its `moq-sync` workflow merges n0-computer/noq weekly as a PR; review it like any other, and `PARENT` names the upstream commit each release includes. -A carried change lists its upstream PR, or the reason it has none, in the fork PR. -For an advisory against noq or Quinn, compare the pinned release's `PARENT` with the fixing upstream commit, then sync, release the fork, and bump the pin here. +`rs/moq-quic` is our hard fork of quinn, the QUIC stack MoQ builds on; iroh keeps upstream noq. +It never merges upstream: fixes are cherry-picked by hand, and each carried change is listed in its README with its upstream PR, or the reason it has none. +For an advisory against Quinn, follow the advisory triage in [`rs/moq-quic/README.md`](rs/moq-quic/README.md). # Releases diff --git a/rs/AGENTS.md b/rs/AGENTS.md index d9bd16d6b4..52ff1bc226 100644 --- a/rs/AGENTS.md +++ b/rs/AGENTS.md @@ -5,7 +5,7 @@ The `/rs` Cargo workspace. Extends the root `AGENTS.md`. One crate per component from the root list, named `moq-` (`hang` is the exception). Keep them modular; a crate does one thing. Beyond that list: - `kio`: "easy async" primitives everything else polls through. -- `moq-tokio`: configures the QUIC backends (Quinn/Quiche/Noq/Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. +- `moq-tokio`: configures the QUIC backends (`moq-quic`, our quinn fork, and Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. - `moq-cli` builds the `moq` binary and owns the CLI surface for the gateway crates. Binaries never carry a `-cli` suffix. `moq-net`, `moq-mux`, `moq-relay` (including the config conventions every binary shares), and `moq-ffi` have their own `AGENTS.md`. diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index d47b7629b0..b31ec65aae 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -72,4 +72,4 @@ Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges `cargo audit` cannot match the renamed crate, so security fixes are tracked by hand. Watch quinn-rs/quinn's [security advisories](https://github.com/quinn-rs/quinn/security/advisories) and releases (including the `0.11.x` branch, which sometimes gets a fix `main` does not need). -For each quinn-proto or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. +For each quinn, quinn-proto, or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. From 2d5026751bea6cecf9e598ee73a0c9fb94a0f486 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:06:36 -0700 Subject: [PATCH 07/37] docs: format the upgrade note Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index e0e6502d99..2bd0678625 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -28,7 +28,7 @@ These land with the next breaking release, not the 2026-09-23 train. it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one - file per endpoint instead of one per connection; on io_uring workers, one per + file per endpoint instead of one per connection; on io\_uring workers, one per endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC From c2993ca71eeaed7bc3fa31d28832fddc96fc2a7b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:16:50 -0700 Subject: [PATCH 08/37] quest: finish quest/m1/quic/fork/switch Relay peak RSS on the fanout workloads matches moq-noq on origin/main, which was the last open part of the Goal. Co-Authored-By: Claude Opus 5.5 --- quest/m1/p2p/native-quic.md | 5 +-- quest/m1/p2p/udp-demux.md | 4 +-- quest/m1/quic/fork/README.md | 10 ++---- quest/m1/quic/fork/switch.md | 58 -------------------------------- quest/m1/quic/receive-credit.md | 4 --- quest/m1/time/README.md | 5 ++- quest/m1/time/relay-seam.md | 1 - quest/m1/time/uring.md | 1 - quest/m2/bbr-ack-cleanup.md | 1 - quest/m2/quic-bbr-loss-parity.md | 4 --- quest/m2/quic-sim.md | 3 +- 11 files changed, 8 insertions(+), 88 deletions(-) delete mode 100644 quest/m1/quic/fork/switch.md diff --git a/quest/m1/p2p/native-quic.md b/quest/m1/p2p/native-quic.md index 912de3082f..81626f6fd3 100644 --- a/quest/m1/p2p/native-quic.md +++ b/quest/m1/p2p/native-quic.md @@ -21,9 +21,7 @@ Decided with the maintainer on 2026-10-13: candidates through the module [webrtc](/quest/m1/p2p/webrtc.md) lifts out of `moq-rtc`, with host candidates, `.local` resolution, and the STUN Binding client; don't copy it. -- QUIC is `moq-quic`, fed datagrams from the nominated pair, so this waits - on [the switch](/quest/m1/quic/fork/switch.md) instead of building on - `moq-noq` and porting it. +- QUIC is `moq-quic`, fed datagrams from the nominated pair. - Authentication: each side generates a self-signed certificate and sends its SHA-256 fingerprint in its offer or answer, as DTLS fingerprints ride in SDP. Both sides pin it with the existing `tls.fingerprint` check. MoQ @@ -61,7 +59,6 @@ covers LAN and NAT pairs between two `moq-cli --p2p` nodes. ## Required -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - `moq-quic` replaces `moq-noq` in moq-tokio - [UDP demux](/quest/m1/p2p/udp-demux.md) - the classifier that splits STUN from QUIC - [Native data channel transport](/quest/m1/p2p/webrtc.md) - the shared ICE gathering module - [Signaling and policy](/quest/m1/p2p/signal.md) - the offer schema and roster this extends diff --git a/quest/m1/p2p/udp-demux.md b/quest/m1/p2p/udp-demux.md index f3d97c8776..01bcd15e71 100644 --- a/quest/m1/p2p/udp-demux.md +++ b/quest/m1/p2p/udp-demux.md @@ -32,9 +32,7 @@ one. Bound queues per stack with a per-source drop rather than unbounded growth; report drops as a counter. QUIC: the server takes the virtual socket through whatever the QUIC stack -accepts: noq's `new_with_abstract_socket` today, `moq-quic`'s fed datagrams -after [the switch](/quest/m1/quic/fork/switch.md). Build on whichever has -landed rather than both. QUIC-bit greasing is already off, from +accepts: the driver's `new_with_abstract_socket` on `moq-quic`. QUIC-bit greasing is already off, from [shard steering](/quest/m1/p2p/shard-steering.md). STUN: an ICE connectivity check is a Binding request carrying USERNAME, so diff --git a/quest/m1/quic/fork/README.md b/quest/m1/quic/fork/README.md index 65e1f27219..091f1bafe6 100644 --- a/quest/m1/quic/fork/README.md +++ b/quest/m1/quic/fork/README.md @@ -1,4 +1,4 @@ -# Hard-fork quinn into the monorepo as moq-quic +# [XS] Hard-fork quinn into the monorepo as moq-quic ## Goal @@ -55,12 +55,8 @@ them: This README's own work: delete moq-dev/noq's `moq-sync.yml` and mark its README frozen. Advisory triage is documented in `rs/moq-quic/README.md`. Every carried change in moq-dev/noq's `CHANGELOG-MOQ.md` -is either ported by a child quest or recorded as not applicable in the -[switch](/quest/m1/quic/fork/switch.md) PR. - -## Required - -- [Switch](/quest/m1/quic/fork/switch.md) - quinn's async layer and `web-transport-moq` join moq-tokio, both runtimes run on `moq-quic`, `moq-noq*` is gone, and relay memory matches `moq-noq` +is either ported by a child quest or recorded as not applicable in +[#5664](https://github.com/moq-dev/moq/pull/5664). ## Related diff --git a/quest/m1/quic/fork/switch.md b/quest/m1/quic/fork/switch.md deleted file mode 100644 index 32e547d53e..0000000000 --- a/quest/m1/quic/fork/switch.md +++ /dev/null @@ -1,58 +0,0 @@ -# [L] Switch MoQ onto moq-quic - -## Goal - -quinn's async layer and `web-transport-moq` live in moq-tokio as modules, -moq-tokio and moq-uring run on `moq-quic`, and no workspace crate depends on -`moq-noq*`. Upstream `noq-proto` remains only for the `iroh` feature. Relay -memory on the bulk and fanout workloads matches `moq-noq`. - -## Plan - -Decided 2026-10-06: `moq-quic` (quinn-proto) is the only crate the fork adds. -quinn's async layer is imported into moq-tokio as a module rather than a -`moq-quic-tokio` crate, tokio-only: drop quinn's `Runtime` abstraction and -its smol and async-io implementations instead of carrying them. Its UDP -sockets come from `moq_sock::udp`. -Keep the first commit verbatim from upstream (quinn's `quinn/src`) so a -reviewer can diff it, and extend the cherry-pick recipe in -`rs/moq-quic/README.md` to map `quinn/src/` onto the module. - -Move `web-transport-moq` from moq-dev/noq into moq-tokio as a module too. It -derives from web-transport-quinn; its noq-only parts are `PathId::ZERO` and -`path_stats`. Report bandwidth from quinn's `PathStats::bandwidth_estimate` -instead of the current cwnd/rtt guess; it is in bytes per second and -`web-transport-trait` wants bits. moq-uring uses it only in tests, -through moq-tokio. - -BBR3 is already `moq-quic`'s default, with `Bbr3Config` and `Bbr3` named as -in noq, so the `Delay` family maps over unchanged. Run the benchmark matrix -against `moq-noq` 1.3.x on the same workloads and report any throughput or -latency difference; the BBR3 port could not, with no relay on `moq-quic` yet. - -moq-uring built against both quinn-proto and noq-proto until #3811, behind -about 20 `cfg` lines (stats fields, CID generator, qlog, BBR). Rename -`rs/moq-uring/src/quic/noq` by role and use that history as the map. Keep -moq-uring's `qlog` feature working on quinn's qlog. - -Rename the `noq` cargo features by role (`quic` is the recommendation); -confirm the name with the maintainer in the PR. Update every doc and example -that names noq, and run `just test interop --all`. - -Decided 2026-10-10: `moq-quic` takes `moq_clock::Instant` at its boundary in -place of `std::time::Instant`, so the sim can drive real QUIC later -([QUIC on the sim](/quest/m2/quic-sim.md)) and the io_uring worker's per-turn -sample passes straight through. `moq_clock::Instant` mirrors std's methods, so -alias it in `moq-quic` and keep the swap an import change; that keeps -upstream cherry-picks mechanical. - -In the PR, list each carried change from moq-dev/noq's `CHANGELOG-MOQ.md` as -ported (with its quest) or not applicable (with the reason). - -Lazy stream slots ([quinn#2601](https://github.com/quinn-rs/quinn/pull/2601), -the change noq took as noq#667) arrived with the import at quinn `7616e6b2`, -so only its measurement remains, and that needs the relay on `moq-quic`. -Re-run #3342's bulk and fanout relay memory workloads after the switch and -report them in the PR. On `moq-noq` they measured 75 MiB (bulk) and 31 MiB -(fanout), against 141 and 97 MiB without lazy slots; `moq-quic` should land -near the former. diff --git a/quest/m1/quic/receive-credit.md b/quest/m1/quic/receive-credit.md index c2bd474fbb..9f93925876 100644 --- a/quest/m1/quic/receive-credit.md +++ b/quest/m1/quic/receive-credit.md @@ -47,10 +47,6 @@ with its consumer. Wire: existing QUIC flow-control and cancellation frames; no new MoQ message or negotiation. Browser and iroh receivers do not gain a capability their transports cannot provide. -## Required - -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - both native adapters use the in-tree transport - ## Related - [Bound message assembly](/quest/m1/quic/receive-memory.md) - retains connection credit until a complete message is handed off diff --git a/quest/m1/time/README.md b/quest/m1/time/README.md index 9b070511dd..b4f507c457 100644 --- a/quest/m1/time/README.md +++ b/quest/m1/time/README.md @@ -85,9 +85,8 @@ reasons, so later sessions don't reopen them: [moq-uring on moq-clock](/quest/m1/time/uring.md). The relay seam [bench-relay](/quest/m2/bench-relay.md) needed is [the relay seam](/quest/m1/time/relay-seam.md). Quests that already asked - for "mocked time" Require the core quest. moq-quic adopts the logical - instant in the [switch](/quest/m1/quic/fork/switch.md), which Requires the - crate. + for "mocked time" Require the core quest. moq-quic takes the logical + instant at its boundary. This milestone verifies the exported clock contract and production consumers needed by the QUIC switch and clustering regressions. Unrelated test/private diff --git a/quest/m1/time/relay-seam.md b/quest/m1/time/relay-seam.md index 2bf68fdcb5..5a938ca285 100644 --- a/quest/m1/time/relay-seam.md +++ b/quest/m1/time/relay-seam.md @@ -33,4 +33,3 @@ Public API: the in-memory session and the relay seam. Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the sim and instant the relay tests run on -- [Switch](/quest/m1/quic/fork/switch.md) - moq-tokio's server types settle first diff --git a/quest/m1/time/uring.md b/quest/m1/time/uring.md index 6151101783..6987317e34 100644 --- a/quest/m1/time/uring.md +++ b/quest/m1/time/uring.md @@ -36,7 +36,6 @@ Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the driver's instant becomes moq-clock's -- [Switch](/quest/m1/quic/fork/switch.md) - the QUIC driver this edits moves onto `moq-quic` ## Closes diff --git a/quest/m2/bbr-ack-cleanup.md b/quest/m2/bbr-ack-cleanup.md index 4791c2bb46..00462c87cc 100644 --- a/quest/m2/bbr-ack-cleanup.md +++ b/quest/m2/bbr-ack-cleanup.md @@ -54,6 +54,5 @@ packet-lifetime comments inline; no new user guide is needed. ## Related -- [Switch](/quest/m1/quic/fork/switch.md) - not a blocker: the core and its BBR3 are already in `rs/moq-quic` - [Benchmark reporting](/bench/README.md) - reusable measurement guidance, not a prerequisite for this fix - [Loss sampling](/quest/m2/quic-bbr-loss-parity.md) - preserve packet metadata needed by the separate loss-sample repair; both edit `bbr3/mod.rs`, so sequence them diff --git a/quest/m2/quic-bbr-loss-parity.md b/quest/m2/quic-bbr-loss-parity.md index b936b04d1c..83639152a5 100644 --- a/quest/m2/quic-bbr-loss-parity.md +++ b/quest/m2/quic-bbr-loss-parity.md @@ -39,7 +39,3 @@ traffic yet. ## Required - [BBR ACK cleanup](/quest/m2/bbr-ack-cleanup.md) - edits the same `bbr3/mod.rs` packet bookkeeping and lands first; this keeps the metadata it preserves - -## Related - -- [Switch](/quest/m1/quic/fork/switch.md) - MoQ runs on `moq-quic`; not a blocker, since this is a local `rs/moq-quic` fix diff --git a/quest/m2/quic-sim.md b/quest/m2/quic-sim.md index 9e3c21b829..423cd102c0 100644 --- a/quest/m2/quic-sim.md +++ b/quest/m2/quic-sim.md @@ -11,8 +11,7 @@ real QUIC are tested deterministically with no sockets. Decided 2026-10-10: the in-memory session comes first ([the relay seam](/quest/m1/time/relay-seam.md)); simulated UDP follows. -`moq-quic` already takes `moq_clock::Instant` after the -[switch](/quest/m1/quic/fork/switch.md), so this is a datagram link plus a +`moq-quic` already takes `moq_clock::Instant`, so this is a datagram link plus a driver. Start from moq-shaper's in-memory `UdpSocket` (`rs/moq-shaper/src/mem.rs`). BBR's controller-level `Sim` in `bbr3/mod.rs` stays separate unless this replaces it cleanly. From e111abf07ff42f489961501f0642cfd510642d5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:50:23 -0700 Subject: [PATCH 09/37] test(moq-tokio): gate the timed helper with its only caller Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/client.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 66166b129b..70c9e0a905 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -787,6 +787,7 @@ mod tests { use super::*; /// A millisecond track: the frames written here carry timestamps. + #[cfg(all(feature = "websocket", feature = "quic"))] fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } From 188b08c30dd4d3bae1ab33dfb8f71ba8a60c572b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 22:46:44 -0700 Subject: [PATCH 10/37] fix(tokio): mark quic::web and quic::driver error enums non_exhaustive Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1 + rs/moq-tokio/src/quic/driver/recv_stream.rs | 4 ++++ rs/moq-tokio/src/quic/driver/send_stream.rs | 2 ++ rs/moq-tokio/src/quic/web/connect.rs | 1 + rs/moq-tokio/src/quic/web/error.rs | 8 ++++++++ rs/moq-tokio/src/quic/web/settings.rs | 1 + 6 files changed, 17 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 605163a673..e86cecef41 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -1422,6 +1422,7 @@ fn wake_all_notify(wakers: &mut FxHashMap>) { /// Errors that can arise when sending a datagram #[derive(Debug, Error, Clone, Eq, PartialEq)] +#[non_exhaustive] pub enum SendDatagramError { /// The peer does not support receiving datagram frames #[error("datagrams not supported by peer")] diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 027349ae12..2bf91f6231 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -506,6 +506,7 @@ impl ReadToEndBuffer { /// Errors from [`RecvStream::read_to_end`] #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadToEndError { /// An error occurred during reading #[error("read error: {0}")] @@ -558,6 +559,7 @@ impl Drop for RecvStream { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadError { /// The peer abandoned transmitting data on this stream /// @@ -618,6 +620,7 @@ impl From for io::Error { /// Errors that arise while waiting for a stream to be reset #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ResetError { /// The connection was lost #[error("connection lost")] @@ -691,6 +694,7 @@ impl Future for ReadExact<'_> { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadExactError { /// The stream finished before all bytes were read #[error("stream finished early ({0} bytes read)")] diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index e51af7cf25..812f2d4c12 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -358,6 +358,7 @@ impl Drop for SendStream { /// Errors that arise from writing to a stream #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum WriteError { /// The peer is no longer accepting data on this stream /// @@ -409,6 +410,7 @@ impl From for io::Error { /// Errors that arise while monitoring for a send stream stop from the peer #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum StoppedError { /// The connection was lost #[error("connection lost")] diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 471f5e416c..d49228d5f4 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -5,6 +5,7 @@ use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ConnectError { #[error("quic stream was closed early")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 5697bf684e..57d9eb1916 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -6,6 +6,7 @@ use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ClientError { #[error("unexpected end of stream")] UnexpectedEnd, @@ -39,6 +40,7 @@ pub enum ClientError { /// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum SessionError { #[error("connection error: {0}")] ConnectionError(crate::quic::driver::ConnectionError), @@ -140,6 +142,7 @@ impl CloseReason { /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WebTransportError { #[error("closed: code={0} reason={1}")] Closed(u32, String), @@ -156,6 +159,7 @@ pub enum WebTransportError { /// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WriteError { #[error("STOP_SENDING: {0}")] Stopped(u32), @@ -188,6 +192,7 @@ impl From for WriteError { /// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadError { #[error("session error: {0}")] SessionError(#[from] SessionError), @@ -223,6 +228,7 @@ impl From for ReadError { /// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadExactError { #[error("finished early")] FinishedEarly(usize), @@ -242,6 +248,7 @@ impl From for ReadExactError { /// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadToEndError { #[error("too long")] TooLong, @@ -272,6 +279,7 @@ impl From for ClosedStream { /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ServerError { #[error("unexpected end of stream")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index fdee95f23c..034c379ce7 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -3,6 +3,7 @@ use futures::try_join; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum SettingsError { #[error("quic stream was closed early")] UnexpectedEnd, From 5d21b4fe829af8857f114dcebb36767c7d288579 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:03:33 -0700 Subject: [PATCH 11/37] fix(tokio): count in-flight accepts in wait_all_draining Endpoint::close now refuses attempts still queued for accept, and wait_all_draining waits for accepts partway through Incoming::accept, so a server shutdown cannot release the socket before those peers hear back. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 60 +++++++++++++++++--- rs/moq-tokio/src/quic/driver/tests.rs | 70 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 8 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2bae9031dc..9093082720 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -292,7 +292,7 @@ impl Endpoint { /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); - let mut endpoint = self.inner.state.lock().unwrap(); + let endpoint = &mut *self.inner.state.lock().unwrap(); endpoint.recv_state.connections.close = Some((error_code, reason.clone())); for sender in endpoint.recv_state.connections.senders.values() { // Ignoring errors from dropped connections @@ -301,7 +301,19 @@ impl Endpoint { reason: reason.clone(), }); } + // Refuse queued attempts now, the same as ones arriving after the close, so the peer + // hears back before a caller of `wait_all_draining` releases the socket. + let mut response_buffer = Vec::new(); + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.stats.refused_handshakes += 1; + response_buffer.clear(); + let transmit = endpoint.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut endpoint.sender); + } self.inner.shared.incoming.notify_waiters(); + if endpoint.all_draining() { + self.inner.shared.all_draining.notify_waiters(); + } } /// Wait for all connections on the endpoint to be cleanly shut down @@ -331,20 +343,22 @@ impl Endpoint { /// Wait until no connection on the endpoint is still active /// /// A connection stops being active once its peer's close arrives, or once its closing - /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out - /// the closing period for a peer that already has the close, nor wait for the + /// period ends if the peer never answers. An attempt queued for [`accept()`] or partway + /// through [`Incoming::accept`] still counts as active. Unlike [`wait_idle()`], this does + /// not sit out the closing period for a peer that already has the close, nor wait for the /// [`Connection`](crate::quic::driver::Connection) handles to drop. /// /// Does not proactively close existing connections or cause incoming connections to be /// rejected. Consider calling [`close()`] if that is desired. /// + /// [`accept()`]: Endpoint::accept /// [`close()`]: Endpoint::close /// [`wait_idle()`]: Endpoint::wait_idle pub async fn wait_all_draining(&self) { loop { { let endpoint = &mut *self.inner.state.lock().unwrap(); - if endpoint.recv_state.connections.all_draining() { + if endpoint.all_draining() { break; } // Construct future while lock is held to avoid race @@ -353,6 +367,22 @@ impl Endpoint { .await; } } + + /// Wait until a connection attempt is queued for [`accept()`](Endpoint::accept) + #[cfg(test)] + pub(crate) async fn wait_queued(&self) { + loop { + { + let endpoint = self.inner.state.lock().unwrap(); + if !endpoint.recv_state.incoming.is_empty() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.incoming.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -419,13 +449,18 @@ impl Future for EndpointDriver { impl Drop for EndpointDriver { fn drop(&mut self) { - let mut endpoint = self.0.state.lock().unwrap(); + let endpoint = &mut *self.0.state.lock().unwrap(); endpoint.driver_lost = true; self.0.shared.incoming.notify_waiters(); // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); endpoint.recv_state.connections.draining.clear(); + // `accept` yields nothing once the driver is lost, so queued attempts would hold off + // `wait_all_draining` forever. + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.inner.ignore(incoming); + } self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else @@ -500,6 +535,10 @@ impl EndpointInner { if state.is_idle() { self.shared.idle.notify_waiters(); } + // The pending accept held off `wait_all_draining`, and a failed one sends no event. + if state.all_draining() { + self.shared.all_draining.notify_waiters(); + } result } @@ -609,9 +648,7 @@ impl State { shared.idle.notify_waiters(); } } - if (event.is_draining() || event.is_drained()) - && self.recv_state.connections.all_draining() - { + if (event.is_draining() || event.is_drained()) && self.all_draining() { shared.all_draining.notify_waiters(); } let Some(event) = self.inner.handle_event(ch, event) else { @@ -633,6 +670,13 @@ impl State { fn is_idle(&self) -> bool { self.recv_state.connections.is_empty() && self.pending_accepts == 0 } + + /// Whether no connection is still active, nor any attempt left that could become one + fn all_draining(&self) -> bool { + self.recv_state.connections.all_draining() + && self.pending_accepts == 0 + && self.recv_state.incoming.is_empty() + } } impl Drop for State { diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0b284f1770..5d50d7a004 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1285,6 +1285,76 @@ async fn split_accept_close_during_pending_accept() { assert_eq!(server.open_connections(), 0); } +/// An accept still in progress when the endpoint closes will send a close of its own, so +/// `wait_all_draining` must not report the endpoint drained until that accept finishes. +#[tokio::test] +async fn split_accept_close_holds_off_all_draining() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, _) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + + blocker.release(); + // The waiter parked above resolves only if the accept's close is accounted for and announced. + timeout(Duration::from_secs(5), draining).await.unwrap(); + assert!(await_connection(connect).await.is_err()); + + let _ = await_connection(accept).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// `close` refuses attempts still queued for `accept`, so the peer hears back before +/// `wait_all_draining` lets the caller release the socket. +#[tokio::test] +async fn close_refuses_queued_incoming() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + server.close(0u32.into(), b"closing"); + timeout(Duration::from_secs(5), server.wait_all_draining()) + .await + .unwrap(); + assert!(server.accept().await.is_none()); + + // Without the refusal, the attempt sits in the queue and the client waits out its timeout. + let error = timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .unwrap_err(); + assert!(matches!( + error, + crate::quic::driver::ConnectionError::ConnectionClosed(close) + if close.error_code == moq_quic::TransportErrorCode::CONNECTION_REFUSED + )); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From e05a71197204de784a14145f38a342b8bf2a2c85 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:44:19 -0700 Subject: [PATCH 12/37] feat(moq-quic)!: schedule streams in send groups Replace quinn's single pending-stream heap with a hierarchical schedule: - Streams outside every group go first, in quinn's order (priority, then round robin), so control and HTTP/3 streams are never queued behind media. - Groups belong to a scheduling domain (one default). Inside a domain the highest backlogged priority is served strictly, and equal priorities share bytes by deficit round robin, charged in wire bytes. Domains share bytes the same way, so opening more groups buys a domain nothing. - A stream's priority widens to i64 and orders it only within its group. Debt carries across a group going idle; unused credit does not, and a blocked stream holds no turn. Connection::send_groups opens, moves and closes groups and domains as owned tokens. moq-tokio's and moq-uring's drivers wrap a group in a shared SendGroup handle that closes on drop, and their streams gain set_group. Retransmits re-queue through the same schedule, so a lost range keeps its group's share. Tests drive the real send path with differently sized streams: byte fairness within a bounded window, preemption, per-group order, priority updates, sparse sequences, blocked streams, retransmits, domains, resets and leaving a group. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/README.md | 1 + rs/moq-quic/src/connection/mod.rs | 12 +- rs/moq-quic/src/connection/streams/mod.rs | 166 ++--- .../src/connection/streams/saturated.rs | 441 ++++++++++++ .../src/connection/streams/schedule.rs | 660 ++++++++++++++++++ rs/moq-quic/src/connection/streams/send.rs | 5 +- rs/moq-quic/src/connection/streams/state.rs | 81 +-- rs/moq-quic/src/lib.rs | 4 +- rs/moq-tokio/src/quic/driver/connection.rs | 9 +- rs/moq-tokio/src/quic/driver/mod.rs | 2 + rs/moq-tokio/src/quic/driver/send_group.rs | 71 ++ rs/moq-tokio/src/quic/driver/send_stream.rs | 19 +- rs/moq-tokio/src/quic/web/send.rs | 15 +- rs/moq-tokio/src/quic/web/session.rs | 12 +- rs/moq-uring/src/quic/driver/connection.rs | 6 + rs/moq-uring/src/quic/driver/group.rs | 74 ++ rs/moq-uring/src/quic/driver/mod.rs | 2 + rs/moq-uring/src/quic/driver/stream.rs | 18 +- rs/moq-uring/src/quic/mod.rs | 2 +- rs/moq-uring/src/quic/web.rs | 21 +- 20 files changed, 1478 insertions(+), 143 deletions(-) create mode 100644 rs/moq-quic/src/connection/streams/saturated.rs create mode 100644 rs/moq-quic/src/connection/streams/schedule.rs create mode 100644 rs/moq-tokio/src/quic/driver/send_group.rs create mode 100644 rs/moq-uring/src/quic/driver/group.rs diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index d47b7629b0..2a6a1cfbfa 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -64,6 +64,7 @@ Changes on top of the upstream commit, besides the renames: - `TransportConfig::handshake_idle_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. - **Instants on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. +- **Send groups** (`streams/schedule.rs`, `Connection::send_groups`): streams outside every group go first, in quinn's order; groups in one scheduling domain are served strictly by priority, with equal priorities sharing bytes by deficit round robin; domains share bytes the same way. A stream's priority is an `i64` ordering it only within its group. This replaces quinn's single pending-stream heap, so a cherry-pick touching `PendingStreamsQueue` has to be redone against the schedule. moq-tokio's `quic::driver` and moq-uring's driver wrap a group in a shared `SendGroup` handle that closes on drop. - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index e828b46055..7812de3bec 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -84,8 +84,8 @@ pub use streams::StreamsState; #[cfg(not(fuzzing))] use streams::StreamsState; pub use streams::{ - Chunks, ClosedStream, FinishError, ReadError, ReadableError, RecvStream, SendStream, - ShouldTransmit, StreamEvent, Streams, WriteError, Written, + Chunks, ClosedStream, FinishError, ReadError, ReadableError, RecvStream, SendDomain, SendGroup, + SendGroups, SendStream, ShouldTransmit, StreamEvent, Streams, WriteError, Written, }; mod timer; @@ -433,6 +433,14 @@ impl Connection { } } + /// Provide control over the send groups that schedule outgoing streams + #[must_use] + pub fn send_groups(&mut self) -> SendGroups<'_> { + SendGroups { + state: &mut self.streams, + } + } + /// Provide control over streams #[must_use] pub fn recv_stream(&mut self, id: StreamId) -> RecvStream<'_> { diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index 745555db11..5348671233 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BinaryHeap, hash_map}, - io, -}; +use std::{collections::hash_map, io}; use bytes::Bytes; use thiserror::Error; @@ -12,7 +9,12 @@ use crate::{Dir, StreamId, VarInt, connection::streams::state::get_or_insert_rec mod recv; use recv::Recv; + +#[cfg(test)] +mod saturated; +mod schedule; pub use recv::{Chunks, ReadError, ReadableError}; +use schedule::Schedule; mod send; pub(crate) use send::{ByteSlice, BytesArray}; @@ -282,7 +284,9 @@ impl<'a> SendStream<'a> { self.state.buffered_data += written.bytes as u64; trace!(stream = %self.id, "wrote {} bytes", written.bytes); if !was_pending { - self.state.pending.push_pending(self.id, stream.priority); + self.state + .pending + .push(self.id, &mut stream.slot, stream.priority); } Ok(written) } @@ -313,7 +317,9 @@ impl<'a> SendStream<'a> { let was_pending = stream.is_pending(); stream.finish()?; if !was_pending { - self.state.pending.push_pending(self.id, stream.priority); + self.state + .pending + .push(self.id, &mut stream.slot, stream.priority); } Ok(()) @@ -342,17 +348,22 @@ impl<'a> SendStream<'a> { // credit based on the final offset communicated in the RESET_STREAM frame we send. self.state.buffered_data -= stream.pending.buffered(); stream.reset(); + self.state.pending.remove(&mut stream.slot); self.pending.reset_stream.push((self.id, error_code)); // Don't reopen an already-closed stream we haven't forgotten yet Ok(()) } - /// Set the priority of a stream + /// Set the priority of a stream within its send group + /// + /// Higher values go first; equal values share bytes round robin. A queued stream moves to + /// its new priority the next time it is requeued, which is after every frame when + /// [`TransportConfig::send_fairness`](crate::TransportConfig::send_fairness) is on. /// /// # Panics /// - when applied to a receive stream - pub fn set_priority(&mut self, priority: i32) -> Result<(), ClosedStream> { + pub fn set_priority(&mut self, priority: i64) -> Result<(), ClosedStream> { let max_send_data = self.state.max_send_data(self.id); let stream = self .state @@ -369,7 +380,7 @@ impl<'a> SendStream<'a> { /// /// # Panics /// - when applied to a receive stream - pub fn priority(&self) -> Result { + pub fn priority(&self) -> Result { let stream = self .state .send @@ -378,94 +389,91 @@ impl<'a> SendStream<'a> { Ok(stream.as_ref().map(|s| s.priority).unwrap_or_default()) } -} -/// A queue of streams with pending outgoing data, sorted by priority -struct PendingStreamsQueue { - streams: BinaryHeap, - /// The next stream to write out. This is `Some` when `TransportConfig::send_fairness(false)` and writing a stream is - /// interrupted while the stream still has some pending data. See `reinsert_pending()`. - next: Option, - /// A monotonically decreasing counter, used to implement round-robin scheduling for streams of the same priority. - /// Underflowing is not a practical concern, as it is initialized to u64::MAX and only decremented by 1 in `push_pending` - recency: u64, -} + /// Move the stream into a send group, or with `None` back to the ungrouped streams + /// + /// Ungrouped streams are sent before any send group. A queued stream keeps its place. + /// + /// # Panics + /// - when applied to a receive stream + /// - when `group` was opened on another connection + pub fn set_group(&mut self, group: Option<&SendGroup>) -> Result<(), ClosedStream> { + let max_send_data = self.state.max_send_data(self.id); + let stream = self + .state + .send + .get_mut(&self.id) + .ok_or(ClosedStream { _private: () })? + .get_or_insert_with(|| Send::new(max_send_data)); -impl PendingStreamsQueue { - fn new() -> Self { - Self { - streams: BinaryHeap::new(), - next: None, - recency: u64::MAX, - } + self.state.pending.join( + self.id, + &mut stream.slot, + group.map(|g| g.0), + stream.priority, + ); + Ok(()) } +} - /// Reinsert a stream that was pending and still contains unsent data. - fn reinsert_pending(&mut self, id: StreamId, priority: i32) { - assert!(self.next.is_none()); +/// Scheduling of outgoing streams between send groups +/// +/// A send group shares bytes with the other groups of its domain: a higher priority group +/// preempts every lower one, and groups of equal priority get an equal share of bytes no +/// matter how many streams each has queued. Domains share bytes equally whatever their +/// groups' priorities. Streams outside any group go before all of it. +pub struct SendGroups<'a> { + pub(super) state: &'a mut StreamsState, +} - self.next = Some(PendingStream { - priority, - recency: self.recency, // the value here doesn't really matter - id, - }); +impl SendGroups<'_> { + /// Open a send group at `priority` in `domain`, or in the default domain + /// + /// # Panics + /// - when `domain` was opened on another connection + pub fn open(&mut self, priority: i32, domain: Option<&SendDomain>) -> SendGroup { + SendGroup(self.state.pending.open_group(priority, domain.map(|d| d.0))) } - /// Push a pending stream ID with the given priority, queued after any already-queued streams for the priority - fn push_pending(&mut self, id: StreamId, priority: i32) { - // Note that in the case where fairness is disabled, if we have a reinserted stream we don't - // bump it even if priority > next.priority. In order to minimize fragmentation we - // always try to complete a stream once part of it has been written. - - // As the recency counter is monotonically decreasing, we know that using its value to sort this stream will queue it - // after all other queued streams of the same priority. - // This is enough to implement round-robin scheduling for streams that are still pending even after being handled, - // as in that case they are removed from the `BinaryHeap`, handled, and then immediately reinserted. - self.recency -= 1; - self.streams.push(PendingStream { - priority, - recency: self.recency, - id, - }); + /// Release a send group; its streams keep their place until they finish + pub fn close(&mut self, group: SendGroup) { + self.state.pending.close_group(group.0); } - fn pop(&mut self) -> Option { - self.next.take().or_else(|| self.streams.pop()) + /// Change a group's priority, moving any data it has queued along with it + pub fn set_priority(&mut self, group: &SendGroup, priority: i32) { + self.state.pending.set_group_priority(group.0, priority); } - fn clear(&mut self) { - self.next = None; - self.streams.clear(); + /// The group's priority + pub fn priority(&self, group: &SendGroup) -> i32 { + self.state.pending.group_priority(group.0) } - fn iter(&self) -> impl Iterator { - self.next.iter().chain(self.streams.iter()) + /// Open a domain, which gets an equal share of bytes with every other domain + pub fn open_domain(&mut self) -> SendDomain { + SendDomain(self.state.pending.open_domain()) } - #[cfg(test)] - fn len(&self) -> usize { - self.streams.len() + self.next.is_some() as usize + /// Release a domain; its groups keep their share until they are released + pub fn close_domain(&mut self, domain: SendDomain) { + self.state.pending.close_domain(domain.0); } } -/// The [`StreamId`] of a stream with pending data queued, ordered by its priority and recency -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] -struct PendingStream { - /// The priority of the stream - // Note that this field should be kept above the `recency` field, in order for the `Ord` derive to be correct - // (See https://doc.rust-lang.org/stable/std/cmp/trait.Ord.html#derivable) - priority: i32, - /// A tie-breaker for streams of the same priority, used to improve fairness by implementing round-robin scheduling: - /// Larger values are prioritized, so it is initialised to `u64::MAX`, and when a stream writes data, we know - /// that it currently has the highest recency value, so it is deprioritized by setting its recency to 1 less than the - /// previous lowest recency value, such that all other streams of this priority will get processed once before we get back - /// round to this one - recency: u64, - /// The ID of the stream - // The way this type is used ensures that every instance has a unique `recency` value, so this field should be kept below - // the `priority` and `recency` fields, so that it does not interfere with the behaviour of the `Ord` derive - id: StreamId, -} +/// A send group, from [`SendGroups::open`] +/// +/// Release it with [`SendGroups::close`] on the connection that opened it. +#[derive(Debug, PartialEq, Eq)] +#[must_use = "a send group lives until it is closed"] +pub struct SendGroup(usize); + +/// A domain of send groups, from [`SendGroups::open_domain`] +/// +/// Release it with [`SendGroups::close_domain`] on the connection that opened it. +#[derive(Debug, PartialEq, Eq)] +#[must_use = "a domain lives until it is closed"] +pub struct SendDomain(usize); /// Application events about streams #[derive(Debug, PartialEq, Eq)] diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs new file mode 100644 index 0000000000..4a3c511de9 --- /dev/null +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -0,0 +1,441 @@ +//! Scheduling proofs through the real send path: streams written into send groups and +//! drained into packets by `write_stream_frames`, with loss and resets applied the way +//! the connection applies them. + +use std::collections::HashMap; + +use super::state::StreamsState; +use super::{SendDomain, SendGroup, SendGroups, SendStream, Streams}; +use crate::{ + Dir, Side, StreamId, VarInt, + connection::{State as ConnState, spaces::Retransmits}, + frame::StreamMeta, + transport_parameters::TransportParameters, +}; + +const PACKET: usize = 1200; + +/// How far two equally entitled groups may drift apart: each round robin may +/// overdraw by a frame, at both the group and the domain tier. +const SLACK: u64 = 2 * PACKET as u64; + +/// A sender with more queued than the path takes, drained a packet at a time. +struct Sender { + state: StreamsState, + pending: Retransmits, + conn: ConnState, + /// Payload bytes each stream got out. + sent: HashMap, + /// Bytes each stream's frames took on the wire, headers included, which is what + /// the round robins share. + wire: HashMap, +} + +impl Sender { + fn new() -> Self { + Self::limited(VarInt::MAX) + } + + /// With each stream's flow control window at `window` bytes. + fn limited(window: VarInt) -> Self { + let mut state = StreamsState::new( + Side::Server, + 0u32.into(), + 0u32.into(), + u64::MAX, + VarInt::MAX, + VarInt::MAX, + ); + state.set_params(&TransportParameters { + initial_max_streams_uni: VarInt::MAX, + initial_max_data: VarInt::MAX, + initial_max_stream_data_uni: window, + ..TransportParameters::default() + }); + Self { + state, + pending: Retransmits::default(), + conn: ConnState::Established, + sent: HashMap::new(), + wire: HashMap::new(), + } + } + + fn groups(&mut self) -> SendGroups<'_> { + SendGroups { + state: &mut self.state, + } + } + + fn group(&mut self, priority: i32) -> SendGroup { + self.groups().open(priority, None) + } + + fn send(&mut self, id: StreamId) -> SendStream<'_> { + SendStream { + id, + state: &mut self.state, + pending: &mut self.pending, + conn_state: &self.conn, + } + } + + /// Open a stream in `group` at `priority` and queue `bytes` on it. + fn open(&mut self, group: Option<&SendGroup>, priority: i64, bytes: usize) -> StreamId { + let id = Streams { + state: &mut self.state, + conn_state: &self.conn, + } + .open(Dir::Uni) + .unwrap(); + let mut stream = self.send(id); + stream.set_group(group).unwrap(); + stream.set_priority(priority).unwrap(); + self.write(id, bytes); + id + } + + fn write(&mut self, id: StreamId, bytes: usize) { + assert_eq!(self.send(id).write(&vec![0; bytes]), Ok(bytes)); + } + + /// Send one packet, returning its frames. + fn packet(&mut self) -> Vec { + let mut buf = Vec::with_capacity(PACKET); + let frames = self.state.write_stream_frames(&mut buf, PACKET, true); + + // Every frame carries its length but perhaps the last, which can run to the + // end of the packet instead. + let mut sizes: Vec<_> = frames + .iter() + .map(|frame| { + let mut header = Vec::new(); + frame.encode(true, &mut header); + (header.len() as u64, frame.offsets.end - frame.offsets.start) + }) + .collect(); + if sizes.iter().map(|(h, p)| h + p).sum::() != buf.len() as u64 { + let (header, payload) = sizes.last_mut().unwrap(); + *header -= VarInt::from_u64(*payload).unwrap().size() as u64; + } + assert_eq!( + sizes.iter().map(|(h, p)| h + p).sum::(), + buf.len() as u64 + ); + + for (frame, (header, payload)) in frames.iter().zip(sizes) { + *self.sent.entry(frame.id).or_default() += payload; + *self.wire.entry(frame.id).or_default() += header + payload; + } + frames.into_iter().collect() + } + + /// Send `packets` packets, returning the stream of every frame in order. + fn drain(&mut self, packets: usize) -> Vec { + (0..packets) + .flat_map(|_| self.packet()) + .map(|frame| frame.id) + .collect() + } + + fn bytes(&self, streams: &[StreamId]) -> u64 { + streams + .iter() + .map(|id| self.sent.get(id).copied().unwrap_or(0)) + .sum() + } + + fn wire(&self, streams: &[StreamId]) -> u64 { + streams + .iter() + .map(|id| self.wire.get(id).copied().unwrap_or(0)) + .sum() + } + + fn clear(&mut self) { + self.sent.clear(); + self.wire.clear(); + } +} + +fn assert_fair(a: u64, b: u64) { + assert!(a.abs_diff(b) <= SLACK, "unfair split: {a} vs {b}"); + assert!(a + b > 0, "nothing sent"); +} + +/// The issue's Alice and Bob: two subscriptions at one priority, an audio track of +/// many small groups and a video track with a deep backlog of large ones. Each gets +/// half the bytes, not half the streams. +#[test] +fn equal_groups_share_bytes_not_streams() { + let mut sender = Sender::new(); + let audio = sender.group(4); + let video = sender.group(4); + let audio: Vec<_> = (0..1_000) + .map(|seq| sender.open(Some(&audio), seq, 300)) + .collect(); + let video: Vec<_> = (0..4) + .map(|seq| sender.open(Some(&video), seq, 1_000_000)) + .collect(); + + sender.drain(400); + assert_fair(sender.wire(&audio), sender.wire(&video)); +} + +/// A group that keeps opening newer streams gains nothing over one draining a single +/// stream: new streams spend the group's credit rather than earning more. +#[test] +fn newer_streams_do_not_reset_a_groups_credit() { + let mut sender = Sender::new(); + let busy = sender.group(0); + let steady = sender.group(0); + let mut busy_streams = vec![sender.open(Some(&busy), 0, 1_000_000)]; + let steady = [sender.open(Some(&steady), 0, 1_000_000)]; + + for seq in 1..400 { + busy_streams.push(sender.open(Some(&busy), seq, 2_000)); + sender.packet(); + } + assert_fair(sender.wire(&busy_streams), sender.wire(&steady)); +} + +#[test] +fn higher_priority_preempts() { + let mut sender = Sender::new(); + let high = sender.group(1); + let low = sender.group(0); + let low = sender.open(Some(&low), 0, 50_000); + let high = sender.open(Some(&high), 0, 50_000); + + let order = sender.drain(100); + let first_low = order.iter().position(|&id| id == low).unwrap(); + assert!(order[..first_low].iter().all(|&id| id == high)); + assert!(order[first_low..].iter().all(|&id| id == low)); + assert_eq!(sender.bytes(&[high, low]), 100_000); +} + +#[test] +fn priority_updates_apply_to_queued_data() { + let mut sender = Sender::new(); + let high = sender.group(1); + let low = sender.group(0); + let low_stream = sender.open(Some(&low), 0, 50_000); + let high_stream = sender.open(Some(&high), 0, 50_000); + assert!(sender.drain(5).iter().all(|&id| id == high_stream)); + + sender.groups().set_priority(&low, 2); + assert_eq!(sender.groups().priority(&low), 2); + assert!(sender.drain(5).iter().all(|&id| id == low_stream)); + + sender.groups().set_priority(&low, 1); + sender.clear(); + sender.drain(20); + assert_fair(sender.wire(&[low_stream]), sender.wire(&[high_stream])); +} + +/// Each group sends its own newest stream first, whatever the other groups' numbering. +#[test] +fn each_group_sends_in_its_own_order() { + let mut sender = Sender::new(); + let descending = sender.group(0); + let ascending = sender.group(0); + let newest: Vec<_> = [3, 1, 4, 2] + .into_iter() + .map(|seq| (seq, sender.open(Some(&descending), seq, 3_000))) + .collect(); + // An oldest-first subscription negates its sequences. + let oldest: Vec<_> = [7, 5, 6] + .into_iter() + .map(|seq| (seq, sender.open(Some(&ascending), -seq, 3_000))) + .collect(); + + let order = sender.drain(100); + let finished = |streams: &[(i64, StreamId)]| { + let mut last: Vec<_> = streams + .iter() + .map(|&(seq, id)| (order.iter().rposition(|&x| x == id).unwrap(), seq)) + .collect(); + last.sort(); + last.into_iter().map(|(_, seq)| seq).collect::>() + }; + assert_eq!(finished(&newest), [4, 3, 2, 1]); + assert_eq!(finished(&oldest), [5, 6, 7]); +} + +/// Group order takes full 62-bit group sequences and sparse jumps between them, +/// which a 32-bit scalar shared with the track priority would truncate or wrap. +#[test] +fn group_order_holds_wide_and_sparse_sequences() { + let mut sender = Sender::new(); + let group = sender.group(0); + let seqs = [ + 0, + 1 << 40, + (1 << 62) - 1, + 7, + 1 << 33, + (1 << 32) + 1, + 1 << 32, + ]; + let mut streams: Vec<_> = seqs + .into_iter() + .map(|seq| (seq, sender.open(Some(&group), seq, 100))) + .collect(); + + let order = sender.drain(10); + streams.sort_by_key(|&(seq, _)| std::cmp::Reverse(seq)); + let expected: Vec<_> = streams.into_iter().map(|(_, id)| id).collect(); + assert_eq!(order, expected); +} + +/// A stream blocked by flow control holds no turn and saves up no credit while it +/// waits, so its group neither stalls the other nor bursts once unblocked. +#[test] +fn blocked_streams_hold_no_turn() { + let mut sender = Sender::limited(5_000u32.into()); + let blocked = sender.group(0); + let open = sender.group(0); + let blocked = sender.open(Some(&blocked), 0, 5_000); + let open = sender.open(Some(&open), 0, 5_000); + let unlimited = 1 << 60; + sender + .state + .received_max_stream_data(open, unlimited) + .unwrap(); + sender.write(open, 1_000_000); + + sender.drain(50); + assert_eq!(sender.bytes(&[blocked]), 5_000); + assert!(sender.bytes(&[open]) > 50_000); + + sender + .state + .received_max_stream_data(blocked, unlimited) + .unwrap(); + sender.write(blocked, 1_000_000); + sender.clear(); + sender.drain(100); + assert_fair(sender.wire(&[blocked]), sender.wire(&[open])); +} + +/// A lost frame retransmits through its group, so new data from an equal group +/// delays it by a turn at most. +#[test] +fn retransmits_are_not_starved_by_equal_groups() { + let mut sender = Sender::new(); + let lossy = sender.group(0); + let busy = sender.group(0); + let lossy = sender.open(Some(&lossy), 0, 1_000); + let busy = sender.open(Some(&busy), 0, 10_000_000); + + let lost = sender + .packet() + .into_iter() + .find(|frame| frame.id == lossy) + .unwrap(); + sender.drain(10); + sender.state.retransmit(lost); + sender.clear(); + sender.drain(2); + assert_eq!(sender.bytes(&[lossy]), 1_000); + assert!(sender.bytes(&[busy]) > 0); +} + +/// Streams outside every group, such as control streams, go before all of them. +#[test] +fn ungrouped_streams_preempt_groups() { + let mut sender = Sender::new(); + let group = sender.group(i32::MAX); + let grouped = sender.open(Some(&group), i64::MAX, 10_000); + let control = sender.open(None, i64::MIN, 10_000); + + let order = sender.drain(20); + let first = order.iter().position(|&id| id == grouped).unwrap(); + assert!(order[..first].iter().all(|&id| id == control)); + assert_eq!(sender.bytes(&[control]), 10_000); +} + +/// Domains split bytes evenly however many groups each holds and whatever their +/// priorities, while priority still orders the groups inside one domain. +#[test] +fn domains_share_bytes_and_confine_priority() { + let mut sender = Sender::new(); + let crowded = sender.groups().open_domain(); + let quiet = sender.groups().open_domain(); + + let crowded_groups: Vec<_> = (0..10) + .map(|priority| sender.groups().open(priority, Some(&crowded))) + .collect(); + let crowded_streams: Vec<_> = crowded_groups + .iter() + .map(|group| sender.open(Some(group), 0, 1_000_000)) + .collect(); + let quiet_group = sender.groups().open(-100, Some(&quiet)); + let quiet_stream = sender.open(Some(&quiet_group), 0, 1_000_000); + + sender.drain(200); + assert_fair(sender.wire(&crowded_streams), sender.wire(&[quiet_stream])); + // Only the crowded domain's top group sent anything. + assert_eq!( + sender.bytes(&crowded_streams[9..]), + sender.bytes(&crowded_streams) + ); +} + +/// Groups outside any explicit domain share the default domain, which takes one share +/// of the domain round robin. +#[test] +fn the_default_domain_is_one_share() { + let mut sender = Sender::new(); + let domain = sender.groups().open_domain(); + let inside = sender.groups().open(0, Some(&domain)); + let outside: Vec = (0..3).map(|_| sender.group(0)).collect(); + let inside = sender.open(Some(&inside), 0, 1_000_000); + let outside: Vec<_> = outside + .iter() + .map(|group| sender.open(Some(group), 0, 1_000_000)) + .collect(); + + sender.drain(200); + assert_fair(sender.wire(&[inside]), sender.wire(&outside)); +} + +/// A released group lives until its last stream is gone, here by a reset, and then +/// leaves nothing behind in the schedule. +#[test] +fn reset_frees_a_released_group() { + let mut sender = Sender::new(); + let domain: SendDomain = sender.groups().open_domain(); + let group = sender.groups().open(0, Some(&domain)); + let reset = sender.open(Some(&group), 0, 10_000); + let finished = sender.open(Some(&group), 1, 100); + sender.send(finished).finish().unwrap(); + sender.groups().close(group); + sender.groups().close_domain(domain); + + assert_eq!(sender.drain(1)[0], finished); + sender.send(reset).reset(0u32.into()).unwrap(); + assert!(!sender.state.can_send_stream_data()); + assert!(sender.drain(1).is_empty()); + + sender.state.received_ack_of(StreamMeta { + id: finished, + offsets: 0..100, + fin: true, + }); + assert!(!sender.state.pending.is_clean()); + sender.state.reset_acked(reset); + assert!(sender.state.pending.is_clean()); +} + +/// A stream can leave its group for the ungrouped queue, keeping its data queued. +#[test] +fn streams_can_leave_their_group() { + let mut sender = Sender::new(); + let group = sender.group(0); + let id = sender.open(Some(&group), 0, 100); + sender.send(id).set_group(None).unwrap(); + sender.groups().close(group); + assert!(sender.state.pending.is_clean()); + assert_eq!(sender.drain(1), [id]); +} diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs new file mode 100644 index 0000000000..1458cfa9de --- /dev/null +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -0,0 +1,660 @@ +//! The order streams with queued data are sent in. +//! +//! Streams outside any send group go first, in quinn's order: highest priority first, round +//! robin between equal priorities. Everything else is a hierarchy: +//! +//! 1. Domains share the remaining bytes by deficit round robin. +//! 2. Inside a domain, a higher-priority send group preempts every lower one. +//! 3. Send groups of equal priority share their domain's bytes by deficit round robin. +//! 4. Inside a group, streams go in their own priority order, round robin between ties. +//! +//! Both round robins charge the bytes actually written, so a group of many small streams and a +//! group of one large stream get the same rate. A turn may overdraw its credit by one frame; +//! the debt carries into the next turn, so the error stays bounded by a frame. +//! +//! Self-contained (std and slab only) so the scheduler benchmark can compile it from source. + +use std::collections::{BTreeMap, VecDeque}; + +use slab::Slab; + +/// Bytes a domain or group may send per turn of its round robin before it yields. +/// +/// One minimum-size QUIC packet, so equal groups interleave about once per packet. +const QUANTUM: i64 = 1200; + +/// The index of the domain every group outside an explicit domain belongs to. +const DEFAULT_DOMAIN: usize = 0; + +/// A stream's place in the schedule, stored with the stream. +#[derive(Debug, Default)] +pub(super) struct Slot { + /// The send group the stream belongs to, or `None` for the ungrouped queue. + group: Option, + /// Where the stream is queued, always in its own group's queue. + queued: Option, +} + +#[derive(Debug, Clone, Copy)] +enum Queued { + /// The stream a group resumes first, when fairness between its streams is off. + Next, + At(Key), +} + +/// A queued stream's priority, then a decreasing counter so ties go round robin. +type Key = (i64, u64); + +#[derive(Debug)] +struct Queue { + streams: BTreeMap, + next: Option, +} + +impl Queue { + fn new() -> Self { + Self { + streams: BTreeMap::new(), + next: None, + } + } + + fn pop(&mut self) -> Option { + self.next + .take() + .or_else(|| self.streams.pop_last().map(|(_, id)| id)) + } + + fn remove(&mut self, queued: Queued) { + match queued { + Queued::Next => self.next = None, + Queued::At(key) => { + self.streams.remove(&key); + } + } + } + + fn is_empty(&self) -> bool { + self.next.is_none() && self.streams.is_empty() + } + + #[cfg(test)] + fn len(&self) -> usize { + self.streams.len() + self.next.is_some() as usize + } +} + +#[derive(Debug)] +struct Group { + domain: usize, + priority: i32, + queue: Queue, + /// Bytes this group may still send in its current turn; negative after an overdraw. + deficit: i64, + /// Whether the group has queued streams, or is being served one. It stays set between + /// [`Schedule::pop`] and [`Schedule::sent`], so requeueing the stream is not a new turn. + active: bool, + /// Whether an entry for this group sits in its level's round robin. Usually equal to + /// having queued streams, but a group that runs dry away from the front of the round + /// robin keeps its entry until the round robin reaches it. + ringed: bool, + /// Streams that name this group, queued or not. + members: usize, + /// Whether the owner released the group, which is freed once no stream names it. + closed: bool, +} + +#[derive(Debug)] +struct Domain { + /// The round robin of groups at each priority. Only the highest level with a backlogged + /// group is served. + levels: BTreeMap>, + deficit: i64, + /// Whether an entry for this domain sits in the domain round robin, as for groups. + ringed: bool, + /// Groups in this domain with queued streams. + backlogged: usize, + /// Groups in this domain that are not yet freed. + groups: usize, + closed: bool, +} + +impl Domain { + fn new() -> Self { + Self { + levels: BTreeMap::new(), + deficit: 0, + ringed: false, + backlogged: 0, + groups: 0, + closed: false, + } + } +} + +/// Every stream with data to send, in the order to send it. +#[derive(Debug)] +pub(super) struct Schedule { + ungrouped: Queue, + groups: Slab>, + domains: Slab, + /// The domain round robin. + ring: VecDeque, + /// Domains with a backlogged group. + backlogged: usize, + /// Decreases with every push, so a requeued stream goes behind its equal-priority peers. + recency: u64, +} + +impl Schedule { + pub(super) fn new() -> Self { + let mut domains = Slab::new(); + assert_eq!(domains.insert(Domain::new()), DEFAULT_DOMAIN); + Self { + ungrouped: Queue::new(), + groups: Slab::new(), + domains, + ring: VecDeque::new(), + backlogged: 0, + recency: u64::MAX, + } + } + + /// Whether no stream is queued. + pub(super) fn is_empty(&self) -> bool { + self.ungrouped.is_empty() && self.backlogged == 0 + } + + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.ungrouped.len() + + self + .groups + .iter() + .map(|(_, g)| g.queue.len()) + .sum::() + } + + /// Whether every group and domain the application released has been freed. + #[cfg(test)] + pub(super) fn is_clean(&self) -> bool { + self.groups.iter().all(|(_, g)| !g.closed) && self.domains.iter().all(|(_, d)| !d.closed) + } + + /// Create a domain, sharing bytes equally with every other domain. + pub(super) fn open_domain(&mut self) -> usize { + self.domains.insert(Domain::new()) + } + + /// Release a domain; it is freed once its last group is. + pub(super) fn close_domain(&mut self, domain: usize) { + assert_ne!(domain, DEFAULT_DOMAIN, "the default domain is never closed"); + self.domains[domain].closed = true; + self.free_domain(domain); + } + + /// Create a send group in `domain`, or the default domain. + pub(super) fn open_group(&mut self, priority: i32, domain: Option) -> usize { + let domain = domain.unwrap_or(DEFAULT_DOMAIN); + let state = &mut self.domains[domain]; + assert!(!state.closed, "group opened in a closed domain"); + state.groups += 1; + self.groups.insert(Group { + domain, + priority, + queue: Queue::new(), + deficit: 0, + active: false, + ringed: false, + members: 0, + closed: false, + }) + } + + /// Release a group; it is freed once no stream names it. + pub(super) fn close_group(&mut self, group: usize) { + self.groups[group].closed = true; + self.free_group(group); + } + + pub(super) fn group_priority(&self, group: usize) -> i32 { + self.groups[group].priority + } + + /// Move a group to another priority level, keeping its credit. + pub(super) fn set_group_priority(&mut self, group: usize, priority: i32) { + let state = &mut self.groups[group]; + if state.priority == priority { + return; + } + + let old = std::mem::replace(&mut state.priority, priority); + let domain = state.domain; + if state.ringed { + Self::unring_group(&mut self.domains[domain], group, old); + let state = &mut self.groups[group]; + state.ringed = !state.queue.is_empty(); + if state.ringed { + self.domains[domain] + .levels + .entry(priority) + .or_default() + .push_back(group); + } + } + } + + /// Move a stream to `group`, or out of every group, keeping its place in line if queued. + pub(super) fn join(&mut self, id: K, slot: &mut Slot, group: Option, priority: i64) { + if slot.group == group { + return; + } + if let Some(group) = group { + assert!(!self.groups[group].closed, "stream joined a closed group"); + } + + let queued = slot.queued.is_some(); + self.remove(slot); + self.release(slot); + slot.group = group; + if let Some(group) = group { + self.groups[group].members += 1; + } + if queued { + self.push(id, slot, priority); + } + } + + /// Forget a stream that is being freed. + pub(super) fn leave(&mut self, slot: &mut Slot) { + self.remove(slot); + self.release(slot); + } + + /// Queue a stream behind every queued stream of its group at the same priority. + /// + /// A stream already queued keeps its place. + pub(super) fn push(&mut self, id: K, slot: &mut Slot, priority: i64) { + if slot.queued.is_some() { + return; + } + self.recency -= 1; + let key = (priority, self.recency); + slot.queued = Some(Queued::At(key)); + self.queue(slot.group).streams.insert(key, id); + self.activate(slot.group); + } + + /// Queue a stream ahead of every other stream of its group, to finish what it started. + pub(super) fn reinsert(&mut self, id: K, slot: &mut Slot) { + debug_assert!(slot.queued.is_none(), "stream queued twice"); + let queue = self.queue(slot.group); + assert!(queue.next.is_none()); + queue.next = Some(id); + slot.queued = Some(Queued::Next); + self.activate(slot.group); + } + + /// Dequeue a stream that has nothing left to send, such as one being reset. + pub(super) fn remove(&mut self, slot: &mut Slot) { + let Some(queued) = slot.queued.take() else { + return; + }; + self.queue(slot.group).remove(queued); + self.deactivate(slot.group); + } + + /// Take the next stream to send from. + /// + /// The caller writes from it and then reports back with [`Self::sent`], which requeues + /// it and charges the bytes to its group. + pub(super) fn pop(&mut self) -> Option { + if let Some(id) = self.ungrouped.pop() { + return Some(id); + } + + loop { + let &domain = self.ring.front()?; + let state = &mut self.domains[domain]; + if state.backlogged == 0 { + self.ring.pop_front(); + state.ringed = false; + continue; + } + if state.deficit <= 0 { + state.deficit += QUANTUM; + self.ring.rotate_left(1); + continue; + } + + let mut level = state + .levels + .last_entry() + .expect("backlogged domain has a level"); + let Some(&group) = level.get().front() else { + level.remove(); + continue; + }; + let group_state = &mut self.groups[group]; + if !group_state.active { + level.get_mut().pop_front(); + group_state.ringed = false; + continue; + } + if group_state.deficit <= 0 { + group_state.deficit += QUANTUM; + level.get_mut().rotate_left(1); + continue; + } + + return Some(group_state.queue.pop().expect("checked non-empty")); + } + } + + /// Report the `bytes` written from a stream [`Self::pop`] returned, requeueing it at + /// `priority` if it has more to send. + /// + /// With `fair`, it goes behind its group's equal-priority streams; without, its group + /// resumes it before any other. + pub(super) fn sent( + &mut self, + id: K, + slot: &mut Slot, + bytes: usize, + requeue: Option, + fair: bool, + ) { + slot.queued = None; + if let Some(group) = slot.group { + let bytes = bytes as i64; + let state = &mut self.groups[group]; + state.deficit -= bytes; + self.domains[state.domain].deficit -= bytes; + } + + match requeue { + Some(priority) if fair => self.push(id, slot, priority), + Some(_) => self.reinsert(id, slot), + None => self.deactivate(slot.group), + } + } + + fn queue(&mut self, group: Option) -> &mut Queue { + match group { + Some(group) => &mut self.groups[group].queue, + None => &mut self.ungrouped, + } + } + + /// Give a group that just got its first queued stream a turn. + fn activate(&mut self, group: Option) { + let Some(group) = group else { + return; + }; + let state = &mut self.groups[group]; + if state.active { + return; + } + state.active = true; + + // Debt carries over, but unused credit does not, so going idle saves nothing up. + state.deficit = state.deficit.min(0) + QUANTUM; + let domain = &mut self.domains[state.domain]; + if !state.ringed { + state.ringed = true; + domain + .levels + .entry(state.priority) + .or_default() + .push_back(group); + } + + domain.backlogged += 1; + if domain.backlogged > 1 { + return; + } + + self.backlogged += 1; + domain.deficit = domain.deficit.min(0) + QUANTUM; + if !domain.ringed { + domain.ringed = true; + self.ring.push_back(self.groups[group].domain); + } + } + + /// Account for a group whose queue may have run dry. + /// + /// A group or domain heading its round robin, as one just served does, leaves it now. + /// Anywhere else its entry stays until the round robin reaches it, since finding it + /// would mean a scan. + fn deactivate(&mut self, group: Option) { + let Some(group) = group else { + return; + }; + let state = &mut self.groups[group]; + if !state.active || !state.queue.is_empty() { + return; + } + state.active = false; + + let domain = &mut self.domains[state.domain]; + if let Some(level) = domain.levels.get_mut(&state.priority) + && level.front() == Some(&group) + { + level.pop_front(); + if level.is_empty() { + domain.levels.remove(&state.priority); + } + state.ringed = false; + } + + domain.backlogged -= 1; + if domain.backlogged > 0 { + return; + } + + self.backlogged -= 1; + if self.ring.front() == Some(&state.domain) { + self.ring.pop_front(); + domain.ringed = false; + } + } + + /// Drop a stream's membership of its group. + fn release(&mut self, slot: &mut Slot) { + if let Some(group) = slot.group.take() { + self.groups[group].members -= 1; + self.free_group(group); + } + } + + fn free_group(&mut self, group: usize) { + let state = &self.groups[group]; + if !state.closed || state.members > 0 { + return; + } + + // No members means nothing queued, but a stale round-robin entry may remain. + let state = self.groups.remove(group); + let domain = &mut self.domains[state.domain]; + if state.ringed { + Self::unring_group(domain, group, state.priority); + } + domain.groups -= 1; + self.free_domain(state.domain); + } + + fn free_domain(&mut self, domain: usize) { + let state = &self.domains[domain]; + if !state.closed || state.groups > 0 { + return; + } + + if self.domains.remove(domain).ringed { + let index = self.ring.iter().position(|&d| d == domain); + self.ring + .remove(index.expect("ringed domain is in the ring")); + } + } + + fn unring_group(domain: &mut Domain, group: usize, priority: i32) { + let level = domain + .levels + .get_mut(&priority) + .expect("ringed group has a level"); + let index = level.iter().position(|&g| g == group); + level.remove(index.expect("ringed group is in its level")); + if level.is_empty() { + domain.levels.remove(&priority); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + /// Streams with a fixed number of bytes each, drained a frame at a time. + struct Sim { + schedule: Schedule, + streams: HashMap, + next: u64, + fair: bool, + } + + struct Stream { + slot: Slot, + priority: i64, + remaining: usize, + } + + impl Sim { + fn new() -> Self { + Self { + schedule: Schedule::new(), + streams: HashMap::new(), + next: 0, + fair: true, + } + } + + fn stream(&mut self, group: Option, priority: i64, bytes: usize) -> u64 { + let id = self.next; + self.next += 1; + let mut stream = Stream { + slot: Slot::default(), + priority, + remaining: bytes, + }; + self.schedule.join(id, &mut stream.slot, group, priority); + self.schedule.push(id, &mut stream.slot, priority); + self.streams.insert(id, stream); + id + } + + /// Send one frame of up to `frame` bytes, returning the stream it came from. + fn step(&mut self, frame: usize) -> Option { + let id = self.schedule.pop()?; + let stream = self.streams.get_mut(&id).unwrap(); + let bytes = frame.min(stream.remaining); + stream.remaining -= bytes; + let requeue = (stream.remaining > 0).then_some(stream.priority); + self.schedule + .sent(id, &mut stream.slot, bytes, requeue, self.fair); + if requeue.is_none() { + let mut stream = self.streams.remove(&id).unwrap(); + self.schedule.leave(&mut stream.slot); + } + Some(id) + } + } + + #[test] + fn ungrouped_streams_go_first() { + let mut sim = Sim::new(); + let group = sim.schedule.open_group(i32::MAX, None); + let grouped = sim.stream(Some(group), i64::MAX, 100); + let ungrouped = sim.stream(None, i64::MIN, 100); + assert_eq!(sim.step(1000), Some(ungrouped)); + assert_eq!(sim.step(1000), Some(grouped)); + assert_eq!(sim.step(1000), None); + assert!(sim.schedule.is_empty()); + } + + #[test] + fn a_dry_group_away_from_the_front_is_skipped() { + let mut sim = Sim::new(); + let a = sim.schedule.open_group(0, None); + let b = sim.schedule.open_group(0, None); + let first = sim.stream(Some(a), 0, 100_000); + let second = sim.stream(Some(b), 0, 100_000); + + // `b` runs dry while `a` heads the round robin. + let mut stream = sim.streams.remove(&second).unwrap(); + sim.schedule.leave(&mut stream.slot); + for _ in 0..10 { + assert_eq!(sim.step(1000), Some(first)); + } + + // And rejoins with a fresh turn. + let second = sim.stream(Some(b), 0, 100_000); + let mut seen = [0usize; 2]; + for _ in 0..100 { + let id = sim.step(1000).unwrap(); + seen[(id == second) as usize] += 1; + } + assert!(seen[0].abs_diff(seen[1]) <= 2, "{seen:?}"); + } + + #[test] + fn closed_groups_and_domains_free_with_their_last_stream() { + let mut sim = Sim::new(); + let domain = sim.schedule.open_domain(); + let group = sim.schedule.open_group(0, Some(domain)); + let id = sim.stream(Some(group), 0, 10); + sim.schedule.close_group(group); + sim.schedule.close_domain(domain); + assert!(!sim.schedule.is_clean()); + + assert_eq!(sim.step(1000), Some(id)); + assert!(sim.schedule.is_clean()); + assert!(sim.schedule.groups.is_empty()); + assert_eq!(sim.schedule.domains.len(), 1); + assert!(sim.schedule.ring.is_empty()); + } + + #[test] + fn moving_a_queued_stream_keeps_it_queued() { + let mut sim = Sim::new(); + let low = sim.schedule.open_group(0, None); + let high = sim.schedule.open_group(1, None); + let a = sim.stream(Some(low), 0, 10); + let b = sim.stream(Some(low), 0, 10); + + let stream = sim.streams.get_mut(&b).unwrap(); + sim.schedule.join(b, &mut stream.slot, Some(high), 0); + assert_eq!(sim.step(1000), Some(b)); + assert_eq!(sim.step(1000), Some(a)); + assert!(sim.schedule.is_empty()); + assert!(sim.schedule.ring.is_empty()); + } + + #[test] + fn priority_changes_move_a_backlogged_group() { + let mut sim = Sim::new(); + let a = sim.schedule.open_group(1, None); + let b = sim.schedule.open_group(0, None); + let first = sim.stream(Some(a), 0, 100_000); + let second = sim.stream(Some(b), 0, 100_000); + assert_eq!(sim.step(1000), Some(first)); + + sim.schedule.set_group_priority(b, 2); + assert_eq!(sim.step(1000), Some(second)); + sim.schedule.set_group_priority(a, 2); + // Equal again, so the two alternate. + let ids: Vec<_> = (0..4).map(|_| sim.step(1000).unwrap()).collect(); + assert!(ids.contains(&first) && ids.contains(&second), "{ids:?}"); + } +} diff --git a/rs/moq-quic/src/connection/streams/send.rs b/rs/moq-quic/src/connection/streams/send.rs index 2e2c847fa2..237e8ab93b 100644 --- a/rs/moq-quic/src/connection/streams/send.rs +++ b/rs/moq-quic/src/connection/streams/send.rs @@ -1,6 +1,7 @@ use bytes::Bytes; use thiserror::Error; +use super::schedule::Slot; use crate::{VarInt, connection::send_buffer::SendBuffer, frame}; #[derive(Debug)] @@ -8,7 +9,8 @@ pub(super) struct Send { pub(super) max_data: u64, pub(super) state: SendState, pub(super) pending: SendBuffer, - pub(super) priority: i32, + pub(super) priority: i64, + pub(super) slot: Slot, /// Whether a frame containing a FIN bit must be transmitted, even if we don't have any new data pub(super) fin_pending: bool, /// Whether this stream is in the `connection_blocked` list of `Streams` @@ -28,6 +30,7 @@ impl Send { state: SendState::Ready, pending: SendBuffer::new(), priority: 0, + slot: Slot::default(), fin_pending: false, connection_blocked: false, data_blocked_limit: None, diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 0c47d3df7f..b7624b45d4 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -9,8 +9,8 @@ use rustc_hash::FxHashMap; use tracing::{debug, trace}; use super::{ - PendingStreamsQueue, Recv, Retransmits, Send, SendState, ShouldTransmit, StreamEvent, - StreamHalf, ThinRetransmits, + Recv, Retransmits, Schedule, Send, SendState, ShouldTransmit, StreamEvent, StreamHalf, + ThinRetransmits, }; use crate::{ Dir, MAX_STREAM_COUNT, Side, StreamId, TransportError, VarInt, @@ -100,8 +100,8 @@ pub struct StreamsState { /// This differs from `self.send.len()` in that it does not include streams that the peer is /// permitted to open but which have not yet been opened. pub(super) send_streams: usize, - /// Streams with outgoing data queued, sorted by priority - pub(super) pending: PendingStreamsQueue, + /// Streams with outgoing data queued, in the order to send them + pub(super) pending: Schedule, events: VecDeque, /// Streams blocked on connection-level flow control or stream window space @@ -171,7 +171,7 @@ impl StreamsState { opened: [false, false], next_reported_remote: [0, 0], send_streams: 0, - pending: PendingStreamsQueue::new(), + pending: Schedule::new(), events: VecDeque::new(), connection_blocked: Vec::new(), max_data: 0, @@ -225,7 +225,9 @@ impl StreamsState { // We don't bother calling `stream_freed` here because we explicitly reset affected // counters below. let id = StreamId::new(self.side, dir, i); - self.send.remove(&id).unwrap(); + if let Some(mut stream) = self.send.remove(&id).unwrap() { + self.pending.leave(&mut stream.slot); + } if let Dir::Bi = dir { self.recv.remove(&id).unwrap(); } @@ -239,7 +241,6 @@ impl StreamsState { } } - self.pending.clear(); self.send_streams = 0; self.data_sent = 0; self.buffered_data = 0; @@ -380,7 +381,9 @@ impl StreamsState { hash_map::Entry::Vacant(_) => {} hash_map::Entry::Occupied(e) => { if let Some(SendState::ResetSent) = e.get().as_ref().map(|s| s.state) { - e.remove_entry(); + if let (_, Some(mut stream)) = e.remove_entry() { + self.pending.leave(&mut stream.slot); + } self.stream_freed(id, StreamHalf::Send); } } @@ -389,13 +392,7 @@ impl StreamsState { /// Whether any stream data is queued, regardless of control frames pub(crate) fn can_send_stream_data(&self) -> bool { - // Reset streams may linger in the pending stream list, but will never produce stream frames - self.pending.iter().any(|stream| { - self.send - .get(&stream.id) - .and_then(|s| s.as_ref()) - .is_some_and(|s| !s.is_reset()) - }) + !self.pending.is_empty() } /// Whether MAX_STREAM_DATA frames could be sent for stream `id` @@ -620,25 +617,17 @@ impl StreamsState { break; } - // Pop the stream of the highest priority that currently has pending data - // If the stream still has some pending data left after writing, it will be reinserted, otherwise not - let Some(stream) = self.pending.pop() else { + // Pop the next stream to send from. Reset streams leave the schedule, so every + // stream in it has data or a FIN to send. + let Some(id) = self.pending.pop() else { break; }; - - let id = stream.id; - - let Some(stream) = self.send.get_mut(&id).and_then(|s| s.as_mut()) else { - // Stream was reset with pending data and the reset was acknowledged - continue; - }; - - // Reset streams aren't removed from the pending list and still exist while the peer - // hasn't acknowledged the reset, but should not generate STREAM frames, so we need to - // check for them explicitly. - if stream.is_reset() { - continue; - } + let stream = self + .send + .get_mut(&id) + .and_then(|s| s.as_mut()) + .expect("scheduled stream is open"); + let start = buf.len(); // Now that we know the `StreamId`, we can better account for how many bytes // are required to encode it. @@ -650,18 +639,6 @@ impl StreamsState { stream.fin_pending = false; } - if stream.is_pending() { - // If the stream still has pending data, reinsert it, possibly with an updated priority value - // Fairness with other streams is achieved by implementing round-robin scheduling, - // so that the other streams will have a chance to write data - // before we touch this stream again. - if fair { - self.pending.push_pending(id, stream.priority); - } else { - self.pending.reinsert_pending(id, stream.priority); - } - } - let meta = frame::StreamMeta { id, offsets, fin }; trace!(id = %meta.id, off = meta.offsets.start, len = meta.offsets.end - meta.offsets.start, fin = meta.fin, "STREAM"); meta.encode(encode_length, buf); @@ -675,6 +652,13 @@ impl StreamsState { offsets.start += data.len() as u64; buf.put_slice(data); } + + // Requeue a stream with data left, possibly at an updated priority. With fairness, + // it goes behind its equal-priority peers so they get a turn first. The frame's + // bytes count against its group's share either way. + let requeue = stream.is_pending().then_some(stream.priority); + self.pending + .sent(id, &mut stream.slot, buf.len() - start, requeue, fair); stream_frames.push(meta); } @@ -708,7 +692,9 @@ impl StreamsState { return; } - entry.remove_entry(); + if let (_, Some(mut stream)) = entry.remove_entry() { + self.pending.leave(&mut stream.slot); + } self.stream_freed(id, StreamHalf::Send); self.events.push_back(StreamEvent::Finished { id }); } @@ -722,7 +708,8 @@ impl StreamsState { return; } if !stream.is_pending() { - self.pending.push_pending(frame.id, stream.priority); + self.pending + .push(frame.id, &mut stream.slot, stream.priority); } stream.fin_pending |= frame.fin; stream.pending.retransmit(frame.offsets); @@ -741,7 +728,7 @@ impl StreamsState { continue; } if !stream.is_pending() { - self.pending.push_pending(id, stream.priority); + self.pending.push(id, &mut stream.slot, stream.priority); } stream.pending.retransmit_all_for_0rtt(); } diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index a5c39cb957..07c9009c18 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -46,8 +46,8 @@ mod connection; pub use crate::connection::{ Chunk, Chunks, ClosedStream, Connection, ConnectionError, ConnectionStats, Datagrams, Event, FinishError, FrameStats, PathStats, ReadError, ReadableError, RecvStream, RttEstimator, - SendDatagramError, SendStream, ShouldTransmit, StreamEvent, Streams, UdpStats, WriteError, - Written, + SendDatagramError, SendDomain, SendGroup, SendGroups, SendStream, ShouldTransmit, StreamEvent, + Streams, UdpStats, WriteError, Written, }; #[cfg(feature = "qlog")] pub use connection::qlog::QlogStream; diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 605163a673..8c9e270d36 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -25,7 +25,7 @@ use tracing::{Instrument, Span, debug_span}; use crate::quic::driver::{ ConnectionEvent, Duration, Instant, VarInt, mutex::Mutex, recv_stream::RecvStream, - runtime::UdpSender, send_stream::SendStream, udp_transmit, + runtime::UdpSender, send_group::SendGroup, send_stream::SendStream, udp_transmit, }; use moq_quic::{ ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, @@ -328,6 +328,13 @@ impl Connection { } } + /// Open a send group at `priority`, for streams to join with [`SendStream::set_group`]. + /// + /// [`SendStream::set_group`]: crate::quic::driver::SendStream::set_group + pub fn send_group(&self, priority: i32) -> SendGroup { + SendGroup::new(self.0.0.clone(), priority) + } + /// Initiate a new outgoing bidirectional stream. /// /// Streams are cheap and instantaneous to open unless blocked by flow control. As a diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index d758cc9c7c..86b4e6dd30 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -47,6 +47,7 @@ mod incoming; mod mutex; mod recv_stream; mod runtime; +mod send_group; mod send_stream; mod work_limiter; @@ -78,6 +79,7 @@ pub use crate::quic::driver::recv_stream::{ #[cfg(test)] pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; pub use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender}; +pub use crate::quic::driver::send_group::SendGroup; pub use crate::quic::driver::send_stream::{SendStream, StoppedError, WriteError}; #[cfg(test)] diff --git a/rs/moq-tokio/src/quic/driver/send_group.rs b/rs/moq-tokio/src/quic/driver/send_group.rs new file mode 100644 index 0000000000..82609bfde1 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/send_group.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use crate::quic::driver::connection::ConnectionInner; + +/// A send group on a [`Connection`](crate::quic::driver::Connection) +/// +/// Streams that [join](crate::quic::driver::SendStream::set_group) a group share bytes with +/// the connection's other groups: a higher priority group preempts every lower one, and groups +/// of equal priority get an equal share of bytes however many streams each has queued. A +/// stream's own priority orders it only against the group's other streams. Streams outside any +/// group go before every group. See [`moq_quic::SendGroups`]. +/// +/// Clones share one group, released once the last clone is dropped and its streams finish. A +/// group does not keep its connection open. +#[derive(Debug, Clone)] +pub struct SendGroup(Arc); + +#[derive(Debug)] +struct Inner { + conn: Arc, + group: Option, +} + +impl SendGroup { + pub(crate) fn new(conn: Arc, priority: i32) -> Self { + let group = conn + .state + .lock("SendGroup::new") + .inner + .send_groups() + .open(priority, None); + Self(Arc::new(Inner { + conn, + group: Some(group), + })) + } + + /// Change the group's priority, moving any data it has queued along with it + pub fn set_priority(&self, priority: i32) { + let mut conn = self.0.conn.state.lock("SendGroup::set_priority"); + conn.inner.send_groups().set_priority(self.id(), priority); + } + + /// Get the group's priority + pub fn priority(&self) -> i32 { + let mut conn = self.0.conn.state.lock("SendGroup::priority"); + conn.inner.send_groups().priority(self.id()) + } + + /// The group as the connection knows it, if it belongs to `conn`. + pub(crate) fn on(&self, conn: &ConnectionInner) -> &moq_quic::SendGroup { + assert!( + std::ptr::eq(&*self.0.conn, conn), + "send group used on another connection" + ); + self.id() + } + + fn id(&self) -> &moq_quic::SendGroup { + self.0.group.as_ref().expect("only taken on drop") + } +} + +impl Drop for Inner { + fn drop(&mut self) { + if let Some(group) = self.group.take() { + let mut conn = self.conn.state.lock("SendGroup::drop"); + conn.inner.send_groups().close(group); + } + } +} diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index e51af7cf25..5c4fa96fbc 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -10,7 +10,7 @@ use moq_quic::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; use thiserror::Error; use crate::quic::driver::{ - VarInt, + SendGroup, VarInt, connection::{ConnectionRef, State}, }; @@ -221,18 +221,31 @@ impl SendStream { /// the priority of a stream with pending data may only take effect after that data has been /// transmitted. Using many different priority levels per connection may have a negative /// impact on performance. - pub fn set_priority(&self, priority: i32) -> Result<(), ClosedStream> { + /// + /// In a [`SendGroup`], the priority orders the stream only against the group's other streams. + pub fn set_priority(&self, priority: i64) -> Result<(), ClosedStream> { let mut conn = self.conn.state.lock("SendStream::set_priority"); conn.inner.send_stream(self.stream).set_priority(priority)?; Ok(()) } /// Get the priority of the send stream - pub fn priority(&self) -> Result { + pub fn priority(&self) -> Result { let mut conn = self.conn.state.lock("SendStream::priority"); conn.inner.send_stream(self.stream).priority() } + /// Move the stream into a send group, or with `None` back to the ungrouped streams, which + /// go before every group + /// + /// # Panics + /// - when `group` belongs to another connection + pub fn set_group(&self, group: Option<&SendGroup>) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::set_group"); + let group = group.map(|group| group.on(&self.conn)); + conn.inner.send_stream(self.stream).set_group(group) + } + /// Completes when the peer stops the stream or reads the stream to completion /// /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 7f87718527..27e992cd72 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -92,14 +92,23 @@ impl SendStream { self.stream.finish().map_err(Into::into) } - pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + /// Set the stream's priority, within its send group if it joined one. See + /// [`crate::quic::driver::SendStream::set_priority`]. + pub fn set_priority(&self, order: i64) -> Result<(), ClosedStream> { self.stream.set_priority(order).map_err(Into::into) } - pub fn priority(&self) -> Result { + /// Get the stream's priority. See [`crate::quic::driver::SendStream::priority`]. + pub fn priority(&self) -> Result { self.stream.priority().map_err(Into::into) } + /// Move the stream into a send group, or with `None` back out. See + /// [`crate::quic::driver::SendStream::set_group`]. + pub fn set_group(&self, group: Option<&crate::quic::driver::SendGroup>) -> Result<(), ClosedStream> { + self.stream.set_group(group).map_err(Into::into) + } + /// Return the underlying QUIC stream ID. /// /// > **Warning** @@ -132,7 +141,7 @@ impl web_transport_trait::SendStream for SendStream { type Error = WriteError; fn set_priority(&mut self, order: i32) { - Self::set_priority(self, order).ok(); + Self::set_priority(self, order.into()).ok(); } fn reset(&mut self, code: u32) { diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index ad0fc8c2d6..8293ca6624 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -261,7 +261,7 @@ impl Session { // Otherwise the application could write data with lower priority than the header, resulting // in queuing. Also the header is very important for determining the session ID // without reliable reset. - send.set_priority(i32::MAX).ok(); + send.set_priority(i64::MAX).ok(); Self::write_full(&mut send, &self.header_uni) .await .map_err(|e| self.map_error(e))?; @@ -279,7 +279,7 @@ impl Session { // Otherwise the application could write data with lower priority than the header, resulting // in queuing. Also the header is very important for determining the session ID // without reliable reset. - send.set_priority(i32::MAX).ok(); + send.set_priority(i64::MAX).ok(); Self::write_full(&mut send, &self.header_bi) .await .map_err(|e| self.map_error(e))?; @@ -362,6 +362,14 @@ impl Session { Ok(()) } + /// Open a send group at `priority`, for streams to join with [`SendStream::set_group`]. + /// + /// See [`crate::quic::driver::SendGroup`]. The HTTP/3 streams stay outside every group, + /// so they go first. + pub fn send_group(&self, priority: i32) -> crate::quic::driver::SendGroup { + self.conn.send_group(priority) + } + /// Computes the maximum size of datagrams that may be passed to /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. pub fn max_datagram_size(&self) -> usize { diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index fb70a5a112..f39c3c1bd5 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -303,6 +303,12 @@ impl Connection { &self.shared.owner } + /// Open a send group at `priority`, for streams to join with + /// [`SendStream::set_group`](super::SendStream::set_group). + pub fn send_group(&self, priority: i32) -> super::SendGroup { + super::SendGroup::new(self.shared.clone(), priority) + } + /// The peer's certificate chain in DER, leaf first, or `None` if it /// presented none. /// diff --git a/rs/moq-uring/src/quic/driver/group.rs b/rs/moq-uring/src/quic/driver/group.rs new file mode 100644 index 0000000000..24a18df0e8 --- /dev/null +++ b/rs/moq-uring/src/quic/driver/group.rs @@ -0,0 +1,74 @@ +//! Send groups: a shared handle around one of moq-quic's. + +use std::rc::Rc; + +use super::Shared; + +/// A send group on a [`Connection`](super::Connection). +/// +/// Streams that [join](super::SendStream::set_group) a group share bytes with the +/// connection's other groups: a higher priority group preempts every lower one, and groups of +/// equal priority get an equal share of bytes however many streams each has queued. A stream's +/// own priority orders it only against the group's other streams. Streams outside any group go +/// before every group. See [`moq_quic::SendGroups`]. +/// +/// Clones share one group, released once the last clone is dropped and its streams finish. +#[derive(Clone)] +pub struct SendGroup(Rc); + +struct Inner { + shared: Shared, + group: Option, +} + +impl SendGroup { + pub(crate) fn new(shared: Shared, priority: i32) -> Self { + let group = shared.conn.borrow_mut().send_groups().open(priority, None); + Self(Rc::new(Inner { + shared, + group: Some(group), + })) + } + + /// Change the group's priority, moving any data it has queued along with it. + pub fn set_priority(&self, priority: i32) { + self.0 + .shared + .conn + .borrow_mut() + .send_groups() + .set_priority(self.id(), priority); + } + + /// The group's priority. + pub fn priority(&self) -> i32 { + self.0.shared.conn.borrow_mut().send_groups().priority(self.id()) + } + + /// The group as the connection knows it, if it belongs to `shared`. + pub(crate) fn on(&self, shared: &Shared) -> &moq_quic::SendGroup { + assert!( + Rc::ptr_eq(&self.0.shared, shared), + "send group used on another connection" + ); + self.id() + } + + fn id(&self) -> &moq_quic::SendGroup { + self.0.group.as_ref().expect("only taken on drop") + } +} + +impl Drop for Inner { + fn drop(&mut self) { + if let Some(group) = self.group.take() { + self.shared.conn.borrow_mut().send_groups().close(group); + } + } +} + +impl std::fmt::Debug for SendGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SendGroup").finish_non_exhaustive() + } +} diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index b1dc8fc86d..e796c0b46b 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -11,10 +11,12 @@ mod connection; mod endpoint; +mod group; mod stream; pub use connection::Connection; pub use endpoint::Endpoint; +pub use group::SendGroup; pub use stream::{RecvStream, SendStream}; pub(crate) use connection::{End, Shared}; diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index e415f57f30..f20d3f213a 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -70,6 +70,22 @@ impl SendStream { n } + /// Set the stream's priority, which orders it against the other streams + /// of its send group, or of the ungrouped streams. Higher goes first. + pub fn set_priority(&mut self, order: i64) { + let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); + } + + /// Move the stream into a send group, or with `None` back to the + /// ungrouped streams, which go before every group. + /// + /// # Panics + /// When `group` belongs to another connection. + pub fn set_group(&mut self, group: Option<&super::SendGroup>) { + let group = group.map(|group| group.on(&self.shared)); + let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_group(group); + } + /// [`reset`](web_transport_trait::poll::SendStream::reset) with a /// full-width code, for the WebTransport HTTP/3 error mapping. pub(crate) fn reset_code(&mut self, code: u64) { @@ -121,7 +137,7 @@ impl web_transport_trait::poll::SendStream for SendStream { fn set_priority(&mut self, order: i32) { // The trait (like W3C sendOrder) sends HIGHER values first, and so // does moq-quic. - let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); + SendStream::set_priority(self, order.into()); } fn finish(&mut self) -> Result<(), Self::Error> { diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index f6e84c8959..e3e1d4728a 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -30,7 +30,7 @@ pub mod web; mod driver; -pub use driver::{Connection, RecvStream, SendStream}; +pub use driver::{Connection, RecvStream, SendGroup, SendStream}; pub use endpoint::Endpoint; /// The QUIC payload size every full datagram in a GSO train uses, and the diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 192592f9b2..5ab11cdf0e 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -439,6 +439,13 @@ pub struct Session { } impl Session { + /// Open a send group at `priority`, for streams to join with + /// [`SendStream::set_group`]. The HTTP/3 streams stay outside every + /// group, so they go first. + pub fn send_group(&self, priority: i32) -> super::SendGroup { + self.conn.send_group(priority) + } + /// Wrap a raw-QUIC connection in a session, with no /// WebTransport layering: streams and datagrams pass through untouched. pub fn raw(conn: Connection) -> Self { @@ -826,6 +833,18 @@ impl SendStream { false => unmap_legacy(err), } } + + /// Set the stream's priority within its send group. See + /// [`super::SendStream::set_priority`]. + pub fn set_priority(&mut self, order: i64) { + self.inner.set_priority(order); + } + + /// Move the stream into a send group, or with `None` back out. See + /// [`super::SendStream::set_group`]. + pub fn set_group(&mut self, group: Option<&super::SendGroup>) { + self.inner.set_group(group); + } } impl web_transport_trait::poll::SendStream for SendStream { @@ -872,7 +891,7 @@ impl web_transport_trait::poll::SendStream for SendStream { } fn set_priority(&mut self, order: i32) { - web_transport_trait::poll::SendStream::set_priority(&mut self.inner, order); + self.inner.set_priority(order.into()); } fn finish(&mut self) -> Result<(), Self::Error> { From 9db3b98ff5547356fa72c649713a048e3bab0c9a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:53:56 -0700 Subject: [PATCH 13/37] feat(moq-net)!: give every subscription its own send group The poll transport gains send groups: - Session::send_group(priority) opens one, or returns None on a transport without them. - SendStream::Group names the transport's group (Ungrouped, uninhabited, for flat transports), SendStream::set_group joins one, and SendStream::set_priority widens to i64. - SendGroup::set_priority moves a whole group. The lite and IETF publishers open one group per subscription (and per standalone FETCH; a joining FETCH shares its subscription's) at the subscriber priority. Group streams join it ranked by sequence in the subscription's group order, with a group's first stream ahead of the rest of it. SUBSCRIBE_UPDATE moves the group once instead of reranking every stream, and an order flip re-stamps the open streams. On a transport without groups the session-wide PriorityQueue still ranks every stream, as before. moq-tokio's and moq-uring's adapters reach backend groups through new Grouped and GroupedSend extension traits: native QUIC and WebTransport open real groups; iroh, qmux, ffi and wasm stay flat. The moq-tokio adapter defers a join made while a write or FIN watch holds the stream, like a priority change. A benchmark compares the groups with the PriorityQueue and two scalar packings on congested audio and video workloads, and sweeps subscriptions against in-flight streams. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 2 +- rs/moq-ffi/src/transport/adapter.rs | 10 +- rs/moq-net/Cargo.toml | 6 + rs/moq-net/benches/schedule.rs | 489 ++++++++++++++++++++++++++ rs/moq-net/src/client.rs | 22 +- rs/moq-net/src/coding/stream.rs | 2 +- rs/moq-net/src/coding/writer.rs | 19 +- rs/moq-net/src/ietf/adapter.rs | 22 +- rs/moq-net/src/ietf/publisher.rs | 184 ++++++++-- rs/moq-net/src/lite/publisher.rs | 141 +++++++- rs/moq-net/src/lite/test_transport.rs | 81 ++++- rs/moq-net/src/priority.rs | 16 + rs/moq-net/src/server.rs | 11 +- rs/moq-net/src/transport.rs | 62 +++- rs/moq-net/src/transport/memory.rs | 11 +- rs/moq-net/tests/support/mock.rs | 15 +- rs/moq-tokio/src/server.rs | 14 +- rs/moq-tokio/src/transport.rs | 195 ++++++++-- rs/moq-tokio/src/transport/owned.rs | 15 +- rs/moq-uring/src/transport.rs | 28 +- rs/moq-uring/src/transport/adapter.rs | 49 ++- rs/moq-wasm/src/transport/adapter.rs | 10 +- 22 files changed, 1295 insertions(+), 109 deletions(-) create mode 100644 rs/moq-net/benches/schedule.rs diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 9e6d534668..1f6d708b22 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -195,7 +195,7 @@ Each subscription carries the knobs that decide behavior under congestion: | Knob | Effect | | --- | --- | -| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. | +| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. Sent from Rust over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split bandwidth evenly by bytes, however many groups each has queued. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. | diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 0ebe75b7b1..6a3fb5b14e 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -77,6 +77,9 @@ impl + Clone + 'st .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } + fn send_group(&mut self, _: i32) -> Option { + None + } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) } @@ -106,15 +109,20 @@ impl + Clone + 'st pub struct SendStream(S); impl poll::SendStream for SendStream { type Error = Error; + type Group = poll::Ungrouped; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { backend::SendStream::poll_write(&mut self.0, cx, buf).map_err(Error) } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(&mut self.0, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { + let order = order.clamp(i32::MIN.into(), i32::MAX.into()) as i32; backend::SendStream::set_priority(&mut self.0, order) } + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Self::Error> { backend::SendStream::finish(&mut self.0).map_err(Error) } diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index ed67b1e1b6..59845513f2 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -127,6 +127,12 @@ name = "priority" harness = false required-features = ["async"] +# Compiles moq-quic's send scheduler from source to compare it with the priority queue. +[[bench]] +name = "schedule" +harness = false +required-features = ["async"] + [[bench]] name = "stats" harness = false diff --git a/rs/moq-net/benches/schedule.rs b/rs/moq-net/benches/schedule.rs new file mode 100644 index 0000000000..b2a2ba8508 --- /dev/null +++ b/rs/moq-net/benches/schedule.rs @@ -0,0 +1,489 @@ +//! Send scheduling on a congested link: the transport's send groups against the publishers' +//! session-wide priority queue and two scalar orderings. +//! +//! Each workload runs every scheduler once and prints its byte shares, live-edge lag, and +//! queue work, then criterion times the run. The four schedulers: +//! +//! - `groups`: one send group per subscription at its priority, streams ordered by group +//! sequence inside it (what native QUIC uses). +//! - `queue`: the publishers' `PriorityQueue`, ranking every in-flight group on the session +//! into one `u8` send order, re-applied to each stream whose rank moves. +//! - `scalar-seq`: a 32-bit `[priority][group sequence]` key, which leaves 24 bits for the +//! sequence and compares sequences across subscriptions. +//! - `scalar`: the priority alone, so equal subscriptions share by stream count. +//! +//! `fanout` sweeps subscriptions against groups in flight per subscription, timing one frame, +//! so a scheduling cost that grows with the table rather than the touched path is a slope. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::task::{Wake, Waker}; + +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; + +// Both schedulers compile straight from source, so neither widens its crate's API for this. +#[path = "../src/priority.rs"] +#[allow(dead_code)] +mod priority; + +// The queue names `crate::track::Order`, which is public. +mod track { + pub use moq_net::track::Order; +} + +// Keeps moq-quic's own formatting. +#[rustfmt::skip] +#[path = "../../moq-quic/src/connection/streams/schedule.rs"] +#[allow(dead_code)] +mod schedule; + +use priority::{Priority, PriorityHandle, PriorityQueue}; +use schedule::{Schedule, Slot}; + +/// The most a frame carries, one packet. +const PACKET: usize = 1200; + +/// One subscription's media: `rate` bytes a tick into its open group, a new group every +/// `cadence` ticks, numbered from `first` in steps of `step`. +#[derive(Clone, Copy)] +struct Track { + priority: u8, + rate: usize, + cadence: u64, + first: u64, + step: u64, +} + +impl Track { + const fn new(priority: u8, rate: usize, cadence: u64) -> Self { + Self { + priority, + rate, + cadence, + first: 0, + step: 1, + } + } + + const fn numbered(mut self, first: u64, step: u64) -> Self { + self.first = first; + self.step = step; + self + } +} + +struct Workload { + name: &'static str, + tracks: Vec, + /// Bytes the link takes per tick. + capacity: usize, + ticks: u64, +} + +fn workloads() -> Vec { + vec![ + // The issue: audio and video at one priority, under a higher-priority track that + // takes a quarter of the link. Video has a deep backlog of large groups; audio + // opens a small group every tick. + Workload { + name: "alice-bob", + tracks: vec![ + Track::new(5, 5_000, 2), + Track::new(4, 30_000, 30), + Track::new(4, 15_000, 1), + ], + capacity: 20_000, + ticks: 3_000, + }, + // Group sequences crossing 2^24, where the scalar's 24 bits wrap. + Workload { + name: "wrap", + tracks: vec![ + Track::new(4, 15_000, 2).numbered((1 << 24) - 100, 1), + Track::new(4, 15_000, 2).numbered((1 << 24) - 100, 1), + ], + capacity: 20_000, + ticks: 3_000, + }, + // Groups numbered by a millisecond clock, which wraps 24 bits within hours. + Workload { + name: "sparse", + tracks: vec![ + Track::new(4, 15_000, 2).numbered((1 << 24) - 50_000, 1_000), + Track::new(4, 15_000, 2).numbered(1 << 30, 1_000), + ], + capacity: 20_000, + ticks: 3_000, + }, + // Many equal subscriptions with different cadences sharing one link. + Workload { + name: "crowd", + tracks: (0..64) + .map(|i| Track::new(4, 1_000 + 100 * i, 1 + i as u64 % 8)) + .collect(), + capacity: 50_000, + ticks: 2_000, + }, + ] +} + +#[derive(Clone, Copy, Debug)] +enum Kind { + Groups, + Queue, + ScalarSeq, + Scalar, +} + +const KINDS: [Kind; 4] = [Kind::Groups, Kind::Queue, Kind::ScalarSeq, Kind::Scalar]; + +impl Kind { + fn name(self) -> &'static str { + match self { + Self::Groups => "groups", + Self::Queue => "queue", + Self::ScalarSeq => "scalar-seq", + Self::Scalar => "scalar", + } + } +} + +struct Stream { + track: usize, + seq: u64, + slot: Slot, + priority: i64, + /// Bytes written but not yet sent. + queued: usize, + /// Whether the group is complete, so the stream closes once drained. + finished: bool, + opened: u64, + rank: Option, +} + +/// Collects the streams whose rank moved, the way each group serve is woken. +struct Moved { + id: u64, + list: Arc>>, +} + +impl Wake for Moved { + fn wake(self: Arc) { + self.list.lock().unwrap().push(self.id); + } +} + +#[derive(Default, Clone)] +struct Share { + bytes: u64, + /// The newest group opened and the newest delivered whole. + opened: Option, + delivered: Option, + /// Ticks from a group's completion to its last byte, summed over delivered groups. + delay: u64, + groups: u64, +} + +struct Sim { + kind: Kind, + tracks: Vec, + capacity: usize, + schedule: Schedule, + groups: Vec, + queue: PriorityQueue, + moved: Arc>>, + streams: HashMap, + current: Vec, + next: u64, + tick: u64, + shares: Vec, + /// Stream priority changes the scheduler needed, beyond each stream's first. + work: u64, +} + +impl Sim { + fn new(kind: Kind, workload: &Workload) -> Self { + let mut schedule = Schedule::new(); + let groups = workload + .tracks + .iter() + .map(|track| schedule.open_group(track.priority.into(), None)) + .collect(); + let mut sim = Self { + kind, + tracks: workload.tracks.clone(), + capacity: workload.capacity, + schedule, + groups, + queue: PriorityQueue::default(), + moved: Arc::default(), + streams: HashMap::new(), + current: Vec::new(), + next: 0, + tick: 0, + shares: vec![Share::default(); workload.tracks.len()], + work: 0, + }; + for track in 0..sim.tracks.len() { + let first = sim.tracks[track].first; + let id = sim.open(track, first); + sim.current.push(id); + } + sim + } + + fn open(&mut self, track: usize, seq: u64) -> u64 { + let id = self.next; + self.next += 1; + let priority = self.tracks[track].priority; + let mut stream = Stream { + track, + seq, + slot: Slot::default(), + priority: 0, + queued: 0, + finished: false, + opened: self.tick, + rank: None, + }; + match self.kind { + Kind::Groups => { + self.schedule.join(id, &mut stream.slot, Some(self.groups[track]), 0); + stream.priority = seq as i64; + } + Kind::Queue => { + let mut rank = self.queue.insert(Priority::new(priority, track as u64, seq)); + stream.priority = PriorityHandle::send_order_of(rank.current()).into(); + let waker = Waker::from(Arc::new(Moved { + id, + list: self.moved.clone(), + })); + let _ = rank.poll_next(&kio::Waiter::new(waker)); + stream.rank = Some(rank); + } + Kind::ScalarSeq => stream.priority = (i64::from(priority) << 24) | (seq & 0xff_ffff) as i64, + Kind::Scalar => stream.priority = priority.into(), + } + let share = &mut self.shares[track]; + share.opened = Some(share.opened.map_or(seq, |opened| opened.max(seq))); + self.streams.insert(id, stream); + id + } + + fn close(&mut self, id: u64) { + let mut stream = self.streams.remove(&id).unwrap(); + self.schedule.leave(&mut stream.slot); + let share = &mut self.shares[stream.track]; + share.delivered = Some(share.delivered.map_or(stream.seq, |seq| seq.max(stream.seq))); + share.delay += self.tick - stream.opened; + share.groups += 1; + } + + /// Produce a tick of media, then send a tick's worth of it. + fn step(&mut self) { + self.tick += 1; + for track in 0..self.tracks.len() { + let Track { + rate, cadence, step, .. + } = self.tracks[track]; + if self.tick % cadence == 0 { + let old = self.current[track]; + let stream = self.streams.get_mut(&old).unwrap(); + stream.finished = true; + let seq = stream.seq + step; + if stream.queued == 0 { + self.close(old); + } + self.current[track] = self.open(track, seq); + } + let id = self.current[track]; + let stream = self.streams.get_mut(&id).unwrap(); + if stream.queued == 0 { + self.schedule.push(id, &mut stream.slot, stream.priority); + } + stream.queued += rate; + } + + // Apply every rank the queue moved, as each woken group serve would. + let moved = std::mem::take(&mut *self.moved.lock().unwrap()); + for id in moved { + let Some(stream) = self.streams.get_mut(&id) else { + continue; + }; + let rank = stream.rank.as_mut().unwrap(); + let waker = Waker::from(Arc::new(Moved { + id, + list: self.moved.clone(), + })); + let waiter = kio::Waiter::new(waker); + while let std::task::Poll::Ready(rank) = rank.poll_next(&waiter) { + stream.priority = PriorityHandle::send_order_of(rank).into(); + self.work += 1; + } + } + + let mut budget = self.capacity; + while budget > 0 { + let Some(id) = self.schedule.pop() else { + break; + }; + let stream = self.streams.get_mut(&id).unwrap(); + let bytes = stream.queued.min(PACKET).min(budget); + stream.queued -= bytes; + budget -= bytes; + self.shares[stream.track].bytes += bytes as u64; + let requeue = (stream.queued > 0).then_some(stream.priority); + self.schedule.sent(id, &mut stream.slot, bytes, requeue, true); + if stream.queued == 0 && stream.finished { + self.close(id); + } + } + } + + fn run(&mut self, ticks: u64) { + for _ in 0..ticks { + self.step(); + } + } + + fn report(&self, workload: &str) { + let priorities: Vec = { + let mut p: Vec<_> = self.tracks.iter().map(|t| t.priority).collect(); + p.sort(); + p.dedup(); + p + }; + let mut line = format!("{workload:>10} {:>10} work {:>8}", self.kind.name(), self.work); + for priority in priorities.into_iter().rev() { + let bytes: Vec = self + .tracks + .iter() + .zip(&self.shares) + .filter(|(t, _)| t.priority == priority) + .map(|(_, s)| s.bytes as f64) + .collect(); + let sum: f64 = bytes.iter().sum(); + let squares: f64 = bytes.iter().map(|b| b * b).sum(); + let jain = if squares > 0.0 { + sum * sum / (bytes.len() as f64 * squares) + } else { + 0.0 + }; + line += &format!( + " p{priority}: {:.0} KB/tick jain {jain:.3}", + sum / self.tick as f64 / 1e3 + ); + } + let lag: Vec = self + .tracks + .iter() + .zip(&self.shares) + .map(|(track, share)| { + let lag = match (share.opened, share.delivered) { + (Some(opened), Some(delivered)) => (opened.saturating_sub(delivered) / track.step).to_string(), + _ => "-".to_string(), + }; + let delay = match share.groups { + 0 => "-".to_string(), + n => format!("{:.0}", share.delay as f64 / n as f64), + }; + format!("{lag}/{delay}") + }) + .collect(); + line += &format!(" lag/delay {}", lag.join(" ")); + eprintln!("{line}"); + } +} + +fn bench_workloads(c: &mut Criterion) { + eprintln!("byte share and Jain fairness per priority; lag in groups behind the live edge, delay in ticks"); + for workload in workloads() { + let mut group = c.benchmark_group(format!("schedule_{}", workload.name)); + for kind in KINDS { + let mut sim = Sim::new(kind, &workload); + sim.run(workload.ticks); + sim.report(workload.name); + + group.bench_function(kind.name(), |b| { + b.iter_batched( + || Sim::new(kind, &workload), + |mut sim| { + sim.run(workload.ticks); + sim + }, + BatchSize::LargeInput, + ); + }); + } + group.finish(); + } +} + +/// A saturated schedule: `subscriptions` groups at one priority, each with `streams` queued. +fn saturated(subscriptions: usize, streams: usize) -> (Schedule, Vec) { + let mut schedule = Schedule::new(); + let mut slots = Vec::with_capacity(subscriptions * streams); + for _ in 0..subscriptions { + let group = schedule.open_group(0, None); + for seq in 0..streams { + let id = slots.len() as u64; + let mut slot = Slot::default(); + schedule.join(id, &mut slot, Some(group), seq as i64); + schedule.push(id, &mut slot, seq as i64); + slots.push(slot); + } + } + (schedule, slots) +} + +fn bench_fanout(c: &mut Criterion) { + let mut group = c.benchmark_group("schedule_fanout_frame"); + for subscriptions in [1, 16, 256, 4_096] { + for streams in [1, 16, 256] { + if subscriptions * streams > 100_000 { + continue; + } + let (mut schedule, mut slots) = saturated(subscriptions, streams); + group.bench_with_input( + BenchmarkId::new(format!("subs_{subscriptions}"), streams), + &(), + |b, ()| { + b.iter(|| { + let id = schedule.pop().unwrap(); + let slot = &mut slots[id as usize]; + schedule.sent(id, slot, PACKET, Some((id % streams as u64) as i64), true); + }); + }, + ); + } + } + group.finish(); + + // A group opening and closing a stream, as each new MoQ group does. + let mut group = c.benchmark_group("schedule_fanout_churn"); + for subscriptions in [1, 16, 256, 4_096] { + for streams in [1, 16, 256] { + if subscriptions * streams > 100_000 { + continue; + } + // Opening then closing leaves the schedule as it was, so one serves every run. + let (mut schedule, _slots) = saturated(subscriptions, streams); + group.bench_with_input( + BenchmarkId::new(format!("subs_{subscriptions}"), streams), + &(), + |b, ()| { + b.iter(|| { + let mut slot = Slot::default(); + schedule.join(u64::MAX, &mut slot, Some(0), i64::MAX); + schedule.push(u64::MAX, &mut slot, i64::MAX); + schedule.leave(&mut slot); + }); + }, + ); + } + } + group.finish(); +} + +criterion_group!(benches, bench_workloads, bench_fanout); +criterion_main!(benches); diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 395fde91f3..632f5bf09d 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -620,6 +620,10 @@ mod tests { Poll::Pending } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { Poll::Ready(Ok(())) } @@ -678,13 +682,18 @@ mod tests { impl crate::transport::poll::SendStream for FakeSendStream { type Error = FakeError; + type Group = crate::transport::poll::Ungrouped; fn poll_write(&mut self, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { self.writes.lock().unwrap().put_slice(buf); Poll::Ready(Ok(buf.len())) } - fn set_priority(&mut self, _order: i32) {} + fn set_priority(&mut self, _order: i64) {} + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Self::Error> { Ok(()) @@ -1006,6 +1015,10 @@ mod tests { }) } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { self.inner.poll_send_datagram(cx, payload) } @@ -1040,15 +1053,20 @@ mod tests { impl crate::transport::poll::SendStream for LocalSend { type Error = FakeError; + type Group = crate::transport::poll::Ungrouped; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { self.inner.poll_write(cx, buf) } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { self.inner.set_priority(order); } + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } + fn finish(&mut self) -> Result<(), Self::Error> { self.inner.finish() } diff --git a/rs/moq-net/src/coding/stream.rs b/rs/moq-net/src/coding/stream.rs index 1984620c73..223d64179d 100644 --- a/rs/moq-net/src/coding/stream.rs +++ b/rs/moq-net/src/coding/stream.rs @@ -67,7 +67,7 @@ impl Stream { V: Clone, { let mut writer = Writer::new(send, version.clone()); - writer.set_priority(CONTROL_SEND_ORDER); + writer.set_priority(CONTROL_SEND_ORDER.into()); Self { writer, reader: Reader::new(recv, version), diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index 037a095676..c3a46c62d2 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -207,8 +207,14 @@ impl Writer { /// scheduler) and the model's [`Subscription::priority`](crate::track::Subscription), /// where higher values preempt lower ones. The lite priority queue's rank is the /// opposite (0 = most urgent); rank holders convert via `PriorityHandle::send_order`. - pub fn set_priority(&mut self, send_order: u8) { - self.stream.as_mut().unwrap().set_priority(send_order.into()); + /// In a send group, the order ranks the stream only against the group's others. + pub fn set_priority(&mut self, send_order: i64) { + self.stream.as_mut().unwrap().set_priority(send_order); + } + + /// Move the stream into a send group from its session. + pub fn set_group(&mut self, group: &S::Group) { + self.stream.as_mut().unwrap().set_group(group); } /// Cast the writer to a different version, used during version negotiation. @@ -265,12 +271,17 @@ mod tests { impl crate::transport::poll::SendStream for Stopped { type Error = Self; + type Group = crate::transport::poll::Ungrouped; fn poll_write(&mut self, _: &mut Context<'_>, _: &[u8]) -> Poll> { Poll::Ready(Err(self.clone())) } - fn set_priority(&mut self, _: i32) {} + fn set_priority(&mut self, _: i64) {} + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Self::Error> { Err(self.clone()) @@ -323,7 +334,7 @@ mod tests { let mut writer = Writer::new(SinkSend::new(log.clone()), crate::lite::Version::Lite05); for send_order in 0u8..=255 { - writer.set_priority(send_order); + writer.set_priority(send_order.into()); } assert_eq!(log.priorities(), (0u8..=255).collect::>()); diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 03ab85c3f7..64715ff93e 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -364,6 +364,7 @@ impl VirtualSendStream { impl crate::transport::poll::SendStream for VirtualSendStream { type Error = crate::Error; + type Group = crate::transport::poll::Ungrouped; fn poll_write(&mut self, _cx: &mut Context<'_>, buf: &[u8]) -> Poll> { let len = buf.len(); @@ -378,7 +379,11 @@ impl crate::transport::poll::SendStream for VirtualSendStream { Poll::Ready(self.push(chunk).map(|()| len)) } - fn set_priority(&mut self, _order: i32) {} + fn set_priority(&mut self, _order: i64) {} + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Self::Error> { // Flush any remaining buffered data (e.g. if registration never completed). @@ -406,6 +411,7 @@ pub enum AdapterSend { impl crate::transport::poll::SendStream for AdapterSend { type Error = crate::Error; + type Group = ::Group; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { match self { @@ -421,13 +427,21 @@ impl crate::transport::poll::SendStream for } } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { match self { Self::Real(s) => s.set_priority(order), Self::Virtual(s) => s.set_priority(order), } } + fn set_group(&mut self, group: &Self::Group) { + match self { + Self::Real(s) => s.set_group(group), + // Rides the control stream, which no group schedules. + Self::Virtual(_) => {} + } + } + fn finish(&mut self) -> Result<(), Self::Error> { match self { Self::Real(s) => s.finish().map_err(|_| crate::Error::Closed), @@ -1158,6 +1172,10 @@ impl crate::transport::poll::Session for Con .map(|r| r.map(AdapterRecv::Real).map_err(|_| crate::Error::Closed)) } + fn send_group(&mut self, priority: i32) -> Option<::Group> { + self.inner.send_group(priority) + } + fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { self.inner .poll_send_datagram(cx, payload) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index b7eb29f52e..69e789bc7a 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -11,7 +11,8 @@ use std::{ time::Duration, }; -use crate::transport::poll::SendStream as _; +use crate::transport::poll::GroupOf; +use crate::transport::poll::SendGroup as _; use crate::{ AsPath, DecodeError, Error, Timescale, Timestamp, @@ -466,7 +467,7 @@ pub(super) struct Publisher { // What the peer declared in its SETUP, filled when that stream is read. peer_setup: peer::PeerSetup, // Shared across request handlers; None marks a dispatched subscription still resolving. - joins: kio::Shared>>, + joins: kio::Shared>>>>, version: Version, // Our grant (MoQ Auth): only what it lets us publish is advertised and served, and a // shrink withdraws what it no longer covers. @@ -490,7 +491,7 @@ impl Drop for Serve { /// The snapshot a joining FETCH inherits from its subscription. #[derive(Clone)] -enum Joined { +enum Joined { /// This subscription's filter is not supported by the joining FETCH handler. Unsupported, /// No objects existed when the subscription started. @@ -505,7 +506,7 @@ enum Joined { /// the fetch either. token: bool, /// The subscription's ranking, which the fetch joins. - ranking: Ranking, + ranking: Ranking, }, } @@ -513,12 +514,12 @@ enum Joined { /// /// A fetch that arrives afterwards then finds nothing and is refused, rather than being /// answered from a group that is no longer the edge of anything. -struct Join { - joins: kio::Shared>>, +struct Join { + joins: kio::Shared>>>, request_id: RequestId, } -impl Drop for Join { +impl Drop for Join { fn drop(&mut self) { self.joins.lock().remove(&self.request_id); } @@ -1535,7 +1536,7 @@ where async fn run_fill( &self, fill: FillServe, - ranking: &Ranking, + ranking: &Ranking>, track: track::Consumer, timescale: Option, opened: &AtomicU64, @@ -1854,7 +1855,7 @@ where } /// Register a pending subscription until the returned guard drops. - fn register_join(&self, request_id: RequestId) -> Join { + fn register_join(&self, request_id: RequestId) -> Join> { self.joins.lock().insert(request_id, None); Join { joins: self.joins.clone(), @@ -1990,10 +1991,12 @@ where Err(err) => return self.reject_fetch(stream, msg.request_id, &err, &err.to_string()).await, }; - // Its own entry in the queue, so it competes with group streams by its priority - // alone, its request id breaking ties. Nothing updates a FETCH's priority. + // Its own entry in the queue, or its own send group, so it competes with group + // streams by its priority alone, its request id breaking ties in the queue. + // Nothing updates a FETCH's priority. let ranking = Ranking { queue: self.priority.clone(), + group: self.session.clone().send_group(priority.into()), request_id: msg.request_id, priority: kio::Producer::new(priority).consume(), order: Order::Descending, @@ -2303,7 +2306,10 @@ where &mut self, stream: &mut Stream, subscribe_id: RequestId, - ) -> Result, Error> { + ) -> Result< + Result<(Location, track::Consumer, crate::PathOwned, Ranking>, bool), (Error, &'static str)>, + Error, + > { // Request streams can arrive out of order. Wait on registration, while bounding // the lifetime of a request whose subscription never arrives or resolves. let joined = { @@ -3383,8 +3389,9 @@ struct TrackServe { children: kio::Tasks>, /// The subscriber priority, which every group stream follows while it is open. priority: kio::Producer, - /// Where the group streams are ranked: the session's queue once [`Self::ranked`]. - ranking: Ranking, + /// Where the group streams are ranked: the subscription's send group, or the session's + /// queue once [`Self::ranked`] on a transport without groups. + ranking: Ranking>, /// The track finished: the in-flight group machines drain, then FIN. draining: bool, /// Group streams opened, shared with the group machines. @@ -3423,8 +3430,10 @@ impl TrackServe { // objects on those drafts go out unstamped. let timescale = timescale.filter(|_| ietf::Properties::sends_timescale(version)); let priority = kio::Producer::new(track.subscription().priority); + let mut session = session; let ranking = Ranking { queue: PriorityQueue::default(), + group: session.send_group(track.subscription().priority.into()), request_id, priority: priority.consume(), order: track.subscription().order, @@ -3465,6 +3474,10 @@ impl TrackServe { if let Ok(mut current) = self.priority.write() { *current = priority; } + // One move for the whole subscription; its streams keep their place within it. + if let Some(group) = &mut self.ranking.group { + group.set_priority(priority.into()); + } Ok(()) } @@ -3737,19 +3750,27 @@ impl TrackServe { /// Where one request's streams are ranked: the session's queue, keyed by the request's id /// and following its subscriber priority. #[derive(Clone)] -struct Ranking { +struct Ranking { queue: PriorityQueue, + /// The request's send group, which replaces the queue on a transport that has them. + group: Option, request_id: RequestId, priority: kio::Consumer, /// The request's GROUP_ORDER, fixed for its life: no update carries it. order: Order, } -impl Ranking { +impl Ranking { /// A stream carrying a group from `start` on, ranked among the request's others. - fn rank(&self, start: Location) -> Rank { + fn rank(&self, start: Location) -> Rank { + if let Some(group) = &self.group { + return Rank::Group { + group: group.clone(), + order: crate::priority::in_group(start.group, start.object, self.order), + }; + } let applied = *self.priority.read(); - Rank { + Rank::Queue(Queued { handle: self.queue.insert( Priority::new(applied, self.request_id.0, start.group) .with_order(self.order) @@ -3758,7 +3779,7 @@ impl Ranking { priority: self.priority.clone(), applied, order: self.order, - } + }) } } @@ -3768,17 +3789,17 @@ impl Ranking { /// Every write and close poll applies a move first. A stream parked on its group still /// polls its close to notice the subscriber leaving, so a move reaches the transport /// whatever the stream waits on. -struct Ranked { +struct Ranked { stream: T, /// Absent for a stream that only carries a refusal. - rank: Option, + rank: Option>, park: kio::Park, } impl Ranked { - fn new(mut stream: T, mut rank: Option) -> Self { + fn new(mut stream: T, mut rank: Option>) -> Self { if let Some(rank) = &mut rank { - stream.set_priority(rank.handle.send_order().into()); + rank.place(&mut stream); } Self { stream, @@ -3791,13 +3812,14 @@ impl Ranked { if let Some(rank) = &mut self.rank && let Some(order) = rank.poll_moved(self.park.hold(cx)) { - self.stream.set_priority(order.into()); + self.stream.set_priority(order); } } } impl crate::transport::poll::SendStream for Ranked { type Error = T::Error; + type Group = T::Group; fn poll_write(&mut self, cx: &mut std::task::Context<'_>, buf: &[u8]) -> Poll> { self.follow(cx); @@ -3813,10 +3835,14 @@ impl crate::transport::poll::SendStream f self.stream.poll_write_buf(cx, buf) } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { self.stream.set_priority(order); } + fn set_group(&mut self, group: &Self::Group) { + self.stream.set_group(group); + } + fn finish(&mut self) -> Result<(), Self::Error> { self.stream.finish() } @@ -3843,7 +3869,43 @@ async fn settle(writer: &mut Writer { + /// In the request's send group, at a place fixed for the stream's life: the request's + /// priority moves the whole group. + Group { group: G, order: i64 }, + /// In the session's queue, on a transport without send groups. + Queue(Queued), +} + +impl Rank { + /// A group ranked only against itself, for driving one group machine alone. + #[cfg(test)] + fn solo(priority: kio::Consumer) -> Self { + Self::Queue(Queued::solo(priority)) + } + + /// Put a newly opened stream in its place. + fn place>(&mut self, stream: &mut T) { + match self { + Self::Group { group, order } => { + stream.set_group(group); + stream.set_priority(*order); + } + Self::Queue(queued) => stream.set_priority(queued.handle.send_order().into()), + } + } + + /// A send order for the stream once the subscription priority or the queue moved it. + fn poll_moved(&mut self, waiter: &kio::Waiter) -> Option { + match self { + Self::Group { .. } => None, + Self::Queue(queued) => queued.poll_moved(waiter).map(Into::into), + } + } +} + +/// A stream's entry in the session's queue, following its subscription's priority. +struct Queued { handle: PriorityHandle, /// The subscription's priority, followed while the stream is open. priority: kio::Consumer, @@ -3853,7 +3915,7 @@ struct Rank { order: Order, } -impl Rank { +impl Queued { /// A group ranked only against itself, for driving one group machine alone. #[cfg(test)] fn solo(priority: kio::Consumer) -> Self { @@ -3891,7 +3953,7 @@ struct GroupServe { /// The subscription's count of opened streams, bumped once this one opens. opened: Arc, msg: ietf::GroupHeader, - rank: Rank, + rank: Rank>, group: group::Consumer, timescale: Option, version: Version, @@ -3943,7 +4005,7 @@ impl GroupServe { fn new( session: S, msg: ietf::GroupHeader, - rank: Rank, + rank: Rank>, mut group: group::Consumer, timescale: Option, version: Version, @@ -4006,7 +4068,7 @@ impl GroupServe { self.opened.fetch_add(1, Ordering::Relaxed); let mut stream = stream; // The current rank, which already includes anything `moved` saw. - stream.set_priority(self.rank.handle.send_order().into()); + self.rank.place(&mut stream); moved = None; let mut writer = Writer::new(stream, self.version); @@ -4593,14 +4655,14 @@ mod group_priority_test { } /// A group of `subscribe` ranked in `queue` at `priority`'s current value. - fn rank_in(queue: &PriorityQueue, priority: &kio::Producer, subscribe: u64) -> Rank { + fn rank_in(queue: &PriorityQueue, priority: &kio::Producer, subscribe: u64) -> Rank { let applied = *priority.read(); - Rank { + Rank::Queue(Queued { handle: queue.insert(Priority::new(applied, subscribe, 0)), priority: priority.consume(), applied, order: Order::Descending, - } + }) } /// The model's `Subscription::priority` is higher-first ("higher values preempt @@ -4815,6 +4877,56 @@ mod group_priority_test { ); } + /// On a transport with send groups, a subscription opens one at its priority and its + /// group streams join it, ranked newest first by sequence alone: the session-wide queue + /// takes no part. A priority update moves the group once, not each stream. + #[test] + fn grouped_streams_rank_within_their_subscription() { + let log = crate::lite::test_transport::Log::default(); + let session = SinkSession::new(log.clone()).with_groups().with_unacked_fin(); + let track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe( + track::Subscription::default() + .with_priority(3) + .with_max_delay(std::time::Duration::from_secs(3600)), + ); + let mut serve = TrackServe::new( + session, + subscriber, + RequestId(0), + Version::Draft14, + ServeRange::default(), + None, + ); + for sequence in 0..2 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + group + .write_frame(crate::Timestamp::from_millis(sequence).unwrap(), b"x".as_slice()) + .unwrap(); + group.finish().unwrap(); + } + + let waiter = kio::Waiter::noop(); + assert!(serve.poll(&waiter).is_pending()); + assert_eq!(log.groups(), [(0, 3)]); + assert_eq!( + log.joins(), + [(0, 0), (1, 0)], + "each stream joins the subscription's group" + ); + assert_eq!(log.orders(), [(0, 1), (1, 3)], "newer groups first"); + assert!(log.priorities().is_empty(), "the session queue ranks no stream"); + + serve.set_priority(9).unwrap(); + assert!(serve.poll(&waiter).is_pending()); + assert_eq!(log.groups(), [(0, 3), (0, 9)], "one move for the subscription"); + assert_eq!(log.orders().len(), 2, "the streams keep their place in the group"); + } + /// A group with more objects ready than one poll's budget drains over several polls. #[test] fn a_long_group_drains_within_the_budget() { @@ -6924,6 +7036,7 @@ mod serve_tests { let priority = kio::Producer::new(PRIORITY); let ranking = Ranking { queue: queue.clone(), + group: None, request_id: RequestId(REQUEST_ID), priority: priority.consume(), order: Order::Descending, @@ -6978,12 +7091,15 @@ mod serve_tests { let ranking = Ranking { queue: h.publisher.priority.clone(), + group: None, request_id: RequestId(REQUEST_ID), priority: kio::Producer::new(PRIORITY).consume(), order: Order::Descending, }; // The live stream, from object 1 on, queued first so the slab id alone would favor it. - let mut live = ranking.rank(Location { group: 4, object: 1 }); + let Rank::Queue(mut live) = ranking.rank(Location { group: 4, object: 1 }) else { + unreachable!("no send groups") + }; assert_eq!(live.handle.send_order(), 255); let opened = AtomicU64::new(0); diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 75a6cb6fa7..2af6c0377e 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -20,6 +20,7 @@ use crate::{ lite, priority::{Priority, PriorityHandle, PriorityQueue}, track::Order, + transport::poll::{GroupOf, SendGroup as _}, }; use super::Version; @@ -2003,11 +2004,14 @@ impl Request for SubscribeServe { }; let steer_tx = kio::Producer::new(steer); + let mut session = shared.session.clone(); + let group = session.send_group(msg.priority.into()); let sub = Subscription { - session: shared.session.clone(), + session, id: msg.id, track_name: Arc::from(track.name()), priority: shared.priority.clone(), + group, steer: steer_tx.consume(), steer_seen: steer, version: shared.version, @@ -3141,6 +3145,9 @@ struct Subscription { id: u64, track_name: Arc, priority: PriorityQueue, + /// The subscription's send group, ranking it against the session's others, or `None` + /// on a transport without groups, where `priority` ranks every group stream instead. + group: Option>, steer: kio::Consumer, /// Last steer observed by this clone, so a change only fires once. steer_seen: Steer, @@ -3241,7 +3248,13 @@ impl Subscription { priority: PriorityHandle, group: group::Consumer, ) -> Result<(), Error> { - let mut serve = Box::new(GroupServe::new(self, sequence, frame_start, priority, group)); + let mut serve = Box::new(GroupServe::new( + self, + sequence, + frame_start, + Rank::Queue(priority), + group, + )); kio::wait(move |waiter| serve.poll_serve(waiter)).await } } @@ -3329,6 +3342,10 @@ impl TrackRun { order: upd.order, }; } + // One move for the whole subscription; its streams keep their order within it. + if let Some(group) = &mut self.ctx.group { + group.set_priority(upd.priority.into()); + } // Feed the full update into the model subscriber so the producer's // aggregate reflects it (and a relay re-forwards it upstream). // Read first: `update` replaces these preferences. @@ -3610,23 +3627,35 @@ impl TrackRun { // Use the latest steer for new groups so SUBSCRIBE_UPDATE applies to them too. let steer = self.ctx.steer_current(); - // The subscribe id scopes the group tie-break: one queue serves every - // subscription on the session, and only groups of the same one may be - // ranked against each other by sequence, in that subscription's order. - let handle = self - .ctx - .priority - .insert(Priority::new(steer.priority, self.ctx.id, sequence).with_order(steer.order)); + let rank = match &self.ctx.group { + Some(group) => Rank::Group(group.clone()), + // The subscribe id scopes the group tie-break: one queue serves every + // subscription on the session, and only groups of the same one may be + // ranked against each other by sequence, in that subscription's order. + None => Rank::Queue( + self.ctx + .priority + .insert(Priority::new(steer.priority, self.ctx.id, sequence).with_order(steer.order)), + ), + }; self.children - .push(GroupServe::new(self.ctx.clone(), sequence, frame_start, handle, group)); + .push(GroupServe::new(self.ctx.clone(), sequence, frame_start, rank, group)); } } +/// Where a group stream is ranked. +enum Rank { + /// In the subscription's send group, by its sequence in the subscription's order. + Group(G), + /// In the session-wide queue, on a transport without send groups. + Queue(PriorityHandle), +} + /// Serves one group on its own unidirectional stream: the header, then every /// frame, applying queue and SUBSCRIBE_UPDATE priority changes as they land. struct GroupServe { ctx: Subscription, - priority: PriorityHandle, + rank: Rank>, group: group::Consumer, sequence: u64, frame_start: u64, @@ -3667,13 +3696,13 @@ impl GroupServe { ctx: Subscription, sequence: u64, frame_start: u64, - priority: PriorityHandle, + rank: Rank>, group: group::Consumer, ) -> Self { ctx.opens.pending.fetch_add(1, Ordering::Relaxed); Self { ctx, - priority, + rank, group, sequence, frame_start, @@ -3702,8 +3731,10 @@ impl GroupServe { // (the transport still schedules its retransmissions). The rank is re-read // as a send order when handled, since the two conventions are inverted. if let GroupState::Serve { writer, .. } | GroupState::Closed { writer } = &mut self.state { - while let Poll::Ready(rank) = self.priority.poll_next(waiter) { - writer.set_priority(PriorityHandle::send_order_of(rank)); + if let Rank::Queue(handle) = &mut self.rank { + while let Poll::Ready(rank) = handle.poll_next(waiter) { + writer.set_priority(PriorityHandle::send_order_of(rank).into()); + } } let seen = self.ctx.steer_seen; // A dropped producer just disables this arm, like the queue arm above. @@ -3715,8 +3746,14 @@ impl GroupServe { } }) { self.ctx.steer_seen = value; - let rank = self.priority.steer(value.priority, value.order); - writer.set_priority(PriorityHandle::send_order_of(rank)); + // The run loop moves the send group itself; a stream only follows the order. + let order = match &mut self.rank { + Rank::Group(_) => crate::priority::in_group(self.sequence, self.frame_start, value.order), + Rank::Queue(handle) => { + PriorityHandle::send_order_of(handle.steer(value.priority, value.order)).into() + } + }; + writer.set_priority(order); } } @@ -3738,7 +3775,14 @@ impl GroupServe { }; self.settle_open(true); let mut writer = Writer::new(stream, self.ctx.version); - writer.set_priority(self.priority.send_order()); + match &mut self.rank { + Rank::Group(group) => { + writer.set_group(group); + let order = self.ctx.steer_seen.order; + writer.set_priority(crate::priority::in_group(self.sequence, self.frame_start, order)); + } + Rank::Queue(handle) => writer.set_priority(handle.send_order().into()), + } let msg = lite::Group { subscribe: self.ctx.id, @@ -4070,6 +4114,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4114,6 +4159,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4162,6 +4208,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4228,6 +4275,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4303,6 +4351,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4350,6 +4399,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4396,6 +4446,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite06, @@ -4454,6 +4505,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version, @@ -4478,6 +4530,58 @@ mod serve_group_test { group.finish().unwrap(); } + /// On a transport with send groups, a subscription opens one at its priority and its + /// group streams join it, ranked newest first by sequence alone: the session-wide queue + /// takes no part. A SUBSCRIBE_UPDATE moves the group once, and an order flip re-stamps + /// the open streams. + #[moq_net_sim::test] + async fn grouped_streams_rank_within_their_subscription() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + // Old groups stay worth sending, so both are in flight at once. + let subscriber = track.subscribe(track::Subscription::default().with_max_delay(Duration::from_secs(3600))); + let mut session = SinkSession::new(Log::default()).with_groups().with_unacked_fin(); + let log = session.log.clone(); + let (mut run, mut writer, _) = lite_run(Version::Lite06, session.clone(), subscriber); + run.ctx.group = crate::transport::poll::Session::send_group(&mut session, 3); + + write_group(&mut track, 0, 0); + write_group(&mut track, 1, 1); + { + let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); + assert!(futures::poll!(poll.as_mut()).is_pending()); + } + assert_eq!(log.groups(), [(0, 3)]); + assert_eq!( + log.joins(), + [(0, 0), (1, 0)], + "each stream joins the subscription's group" + ); + assert_eq!(log.orders(), [(0, 1), (1, 3)], "newer groups first"); + assert!(log.priorities().is_empty(), "the session queue ranks no stream"); + + run.update(lite::SubscribeUpdate { + order: Order::Ascending, + priority: 7, + max_delay: Duration::ZERO, + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, + }); + { + let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); + assert!(futures::poll!(poll.as_mut()).is_pending()); + } + assert_eq!(log.groups(), [(0, 3), (0, 7)], "one move for the subscription"); + let mut flipped = log.orders()[2..].to_vec(); + flipped.sort(); + assert_eq!(flipped, [(0, 1), (1, -1)], "older groups first"); + } + /// A group with more frames ready than one poll's budget drains over several polls, /// so a sibling (the transport's driver, say) runs in between. #[test] @@ -4870,6 +4974,7 @@ mod serve_group_test { id: 0, track_name: "test".into(), priority: PriorityQueue::default(), + group: None, steer: steer.consume(), steer_seen: Steer::default(), version: Version::Lite07, diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index ab6cc768a3..e6682482a0 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -50,6 +50,11 @@ pub struct Log { closes: Arc>>, bi_opens: Arc, priorities: Arc>>, + /// Hands each send group its id. + group_ids: Arc, + groups: Arc>>, + joins: Arc>>, + orders: Arc>>, } /// One thing a send stream did, as recorded in [`Log::trail`]. @@ -94,6 +99,23 @@ impl Log { self.priorities.lock().unwrap().clone() } + /// Every send group's priority, as (group, priority) in call order: its opening, then + /// each move. + pub fn groups(&self) -> Vec<(usize, i32)> { + self.groups.lock().unwrap().clone() + } + + /// Every send group a stream joined, as (stream, group) in call order. + pub fn joins(&self) -> Vec<(usize, usize)> { + self.joins.lock().unwrap().clone() + } + + /// Every priority set on a stream in a send group, as (stream, priority) in call + /// order. [`Self::priorities`] keeps those of the streams outside every group. + pub fn orders(&self) -> Vec<(usize, i64)> { + self.orders.lock().unwrap().clone() + } + /// The session-level closes, as (code, reason). A list rather than a last-value so /// a second close racing the first is visible. pub fn closes(&self) -> Vec<(u32, String)> { @@ -122,6 +144,29 @@ pub struct SinkSend { finished: bool, /// A finished stream still parks, like a peer that has not acknowledged the FIN. unacked_fin: bool, + /// The send group the stream joined. + group: Option, +} + +/// A send group of a [`SinkSession::with_groups`], recording its moves in the [`Log`]. +#[derive(Clone)] +pub struct SinkGroup { + id: usize, + log: Log, +} + +impl SinkGroup { + fn open(log: Log, priority: i32) -> Self { + let id = log.group_ids.fetch_add(1, Ordering::Relaxed); + log.groups.lock().unwrap().push((id, priority)); + Self { id, log } + } +} + +impl poll::SendGroup for SinkGroup { + fn set_priority(&mut self, priority: i32) { + self.log.groups.lock().unwrap().push((self.id, priority)); + } } impl SinkSend { @@ -133,6 +178,7 @@ impl SinkSend { park: kio::Park::default(), finished: false, unacked_fin: false, + group: None, } } @@ -145,12 +191,14 @@ impl SinkSend { park: kio::Park::default(), finished: false, unacked_fin: false, + group: None, } } } impl poll::SendStream for SinkSend { type Error = SinkError; + type Group = SinkGroup; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { if let Some(gate) = &self.gate { @@ -167,11 +215,20 @@ impl poll::SendStream for SinkSend { Poll::Ready(Ok(buf.len())) } - fn set_priority(&mut self, order: i32) { - let order = u8::try_from(order).expect("moq-net sends u8 send orders"); + fn set_priority(&mut self, order: i64) { + if self.group.is_some() { + self.log.orders.lock().unwrap().push((self.id, order)); + return; + } + let order = u8::try_from(order).expect("moq-net sends u8 send orders outside a group"); self.log.priorities.lock().unwrap().push(order); } + fn set_group(&mut self, group: &Self::Group) { + self.group = Some(group.id); + self.log.joins.lock().unwrap().push((self.id, group.id)); + } + fn finish(&mut self) -> Result<(), Self::Error> { self.finished = true; self.log.record(self.id, Sent::Finish); @@ -339,6 +396,10 @@ impl poll::Session for DeadStreamSession { Poll::Ready(Ok(SinkSend::new(self.log.clone()))) } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { Poll::Ready(Ok(())) } @@ -388,6 +449,8 @@ pub struct SinkSession { uni_open_park: kio::Park, /// Set by [`Self::with_unacked_fin`]. unacked_fin: bool, + /// Set by [`Self::with_groups`]. + groups: bool, /// The ALPN to report, for a test that needs a specific negotiated version rather /// than the SETUP-negotiated fallback an absent one selects. protocol: Option<&'static str>, @@ -412,6 +475,12 @@ impl SinkSession { self } + /// Open send groups, as native QUIC does, recording them in the log. + pub fn with_groups(mut self) -> Self { + self.groups = true; + self + } + /// Report these connection statistics, as a real transport would. pub fn with_stats(self, stats: SinkStats) -> Self { self.set_stats(stats); @@ -517,6 +586,10 @@ impl poll::Session for SinkSession { Poll::Ready(Ok(send)) } + fn send_group(&mut self, priority: i32) -> Option { + self.groups.then(|| SinkGroup::open(self.log.clone(), priority)) + } + fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { Poll::Ready(Ok(())) } @@ -838,6 +911,10 @@ impl poll::Session for ScriptedSession { Poll::Ready(Ok(SinkSend::new(self.log.clone()))) } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { self.log .datagrams diff --git a/rs/moq-net/src/priority.rs b/rs/moq-net/src/priority.rs index 7c8707dad6..be22c33067 100644 --- a/rs/moq-net/src/priority.rs +++ b/rs/moq-net/src/priority.rs @@ -99,6 +99,22 @@ impl PartialOrd for Priority { } } +/// A stream's send order within its subscription's send group, where higher goes first. +/// +/// The [`Priority`] ranking minus the parts the group itself carries: the group sequence in +/// the subscription's `order`, then a group's first stream ahead of a later `start` in the +/// same group. The low bit only splits a stream from the start of its group (`start == 0`) +/// from the rest, which is the joining prefix beside the live stream; any other two streams +/// of one group share. Sequences are wire varints below 2^62, so doubling one fits. +pub fn in_group(group: u64, start: u64, order: Order) -> i64 { + let group = i64::try_from(group).unwrap_or(i64::MAX).saturating_mul(2); + let group = match order { + Order::Descending => group, + Order::Ascending => -group, + }; + group.saturating_add(i64::from(start == 0)) +} + #[derive(Debug, Clone)] struct PriorityItem { id: usize, diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 371e347a16..8bcc522988 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1065,6 +1065,10 @@ mod tests { ) -> std::task::Poll> { std::task::Poll::Pending } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram( &mut self, _cx: &mut std::task::Context<'_>, @@ -1102,6 +1106,7 @@ mod tests { struct FakeSend; impl crate::transport::poll::SendStream for FakeSend { type Error = FakeError; + type Group = crate::transport::poll::Ungrouped; fn poll_write( &mut self, _cx: &mut std::task::Context<'_>, @@ -1109,7 +1114,11 @@ mod tests { ) -> std::task::Poll> { std::task::Poll::Ready(Ok(buf.len())) } - fn set_priority(&mut self, _order: i32) {} + fn set_priority(&mut self, _order: i64) {} + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } + fn finish(&mut self) -> Result<(), Self::Error> { Ok(()) } diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 409ed432e7..4d367536ea 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -154,6 +154,16 @@ pub mod poll { /// concurrent streams. fn poll_open_bi(&mut self, cx: &mut Context<'_>) -> Poll, Self::Error>>; + /// Open a send group at `priority`, or `None` when the transport orders streams by + /// their own priority alone. + /// + /// Streams that [join](SendStream::set_group) a group share bytes with the session's + /// other groups: a higher priority group preempts every lower one, and groups of equal + /// priority get an equal share of bytes however many streams each has queued. Streams + /// outside every group go before all of them. The group is released once its last + /// handle is dropped and its streams finish. + fn send_group(&mut self, priority: i32) -> Option<::Group>; + /// Poll to send a datagram over the network. /// /// Returns [`Poll::Pending`] while the transport has no room for it, so a caller @@ -295,20 +305,54 @@ pub mod poll { /// boxed drivers (`Send` boxes on native): what the moq-transport path /// requires until it too becomes named machines. Implemented automatically. pub trait Boxable: - Session + MaybeSend + MaybeSync + Session< + SendStream: MaybeSend + SendStream, + RecvStream: MaybeSend, + Error: MaybeSend, + > + MaybeSend + + MaybeSync { } impl Boxable for S where - S: Session + MaybeSend + MaybeSync + S: Session< + SendStream: MaybeSend + SendStream, + RecvStream: MaybeSend, + Error: MaybeSend, + > + MaybeSend + + MaybeSync { } + /// The send group of a session's streams. + pub(crate) type GroupOf = <::SendStream as SendStream>::Group; + + /// A transport's send group, from [`Session::send_group`]. + /// + /// Clones share the group. + pub trait SendGroup: Clone + 'static { + /// Change the group's priority: higher preempts lower. + fn set_priority(&mut self, priority: i32); + } + + /// The send group of a transport that has none, which cannot be constructed. + #[derive(Clone, Debug)] + pub enum Ungrouped {} + + impl SendGroup for Ungrouped { + fn set_priority(&mut self, _: i32) { + match *self {} + } + } + /// An outgoing stream with partial writes, priorities, and reset codes. pub trait SendStream: 'static { /// The error type for every operation on this stream. type Error: Error; + /// The transport's send group, [`Ungrouped`] for a transport without them. + type Group: SendGroup; + /// Poll to write some of the buffer to the stream, returning how many bytes were /// written. See [`poll_write_buf`](Self::poll_write_buf) for the partial-write /// contract, which this shares. @@ -342,13 +386,15 @@ pub mod poll { /// /// Streams with higher values will be sent first, but are not guaranteed to /// arrive first. This matches the W3C WebTransport `sendOrder` convention (and - /// quinn's scheduler). + /// quinn's scheduler). In a send group, the priority orders the stream only against + /// the group's other streams. /// - /// The full `i32` range is available so callers can bit-pack a composite ordering - /// (for example a track priority in the high bits and a sequence number in the low - /// bits) into a single value. Backends that cannot express that many distinct - /// levels approximate it, so treat the ordering as best-effort. - fn set_priority(&mut self, order: i32); + /// Backends with a narrower priority saturate or approximate it, so treat the + /// ordering as best-effort. + fn set_priority(&mut self, order: i64); + + /// Move the stream into `group`, which belongs to this stream's session. + fn set_group(&mut self, group: &Self::Group); /// Mark the stream as finished, erroring on any future writes. /// diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 1bccc9948b..41ace4c5f5 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -294,6 +294,10 @@ impl poll::Session for Session { Poll::Ready(opened.map(|()| (send, recv))) } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { if let Some(err) = closed(&self.conn().close) { return Poll::Ready(Err(err)); @@ -390,6 +394,7 @@ pub struct SendStream { impl poll::SendStream for SendStream { type Error = Error; + type Group = crate::transport::poll::Ungrouped; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { let mut buf = buf; @@ -430,7 +435,11 @@ impl poll::SendStream for SendStream { } /// Ignored: with no link to share, there is nothing to schedule. - fn set_priority(&mut self, _order: i32) {} + fn set_priority(&mut self, _order: i64) {} + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Error> { if self.done { diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index da22d32c66..0b54535eca 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -14,7 +14,7 @@ use std::{ collections::VecDeque, sync::{ Arc, Mutex, - atomic::{AtomicBool, AtomicI32, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering}, }, task::{Context, Poll, ready}, time::Duration, @@ -36,7 +36,7 @@ struct Line { /// When the reset reaches the reader. reset: Mutex>, /// The send order last set, for [`MockSession::release_unis_by_priority`]. - priority: AtomicI32, + priority: AtomicI64, /// Opened while its opener held streams back. held: bool, /// Once released, delivers what follows the first write; see [`MockSession::split_unis`]. @@ -205,6 +205,7 @@ impl MockSendStream { impl poll::SendStream for MockSendStream { type Error = memory::Error; + type Group = poll::Ungrouped; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { let written = self.inner.poll_write(cx, buf); @@ -217,11 +218,15 @@ impl poll::SendStream for MockSendStream { self.wrote(first, written) } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { self.line.priority.store(order, Ordering::Relaxed); self.inner.set_priority(order); } + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } + fn finish(&mut self) -> Result<(), Self::Error> { if self .withhold_fin @@ -581,6 +586,10 @@ impl poll::Session for MockSession { Poll::Ready(Ok(send)) } + fn send_group(&mut self, _: i32) -> Option { + None + } + fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { if self.side.lossy.load(Ordering::Relaxed) { return Poll::Ready(Ok(())); diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 754d40ff57..8c250b3519 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -988,7 +988,7 @@ async fn accept_request( deadline: Option, ) -> crate::Result<(PendingRequest>, Option)> where - S: web_transport_trait::Session, + S: crate::transport::Grouped, crate::transport::Session: moq_net::transport::poll::Boxable, as moq_net::transport::poll::Session>::SendStream: moq_net::transport::MaybeSync, as moq_net::transport::poll::Session>::RecvStream: moq_net::transport::MaybeSync, @@ -2195,6 +2195,18 @@ mod tests { } } + #[cfg(feature = "websocket")] + impl crate::transport::GroupedSend for ScriptedSend { + type Group = moq_net::transport::poll::Ungrouped; + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } + } + + #[cfg(feature = "websocket")] + impl crate::transport::Grouped for Scripted {} + /// Delivers `data` once, then nothing more. #[cfg(feature = "websocket")] struct ScriptedRecv { diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 5ab97f52a9..f02c90f59d 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -57,6 +57,33 @@ impl std::fmt::Display for Transport { } } +/// A backend session whose send streams may join send groups, as native QUIC's do. +/// +/// The async interface has no groups, so moq-net's +/// [`send_group`](moq_net::transport::poll::Session::send_group) reaches a backend through +/// this. A backend without groups implements it empty. +pub trait Grouped: web_transport_trait::Session { + /// Open a send group at `priority`, or `None` when the backend has none. + fn send_group(&self, priority: i32) -> Option<::Group> { + let _ = priority; + None + } +} + +/// A backend send stream that may join its [`Grouped`] session's send groups. +pub trait GroupedSend: web_transport_trait::SendStream { + /// The backend's send group, [`Ungrouped`](moq_net::transport::poll::Ungrouped) when it has none. + type Group: moq_net::transport::poll::SendGroup + Send + Sync; + + /// Move the stream into `group`. + fn set_group(&mut self, group: &Self::Group); + + /// Set the stream's full-width priority, saturated into the async interface's by default. + fn set_order(&mut self, order: i64) { + self.set_priority(order.clamp(i32::MIN.into(), i32::MAX.into()) as i32); + } +} + /// A stored in-flight operation future. Native transports are Send, so the /// plain boxed flavor suffices. type OpBox = futures::future::BoxFuture<'static, T>; @@ -157,7 +184,7 @@ impl Clone for Session { impl wt_poll::Session for Session where - S: web_transport_trait::Session, + S: Grouped, S::SendStream: 'static, S::RecvStream: 'static, { @@ -242,13 +269,14 @@ where /// The send half produced by [`Session`]: an async-interface stream adapted to the /// poll interface. See [`Session`] for the deferred `finish`/`reset` behavior. -pub struct SendStream { +pub struct SendStream { // Same never-locked Mutex as [`Session`]'s slots: `get_mut` is free on an // exclusive borrow, and the wrapper keeps the boxed future from stripping // `Sync` off the containing session types. state: std::sync::Mutex>>, // Deferred actions, applied when the in-flight operation settles. - priority: Option, + group: Option, + priority: Option, finish: bool, reset: Option, /// Bytes already transmitted but not yet reported to the caller: a write the @@ -266,7 +294,7 @@ pub struct SendStream { interrupt: Option>, } -enum SendState { +enum SendState { Idle(S), /// A write in flight; `chunk` is the copied bytes, kept (refcounted, no /// extra copy) so a completion absorbed by `poll_closed` can verify the @@ -282,11 +310,12 @@ enum SendState { Closing(#[allow(clippy::type_complexity)] OpBox<(S, Option>)>), } -impl SendStream { +impl SendStream { /// Wrap an async-interface send stream. pub fn new(stream: S) -> Self { Self { state: std::sync::Mutex::new(Some(SendState::Idle(stream))), + group: None, priority: None, finish: false, reset: None, @@ -297,8 +326,11 @@ impl SendStream { /// Apply actions deferred while an operation was in flight. fn settle(&mut self, stream: &mut S) { + if let Some(group) = self.group.take() { + stream.set_group(&group); + } if let Some(order) = self.priority.take() { - stream.set_priority(order); + stream.set_order(order); } if let Some(code) = self.reset.take() { self.finish = false; @@ -312,7 +344,42 @@ impl SendStream { } } -impl wt_poll::SendStream for SendStream { +impl SendStream { + /// Set the full-width priority, deferred while an operation holds the stream. + fn prioritize(&mut self, order: i64) { + match self.state.get_mut().unwrap().as_mut() { + Some(SendState::Idle(stream)) => stream.set_order(order), + // A finished stream still retransmits under its priority until the FIN is + // acknowledged, so reclaim it from the watch rather than wait that out. + Some(SendState::Closing(_)) => { + self.priority = Some(order); + self.reclaim(); + } + _ => self.priority = Some(order), + } + } + + /// Join `group`, deferred like [`prioritize`](Self::prioritize). + fn join(&mut self, group: &S::Group) { + match self.state.get_mut().unwrap().as_mut() { + Some(SendState::Idle(stream)) => stream.set_group(group), + Some(SendState::Closing(_)) => { + self.group = Some(group.clone()); + self.reclaim(); + } + _ => self.group = Some(group.clone()), + } + } + + /// Interrupt the closed() watch so a deferred action applies now. + fn reclaim(&mut self) { + if let Some(tx) = self.interrupt.take() { + let _ = tx.send(()); + } + } +} + +impl wt_poll::SendStream for SendStream { type Error = S::Error; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { @@ -411,18 +478,7 @@ impl wt_poll::SendStream for SendS } fn set_priority(&mut self, order: i32) { - match self.state.get_mut().unwrap().as_mut() { - Some(SendState::Idle(stream)) => stream.set_priority(order), - // A finished stream still retransmits under its priority until the FIN is - // acknowledged, so reclaim it from the watch rather than wait that out. - Some(SendState::Closing(_)) => { - self.priority = Some(order); - if let Some(tx) = self.interrupt.take() { - let _ = tx.send(()); - } - } - _ => self.priority = Some(order), - } + self.prioritize(order.into()); } fn finish(&mut self) -> Result<(), Self::Error> { @@ -527,7 +583,7 @@ impl wt_poll::SendStream for SendS } } -impl Drop for SendStream { +impl Drop for SendStream { fn drop(&mut self) { match self.state.get_mut().unwrap().take() { // Apply a deferred reset if the stream is idle. @@ -823,6 +879,58 @@ impl Drop for RecvStream { } } +#[cfg(feature = "quic")] +impl Grouped for crate::quic::web::Session { + fn send_group(&self, priority: i32) -> Option { + Some(Self::send_group(self, priority)) + } +} + +#[cfg(feature = "quic")] +impl GroupedSend for crate::quic::web::SendStream { + type Group = crate::quic::driver::SendGroup; + + fn set_group(&mut self, group: &Self::Group) { + // A closed stream has nothing left to schedule. + Self::set_group(self, Some(group)).ok(); + } + + fn set_order(&mut self, order: i64) { + Self::set_priority(self, order).ok(); + } +} + +#[cfg(feature = "quic")] +impl moq_net::transport::poll::SendGroup for crate::quic::driver::SendGroup { + fn set_priority(&mut self, priority: i32) { + Self::set_priority(self, priority) + } +} + +#[cfg(feature = "iroh")] +impl Grouped for web_transport_iroh::Session {} + +#[cfg(feature = "iroh")] +impl GroupedSend for web_transport_iroh::SendStream { + type Group = moq_net::transport::poll::Ungrouped; + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } +} + +#[cfg(any(feature = "websocket", feature = "tcp"))] +impl Grouped for qmux::Session {} + +#[cfg(any(feature = "websocket", feature = "tcp"))] +impl GroupedSend for qmux::SendStream { + type Group = moq_net::transport::poll::Ungrouped; + + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } +} + #[cfg(test)] mod tests { use std::sync::{ @@ -857,6 +965,7 @@ mod tests { writes: Arc>>, blocked: Arc, priorities: Arc>>, + groups: Arc>>, finished: Arc, resets: Arc>>, /// The peer never acknowledges the FIN: closed() stays pending forever. @@ -907,6 +1016,22 @@ mod tests { } } + /// A send group that only names itself, so a test can see which one a stream joined. + #[derive(Clone)] + struct FakeGroup(u32); + + impl moq_net::transport::poll::SendGroup for FakeGroup { + fn set_priority(&mut self, _: i32) {} + } + + impl GroupedSend for FakeSend { + type Group = FakeGroup; + + fn set_group(&mut self, group: &FakeGroup) { + self.groups.lock().unwrap().push(group.0); + } + } + /// An async-interface receive stream replaying canned chunks, then EOF. struct FakeRecv { chunks: std::collections::VecDeque, @@ -1008,6 +1133,34 @@ mod tests { assert_eq!(fake.priorities.lock().unwrap().as_slice(), &[7]); } + // A group joined while a write holds the stream applies when it settles, and one + // joined during the FIN watch reclaims the stream, which still retransmits. + #[test] + fn send_defers_a_group_across_an_inflight_write() { + use moq_net::transport::poll; + let fake = FakeSend::default(); + fake.blocked.store(true, Ordering::SeqCst); + let mut send = SendStream::new(fake.clone()); + let mut cx = cx(); + + assert!(send.poll_write(&mut cx, b"queued").is_pending()); + poll::SendStream::set_group(&mut send, &FakeGroup(1)); + poll::SendStream::set_priority(&mut send, i64::MAX); + assert!(fake.groups.lock().unwrap().is_empty(), "deferred while in flight"); + + fake.blocked.store(false, Ordering::SeqCst); + assert_eq!(send.poll_write(&mut cx, b"queued"), Poll::Ready(Ok(6))); + assert_eq!(fake.groups.lock().unwrap().as_slice(), &[1]); + assert_eq!(fake.priorities.lock().unwrap().as_slice(), &[i32::MAX], "saturated"); + + fake.never_ack.store(true, Ordering::SeqCst); + send.finish().unwrap(); + assert!(send.poll_closed(&mut cx).is_pending()); + poll::SendStream::set_group(&mut send, &FakeGroup(2)); + assert!(send.poll_closed(&mut cx).is_pending()); + assert_eq!(fake.groups.lock().unwrap().as_slice(), &[1, 2]); + } + // The cluster-failover corruption: a write held Pending by backpressure is // driven to completion by the peer-close check (poll_closed). Its result must // reach the retrying poll_write; discarding it made the retry put the same @@ -1282,6 +1435,8 @@ mod tests { #[derive(Clone)] struct FakeSession; + impl Grouped for FakeSession {} + impl web_transport_trait::Session for FakeSession { type SendStream = FakeSend; type RecvStream = FakeRecv; diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 44d98ef0d7..037bd1b5ab 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -44,7 +44,7 @@ impl moq::Stats for Stats { self.0.estimated_send_rate() } } -impl> poll::Session for super::Session { +impl> poll::Session for super::Session { type SendStream = super::SendStream; type RecvStream = super::RecvStream; type Error = Error; @@ -60,6 +60,9 @@ impl> poll::Session for fn poll_open_bi(&mut self, cx: &mut Context<'_>) -> Poll, Self::Error>> { backend::Session::poll_open_bi(self, cx).map_err(Error) } + fn send_group(&mut self, priority: i32) -> Option<::Group> { + self.session.send_group(priority) + } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { backend::Session::poll_send_datagram(self, cx, payload).map_err(Error) } @@ -85,16 +88,20 @@ impl> poll::Session for Stats(backend::Session::stats(self)) } } -impl poll::SendStream for super::SendStream { +impl poll::SendStream for super::SendStream { type Error = Error; + type Group = S::Group; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { backend::SendStream::poll_write(self, cx, buf).map_err(Error) } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(self, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i32) { - backend::SendStream::set_priority(self, order) + fn set_priority(&mut self, order: i64) { + self.prioritize(order) + } + fn set_group(&mut self, group: &Self::Group) { + self.join(group) } fn finish(&mut self) -> Result<(), Self::Error> { backend::SendStream::finish(self).map_err(Error) diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 6a26f1d66b..cf74d4da25 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -1,3 +1,29 @@ //! Adapt backend poll sessions to moq-net without changing thread affinity. mod adapter; -pub use adapter::{Error, RecvStream, SendStream, Session}; +pub use adapter::{Error, Grouped, GroupedSend, RecvStream, SendStream, Session}; + +use crate::quic; + +impl Grouped for quic::web::Session { + fn send_group(&mut self, priority: i32) -> Option { + Some(quic::web::Session::send_group(self, priority)) + } +} + +impl GroupedSend for quic::web::SendStream { + type Group = quic::SendGroup; + + fn set_group(&mut self, group: &Self::Group) { + quic::web::SendStream::set_group(self, Some(group)); + } + + fn set_order(&mut self, order: i64) { + quic::web::SendStream::set_priority(self, order); + } +} + +impl moq_net::transport::poll::SendGroup for quic::SendGroup { + fn set_priority(&mut self, priority: i32) { + quic::SendGroup::set_priority(self, priority) + } +} diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 0a751d7671..1e073df907 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -44,6 +44,33 @@ impl moq::Stats for Stats { self.0.estimated_send_rate() } } +/// A backend session whose send streams may join send groups, as native QUIC's do. +/// +/// The backend interface has no groups, so moq-net's +/// [`send_group`](poll::Session::send_group) reaches a backend through this. A backend +/// without groups implements it empty. +pub trait Grouped: backend::Session { + /// Open a send group at `priority`, or `None` when the backend has none. + fn send_group(&mut self, priority: i32) -> Option<::Group> { + let _ = priority; + None + } +} + +/// A backend send stream that may join its [`Grouped`] session's send groups. +pub trait GroupedSend: backend::SendStream { + /// The backend's send group, [`Ungrouped`](poll::Ungrouped) when it has none. + type Group: poll::SendGroup; + + /// Move the stream into `group`. + fn set_group(&mut self, group: &Self::Group); + + /// Set the stream's full-width priority, saturated into the backend's by default. + fn set_order(&mut self, order: i64) { + self.set_priority(order.clamp(i32::MIN.into(), i32::MAX.into()) as i32); + } +} + /// A WebTransport session adapted to moq-net. #[derive(Clone)] pub struct Session(S, moq::Binding); @@ -53,7 +80,7 @@ impl Session { Self(session, binding) } } -impl + Clone + 'static> poll::Session for Session { +impl + Clone + 'static> poll::Session for Session { type SendStream = SendStream; type RecvStream = RecvStream; type Error = Error; @@ -77,6 +104,9 @@ impl + Clone + 'st .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } + fn send_group(&mut self, priority: i32) -> Option<::Group> { + Grouped::send_group(&mut self.0, priority) + } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) } @@ -104,16 +134,20 @@ impl + Clone + 'st } /// An outgoing WebTransport stream adapted to moq-net. pub struct SendStream(S); -impl poll::SendStream for SendStream { +impl poll::SendStream for SendStream { type Error = Error; + type Group = S::Group; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { backend::SendStream::poll_write(&mut self.0, cx, buf).map_err(Error) } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(&mut self.0, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i32) { - backend::SendStream::set_priority(&mut self.0, order) + fn set_priority(&mut self, order: i64) { + self.0.set_order(order) + } + fn set_group(&mut self, group: &Self::Group) { + GroupedSend::set_group(&mut self.0, group) } fn finish(&mut self) -> Result<(), Self::Error> { backend::SendStream::finish(&mut self.0).map_err(Error) @@ -203,6 +237,13 @@ mod tests { Poll::Ready(Err(Failure)) } } + impl GroupedSend for Stream { + type Group = poll::Ungrouped; + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } + } + impl Grouped for Backend {} struct Sample; impl wt::Stats for Sample { fn rtt(&self) -> Option { diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 0ebe75b7b1..6a3fb5b14e 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -77,6 +77,9 @@ impl + Clone + 'st .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } + fn send_group(&mut self, _: i32) -> Option { + None + } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) } @@ -106,15 +109,20 @@ impl + Clone + 'st pub struct SendStream(S); impl poll::SendStream for SendStream { type Error = Error; + type Group = poll::Ungrouped; fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { backend::SendStream::poll_write(&mut self.0, cx, buf).map_err(Error) } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(&mut self.0, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i32) { + fn set_priority(&mut self, order: i64) { + let order = order.clamp(i32::MIN.into(), i32::MAX.into()) as i32; backend::SendStream::set_priority(&mut self.0, order) } + fn set_group(&mut self, group: &Self::Group) { + match *group {} + } fn finish(&mut self) -> Result<(), Self::Error> { backend::SendStream::finish(&mut self.0).map_err(Error) } From e09b3b4905213ae1b85e4e2dd3c366b3baf9b524 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:54:09 -0700 Subject: [PATCH 14/37] quest: finish quic/scheduler Hierarchical send groups landed in moq-quic and every subscription uses one, so drop the quest and the links that waited on it. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/README.md | 3 - quest/m1/quic/datagram-priority.md | 7 +- quest/m1/quic/receive-memory.md | 1 - quest/m1/quic/scheduler.md | 126 ----------------------------- quest/m1/track-priority-scope.md | 5 -- quest/m2/quic-deadline.md | 2 - quest/m2/quic-qmux.md | 2 - quest/m3/cut-through/README.md | 1 - quest/m3/ladder/controller.md | 2 - 9 files changed, 1 insertion(+), 148 deletions(-) delete mode 100644 quest/m1/quic/scheduler.md diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index 5b00ba7b8e..eb7c81f269 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -72,9 +72,6 @@ cache limits remain separate; bitrate pacing composes with this memory cap. - [Bound message assembly](/quest/m1/quic/receive-memory.md) - complete headers and frames return credit, and receiver priority cancellation restores progress - [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - `RESET_STREAM_AT`, so a reset WebTransport stream still delivers its header -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - strict - subscription priority, fair buckets, and newest-first group order replace - the lossy scalar; retransmits follow the same order - [Schedule datagrams by subscription priority](/quest/m1/quic/datagram-priority.md) - datagrams join their subscription's send group instead of preempting every stream - [Relay peers get wider limits](/quest/m1/quic/peer-limits.md) - MAX_STREAMS diff --git a/quest/m1/quic/datagram-priority.md b/quest/m1/quic/datagram-priority.md index 7a00fcd1d4..a407090b20 100644 --- a/quest/m1/quic/datagram-priority.md +++ b/quest/m1/quic/datagram-priority.md @@ -30,7 +30,7 @@ Decided 2026-10-10 with the maintainer: datagram track starve streams up to the congestion window, which is what every stack does today. - **A datagram queue per send group.** Each send group (one per - subscription, from [the scheduler](/quest/m1/quic/scheduler.md)) owns a + subscription, from `moq-quic`'s `SendGroups`) owns a datagram queue. Datagram bytes spend the group's fair-share credit, so a datagram-heavy subscription buys no extra bandwidth over its equal-priority peers. The datagram send API takes @@ -103,11 +103,6 @@ on the browser's WebTransport, which cannot rank a datagram against a stream, so it ignores the handle and order and keeps its current best-effort send. Wire: none. -## Required - -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - supplies - the send groups and fair tier the datagram queues join - ## Related - [Scope track priority](/quest/m1/track-priority-scope.md) - owns what a diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md index f027c78296..f9cb5be0d8 100644 --- a/quest/m1/quic/receive-memory.md +++ b/quest/m1/quic/receive-memory.md @@ -93,6 +93,5 @@ separate guide quest is needed. ## Related - [Scope track priority](/quest/m1/track-priority-scope.md) - broadcast fairness domains and subscription priority semantics -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - share priority semantics without requiring sender-side recovery - [QUIC bitrate caps](/quest/m2/rate-quic.md) - pace credit grants inside the independently bounded memory allowance - [Relay peers get wider limits](/quest/m1/quic/peer-limits.md) - client and peer windows must both fit supported units diff --git a/quest/m1/quic/scheduler.md b/quest/m1/quic/scheduler.md deleted file mode 100644 index b7da42d3b2..0000000000 --- a/quest/m1/quic/scheduler.md +++ /dev/null @@ -1,126 +0,0 @@ -# [XL] Add hierarchical QUIC stream scheduling - -## Goal - -One QUIC connection expresses the MoQ scheduling hierarchy without packing it -into a scalar: higher-priority subscriptions preempt lower-priority ones, -backlogged subscriptions at the same priority receive equal bandwidth within -their scheduling domain, and each subscription serves its own group streams -newest-first. - -This completes [#699](https://github.com/moq-dev/moq/issues/699). In the -reported Alice and Bob case, two priority-4 subscriptions continue to make -progress at the same byte rate even when one has a deeper or faster-growing -group backlog. - -## Plan - -Add a generic send-group primitive to noq-proto. A group owns its -strict priority and fair-share scheduler state; each stream owns only its -order within that group. Prefer an owned group handle whose drop removes its -scheduler state and whose priority can be updated without walking every open -stream. If the backend-neutral WebTransport trait cannot carry a handle -without breaking object safety, allocate an opaque group ID from the session. -Do not accept caller-chosen IDs and do not compress the hierarchy into another -integer. - -Use byte-accounted deficit round robin, or an equivalent bounded-quantum -algorithm, between backlogged groups at equal priority. Round robin by stream -count is insufficient because audio, video, and data streams have different -sizes. Within the chosen group, order streams by the subscription's group order -(newest first by default). A blocked stream must not consume -the group's turn, and opening newer groups must not reset its accumulated -fair-share credit. - -Map conventions only at adapters. MoQ's model remains higher value first, the -IETF wire remains lower value first. Native QUIC and qmux use the full three -levels. Browsers (js/net and web-transport-wasm) never create send groups: -browser groups are flat and byte-fair, which would trade strict priority -between subscriptions (audio over video) for fairness nobody on a browser -session needs, so they keep the default group and pack priority and group -order into `sendOrder` (decided 2026-09-26 alongside Firefox 155's -send groups). - -Give every MoQ subscription one native send group. A SUBSCRIBE_UPDATE changes the -group priority atomically. Group streams use their position within the -subscription, never another subscription's sequence. Remove the session-wide -`lite::PriorityQueue` once every enabled backend has an honest implementation -or fallback. - -Add the scheduling-domain tier above subscriptions: on a shared session, -fairness is one deficit-round-robin bucket per domain, and a domain's -subscriptions share that bucket by their own priorities, so a tenant opening -more subscriptions never buys more bandwidth. The domain key and its scope are -[Scope track priority](/quest/m1/track-priority-scope.md)'s; this quest owns -the tier and the weighting. - -### Prototype and compare the hierarchy - -The delivered API is hierarchical send groups. Do not require a separately -published scalar-widening API or drop fairness as an intermediate release. -Prototype the byte-accounted scheduler in the fork. An existing Quinn prototype can -supply a workload baseline, but a recency field alone does not prove byte -fairness: differently sized writes must spend the group's byte credit, and -requeueing or adding streams must not reset that credit. - -Compare the hierarchy with the existing MoQ queue and two private benchmark -baselines: a scalar `[priority][group sequence]`, and a scalar containing only -priority. The first removes queue coordination but lets unrelated sequence -numbers compete across subscriptions; the second exposes per-stream fairness -but loses newest-first ordering within a subscription. Neither is the public -API this quest ships. - -Keep limitations explicit in the scalar benchmark. A 32-bit key leaves only -24 bits after track priority, so wraparound and sparse sequence values can -invert group order. Browser numeric precision and narrow backend urgency -fields impose different limits. Include these cases and compare audio/video -subscriptions with different group cadences; do not treat scalar ordering as -an exact equivalent of the hierarchy. - -Record queue/lock work, CPU, throughput, latency, and byte fairness on the -same congested workloads. Both equal-priority subscriptions must progress -while a higher-priority subscription preempts them and each subscription -sheds its own old backlog. Measure the full scope of trait and adapter changes -before publishing the API. - -Retransmissions follow the same hierarchy. `moq-quic` already re-queues a lost -range through the stream's priority (`StreamsState::retransmit`), so a lost -video range waits behind new audio at a higher priority. Keep -`retransmit_stream_priority` passing, and add a test proving a retransmission -is never starved indefinitely by an equal-priority group's new data (the -fair tier's byte credit covers retransmits too). - -Tests saturate the sender with differently sized audio and video groups and -prove byte fairness over a bounded window, strict preemption by a higher -priority, newest-first backlog shedding, dynamic priority updates, -blocked-stream handling, sequence wrap and sparse sequence values, and cleanup -on reset. This quest owns native QUIC proof and -reusable scheduling fixtures. [qmux](/quest/m2/quic-qmux.md) owns running those -fixtures through its record writer after adopting the scheduler; native -scheduler completion must not wait for that dependent integration. Preserve -working behavior on backends not yet migrated, and remove queue code only -where the new implementation makes it redundant. - -## Required - -- [Hard fork](/quest/m1/quic/fork/README.md) - lands in `moq-quic`, not the frozen fork - -## Closes - -- [#699](https://github.com/moq-dev/moq/issues/699) - close this issue when the - quest finishes - -## Related - -- [Schedule datagrams by subscription priority](/quest/m1/quic/datagram-priority.md) - - adds a datagram queue to each send group once this lands -- [moq#3320](https://github.com/moq-dev/moq/pull/3320) - removes the current - dense-rank queue from the wide scalar path and records why a scalar cannot - provide this fairness level -- [Ladder controller](/quest/m3/ladder/controller.md) - rendition priority is - a policy consumer of the same hierarchy -- [Scope track priority](/quest/m1/track-priority-scope.md) - owns the - priority semantics this mechanism realizes, including the scheduling-domain - scope and the priority type change; keep the ordering, not just the type -- [noq#816](https://github.com/n0-computer/noq/issues/816) - the send-group proposal to n0; this quest supersedes it -- [noq#817](https://github.com/n0-computer/noq/issues/817) - the 64-bit priority ask; send groups replace the scalar, but the per-stream order within a group must still hold a 64-bit group sequence diff --git a/quest/m1/track-priority-scope.md b/quest/m1/track-priority-scope.md index f43efa89d6..ea657358e0 100644 --- a/quest/m1/track-priority-scope.md +++ b/quest/m1/track-priority-scope.md @@ -119,11 +119,6 @@ Public API impact: `u8` priority fields become `i8` in every language, and in format, but the default byte moves from 127 to 128 on moq-lite and from 128 to 127 on IETF. Report both with the draft change. -## Required - -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - the fair - tier the broadcast domains and subscription send groups ride on - ## Related - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - owns the group diff --git a/quest/m2/quic-deadline.md b/quest/m2/quic-deadline.md index 145ae3ee3c..18c9b101bd 100644 --- a/quest/m2/quic-deadline.md +++ b/quest/m2/quic-deadline.md @@ -52,8 +52,6 @@ Decided in the 2026-09-30 audit: moved to m2. No m1 quest consumes it. ## Required - [Hard fork](/quest/m1/quic/fork/README.md) - the change lands in `moq-quic`, not the frozen fork -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - the - scheduler decides which stream's data a probe carries ## Related diff --git a/quest/m2/quic-qmux.md b/quest/m2/quic-qmux.md index 0d068f8e66..ad3a2324b1 100644 --- a/quest/m2/quic-qmux.md +++ b/quest/m2/quic-qmux.md @@ -81,8 +81,6 @@ the fork plan the same day kept it there. - [Hard fork](/quest/m1/quic/fork/README.md) - qmux drives `moq-quic` - [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - qmux reuses the extension's stream state rather than implementing reset locally -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - qmux must - expose the same scheduling contract as native QUIC ## Related diff --git a/quest/m3/cut-through/README.md b/quest/m3/cut-through/README.md index d555db078e..277b6938c2 100644 --- a/quest/m3/cut-through/README.md +++ b/quest/m3/cut-through/README.md @@ -76,6 +76,5 @@ stats field, which its quest documents inline. ## Related -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - orders streams; offset writes order ranges within one - [QoS](/quest/m1/qos/README.md) - the loss-delay counter follows its per-broadcast ingress row conventions - [Frame slots are cached for free](https://github.com/moq-dev/moq/pull/5239) - changes how the cache charges the in-flight frames a cut-through build would hold diff --git a/quest/m3/ladder/controller.md b/quest/m3/ladder/controller.md index 117a4e2f24..9f61396bc0 100644 --- a/quest/m3/ladder/controller.md +++ b/quest/m3/ladder/controller.md @@ -74,5 +74,3 @@ priority where the subscriber's order wins, and a custom ladder order. ## Related - [Encoder feedback](/quest/m3/stats-encoder-feedback.md) - viewer stalls for `moq import`; its `moq transcode --echo` leg, split out if revived, feeds this controller -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - supplies the - fair subscription buckets beneath this rendition policy From 21cac2b27435c4b2084018c98b0a3cb5be467cc6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 00:06:45 -0700 Subject: [PATCH 15/37] fix(tokio): wake draining waiters when accept empties the queue Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 5 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 38 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 9093082720..2a214d5466 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -821,6 +821,11 @@ impl Future for Accept<'_> { return Poll::Ready(None); } if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // A handed-out attempt no longer holds off `wait_all_draining`, and refusing, + // ignoring, or dropping it sends no event that would wake the waiter later. + if endpoint.all_draining() { + this.endpoint.inner.shared.all_draining.notify_waiters(); + } // Release the mutex lock on endpoint so cloning it doesn't deadlock drop(endpoint); let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 5d50d7a004..aae7054d83 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1355,6 +1355,44 @@ async fn close_refuses_queued_incoming() { )); } +/// Taking the last queued attempt and refusing it sends no event, so `accept` itself must wake a +/// parked `wait_all_draining`. +#[tokio::test] +async fn refused_incoming_wakes_all_draining_waiter() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, wake_counter) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + server.accept().await.unwrap().refuse(); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), draining).await.unwrap(); + + assert!( + timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .is_err() + ); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From c1f4cfd3ef7ad1c6ef7136e1d323ec272708148f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 07:47:22 -0700 Subject: [PATCH 16/37] fix(tokio): surface non-WouldBlock receive errors, cap moq_quic logs The server socket retried every failed recv, so a persistent error spun the endpoint task while tokio kept readiness asserted. Only WouldBlock retries now, matching the driver's tokio runtime. Also cap moq_quic at info like noq was, fix the driver doc example, and point the moq-uring README at the just recipe. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/log.rs | 1 + rs/moq-tokio/src/quic/backend.rs | 8 ++++++-- rs/moq-tokio/src/quic/driver/endpoint.rs | 4 ++-- rs/moq-uring/README.md | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index 5d8a5e4f19..6b4833f9d1 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,6 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) + .add_directive("moq_quic=info".parse()?) .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 666040d324..e743af6ea7 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -627,8 +627,12 @@ impl driver::AsyncUdpSocket for Socket { // Only the endpoint driver receives, so the one readiness waker is enough. loop { ready!(io.poll_recv_ready(cx))?; - if let Ok(res) = io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { - return Poll::Ready(Ok(res)); + match io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`; retrying any other error + // would spin, since `poll_recv_ready` completes at once and the recv fails again. + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} + Err(err) => return Poll::Ready(Err(err)), } } } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2a214d5466..eeaaea5e3d 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -59,8 +59,8 @@ impl Endpoint { /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other /// address. For example: /// - /// ``` - /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ```no_run + /// moq_tokio::quic::driver::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); /// ``` /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index db5a5a8aec..ec84787ea0 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -82,7 +82,7 @@ timers, and UDP socket can be built without QUIC. Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features quic +just rs uring ``` `moq-relay` enables it with `--features io-uring`. From 5f9b8d9ab4cf38e0773a49322687c594e9b8f9fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:42:37 -0700 Subject: [PATCH 17/37] fix(moq-net): gate GroupOf on the async feature Only the async publishers name it, so a model-only build flagged it as dead code. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/transport.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 4d367536ea..41dfbe31d8 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -325,6 +325,7 @@ pub mod poll { } /// The send group of a session's streams. + #[cfg(feature = "async")] pub(crate) type GroupOf = <::SendStream as SendStream>::Group; /// A transport's send group, from [`Session::send_group`]. From 437eee9cb489ddde24906b044633749f48188b7f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:19:09 -0700 Subject: [PATCH 18/37] refactor(transport): one set_priority, seal the group adapter traits Drop GroupedSend::set_order: the adapters use a single set_priority(i64), documented everywhere as the priority within the stream's send group, or session-wide when ungrouped. Grouped and GroupedSend leave the public API. pub(crate) is refused (E0446) because the public adapters' `type Group` projects through GroupedSend, so both are `pub` in a private module and never re-exported: other crates can neither name nor implement them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/schedule.rs | 2 +- rs/moq-net/src/coding/writer.rs | 16 ++++---- rs/moq-net/src/transport.rs | 7 ++-- rs/moq-quic/src/connection/streams/mod.rs | 2 +- rs/moq-tokio/src/quic/driver/send_stream.rs | 4 +- rs/moq-tokio/src/quic/web/send.rs | 7 ++-- rs/moq-tokio/src/transport.rs | 45 +++++---------------- rs/moq-tokio/src/transport/grouped.rs | 34 ++++++++++++++++ rs/moq-tokio/src/transport/owned.rs | 4 +- rs/moq-uring/src/quic/driver/stream.rs | 14 +++++-- rs/moq-uring/src/quic/web.rs | 7 ++-- rs/moq-uring/src/transport.rs | 7 ++-- rs/moq-uring/src/transport/adapter.rs | 16 +++++--- 13 files changed, 91 insertions(+), 74 deletions(-) create mode 100644 rs/moq-tokio/src/transport/grouped.rs diff --git a/rs/moq-net/benches/schedule.rs b/rs/moq-net/benches/schedule.rs index b2a2ba8508..0a531ce542 100644 --- a/rs/moq-net/benches/schedule.rs +++ b/rs/moq-net/benches/schedule.rs @@ -286,7 +286,7 @@ impl Sim { let Track { rate, cadence, step, .. } = self.tracks[track]; - if self.tick % cadence == 0 { + if self.tick.is_multiple_of(cadence) { let old = self.current[track]; let stream = self.streams.get_mut(&old).unwrap(); stream.finished = true; diff --git a/rs/moq-net/src/coding/writer.rs b/rs/moq-net/src/coding/writer.rs index c3a46c62d2..d0a52c4e38 100644 --- a/rs/moq-net/src/coding/writer.rs +++ b/rs/moq-net/src/coding/writer.rs @@ -201,15 +201,15 @@ impl Writer { std::future::poll_fn(|cx| self.poll_closed(cx)).await } - /// Set the stream's send order: streams with HIGHER values are transmitted first. + /// Set the priority within the stream's send group, or session-wide when ungrouped. /// - /// This is the transport trait's convention (matching W3C `sendOrder` and quinn's - /// scheduler) and the model's [`Subscription::priority`](crate::track::Subscription), - /// where higher values preempt lower ones. The lite priority queue's rank is the - /// opposite (0 = most urgent); rank holders convert via `PriorityHandle::send_order`. - /// In a send group, the order ranks the stream only against the group's others. - pub fn set_priority(&mut self, send_order: i64) { - self.stream.as_mut().unwrap().set_priority(send_order); + /// Streams with HIGHER values are transmitted first. This is the transport trait's + /// convention (matching W3C `sendOrder` and quinn's scheduler) and the model's + /// [`Subscription::priority`](crate::track::Subscription), where higher values preempt + /// lower ones. The lite priority queue's rank is the opposite (0 = most urgent); rank + /// holders convert via `PriorityHandle::send_order`. + pub fn set_priority(&mut self, priority: i64) { + self.stream.as_mut().unwrap().set_priority(priority); } /// Move the stream into a send group from its session. diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 41dfbe31d8..288ab796ad 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -383,16 +383,15 @@ pub mod poll { Poll::Ready(Ok(size)) } - /// Set the stream's priority. + /// Set the priority within the stream's send group, or session-wide when ungrouped. /// /// Streams with higher values will be sent first, but are not guaranteed to /// arrive first. This matches the W3C WebTransport `sendOrder` convention (and - /// quinn's scheduler). In a send group, the priority orders the stream only against - /// the group's other streams. + /// quinn's scheduler). /// /// Backends with a narrower priority saturate or approximate it, so treat the /// ordering as best-effort. - fn set_priority(&mut self, order: i64); + fn set_priority(&mut self, priority: i64); /// Move the stream into `group`, which belongs to this stream's session. fn set_group(&mut self, group: &Self::Group); diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index 5348671233..d419c5b333 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -355,7 +355,7 @@ impl<'a> SendStream<'a> { Ok(()) } - /// Set the priority of a stream within its send group + /// Set the priority within the stream's send group, or session-wide when ungrouped /// /// Higher values go first; equal values share bytes round robin. A queued stream moves to /// its new priority the next time it is requeued, which is after every frame when diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index d430b5f5a2..7566d80792 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -214,15 +214,13 @@ impl SendStream { Ok(()) } - /// Set the priority of the send stream + /// Set the priority within the stream's send group, or session-wide when ungrouped /// /// Every send stream has an initial priority of 0. Locally buffered data from streams with /// higher priority will be transmitted before data from streams with lower priority. Changing /// the priority of a stream with pending data may only take effect after that data has been /// transmitted. Using many different priority levels per connection may have a negative /// impact on performance. - /// - /// In a [`SendGroup`], the priority orders the stream only against the group's other streams. pub fn set_priority(&self, priority: i64) -> Result<(), ClosedStream> { let mut conn = self.conn.state.lock("SendStream::set_priority"); conn.inner.send_stream(self.stream).set_priority(priority)?; diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 27e992cd72..bf72fa2b6b 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -92,10 +92,9 @@ impl SendStream { self.stream.finish().map_err(Into::into) } - /// Set the stream's priority, within its send group if it joined one. See - /// [`crate::quic::driver::SendStream::set_priority`]. - pub fn set_priority(&self, order: i64) -> Result<(), ClosedStream> { - self.stream.set_priority(order).map_err(Into::into) + /// Set the priority within the stream's send group, or session-wide when ungrouped. + pub fn set_priority(&self, priority: i64) -> Result<(), ClosedStream> { + self.stream.set_priority(priority).map_err(Into::into) } /// Get the stream's priority. See [`crate::quic::driver::SendStream::priority`]. diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index f02c90f59d..33d7e27fdd 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -16,7 +16,9 @@ use bytes::Bytes; use futures::FutureExt; use web_transport_trait::poll as wt_poll; +mod grouped; mod owned; +pub(crate) use grouped::{Grouped, GroupedSend}; pub use owned::Error; /// The network transport carrying a MoQ session, on either side of it. @@ -57,33 +59,6 @@ impl std::fmt::Display for Transport { } } -/// A backend session whose send streams may join send groups, as native QUIC's do. -/// -/// The async interface has no groups, so moq-net's -/// [`send_group`](moq_net::transport::poll::Session::send_group) reaches a backend through -/// this. A backend without groups implements it empty. -pub trait Grouped: web_transport_trait::Session { - /// Open a send group at `priority`, or `None` when the backend has none. - fn send_group(&self, priority: i32) -> Option<::Group> { - let _ = priority; - None - } -} - -/// A backend send stream that may join its [`Grouped`] session's send groups. -pub trait GroupedSend: web_transport_trait::SendStream { - /// The backend's send group, [`Ungrouped`](moq_net::transport::poll::Ungrouped) when it has none. - type Group: moq_net::transport::poll::SendGroup + Send + Sync; - - /// Move the stream into `group`. - fn set_group(&mut self, group: &Self::Group); - - /// Set the stream's full-width priority, saturated into the async interface's by default. - fn set_order(&mut self, order: i64) { - self.set_priority(order.clamp(i32::MIN.into(), i32::MAX.into()) as i32); - } -} - /// A stored in-flight operation future. Native transports are Send, so the /// plain boxed flavor suffices. type OpBox = futures::future::BoxFuture<'static, T>; @@ -329,8 +304,8 @@ impl SendStream { if let Some(group) = self.group.take() { stream.set_group(&group); } - if let Some(order) = self.priority.take() { - stream.set_order(order); + if let Some(priority) = self.priority.take() { + GroupedSend::set_priority(stream, priority); } if let Some(code) = self.reset.take() { self.finish = false; @@ -346,16 +321,16 @@ impl SendStream { impl SendStream { /// Set the full-width priority, deferred while an operation holds the stream. - fn prioritize(&mut self, order: i64) { + fn prioritize(&mut self, priority: i64) { match self.state.get_mut().unwrap().as_mut() { - Some(SendState::Idle(stream)) => stream.set_order(order), + Some(SendState::Idle(stream)) => GroupedSend::set_priority(stream, priority), // A finished stream still retransmits under its priority until the FIN is // acknowledged, so reclaim it from the watch rather than wait that out. Some(SendState::Closing(_)) => { - self.priority = Some(order); + self.priority = Some(priority); self.reclaim(); } - _ => self.priority = Some(order), + _ => self.priority = Some(priority), } } @@ -895,8 +870,8 @@ impl GroupedSend for crate::quic::web::SendStream { Self::set_group(self, Some(group)).ok(); } - fn set_order(&mut self, order: i64) { - Self::set_priority(self, order).ok(); + fn set_priority(&mut self, priority: i64) { + crate::quic::web::SendStream::set_priority(self, priority).ok(); } } diff --git a/rs/moq-tokio/src/transport/grouped.rs b/rs/moq-tokio/src/transport/grouped.rs new file mode 100644 index 0000000000..4273dc29da --- /dev/null +++ b/rs/moq-tokio/src/transport/grouped.rs @@ -0,0 +1,34 @@ +//! Send-group hooks for the async backends. +//! +//! The traits are `pub` in this private module, so other crates can neither name nor +//! implement them: `pub(crate)` is refused (E0446) because the public adapter's +//! `type Group` projects through [`GroupedSend`]. + +/// A backend session whose send streams may join send groups, as native QUIC's do. +/// +/// The async interface has no groups, so moq-net's +/// [`send_group`](moq_net::transport::poll::Session::send_group) reaches a backend through +/// this. A backend without groups implements it empty. +pub trait Grouped: web_transport_trait::Session { + /// Open a send group at `priority`, or `None` when the backend has none. + fn send_group(&self, priority: i32) -> Option<::Group> { + let _ = priority; + None + } +} + +/// A backend send stream that may join its [`Grouped`] session's send groups. +pub trait GroupedSend: web_transport_trait::SendStream { + /// The backend's send group, [`Ungrouped`](moq_net::transport::poll::Ungrouped) when it has none. + type Group: moq_net::transport::poll::SendGroup + Send + Sync; + + /// Move the stream into `group`. + fn set_group(&mut self, group: &Self::Group); + + /// Set the priority within the stream's send group, or session-wide when ungrouped. + /// + /// Saturated into the async interface's narrower priority by default. + fn set_priority(&mut self, priority: i64) { + web_transport_trait::SendStream::set_priority(self, priority.clamp(i32::MIN.into(), i32::MAX.into()) as i32); + } +} diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 037bd1b5ab..1ae9d06b25 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -97,8 +97,8 @@ impl poll::SendStream for super::SendStream fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(self, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i64) { - self.prioritize(order) + fn set_priority(&mut self, priority: i64) { + self.prioritize(priority) } fn set_group(&mut self, group: &Self::Group) { self.join(group) diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index f20d3f213a..e9b97bd2eb 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -70,10 +70,16 @@ impl SendStream { n } - /// Set the stream's priority, which orders it against the other streams - /// of its send group, or of the ungrouped streams. Higher goes first. - pub fn set_priority(&mut self, order: i64) { - let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); + /// Set the priority within the stream's send group, or session-wide when ungrouped. + /// + /// Higher goes first. + pub fn set_priority(&mut self, priority: i64) { + let _ = self + .shared + .conn + .borrow_mut() + .send_stream(self.id) + .set_priority(priority); } /// Move the stream into a send group, or with `None` back to the diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 5ab11cdf0e..7f026caad0 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -834,10 +834,9 @@ impl SendStream { } } - /// Set the stream's priority within its send group. See - /// [`super::SendStream::set_priority`]. - pub fn set_priority(&mut self, order: i64) { - self.inner.set_priority(order); + /// Set the priority within the stream's send group, or session-wide when ungrouped. + pub fn set_priority(&mut self, priority: i64) { + self.inner.set_priority(priority); } /// Move the stream into a send group, or with `None` back out. See diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index cf74d4da25..9df40de448 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -1,6 +1,7 @@ //! Adapt backend poll sessions to moq-net without changing thread affinity. mod adapter; -pub use adapter::{Error, Grouped, GroupedSend, RecvStream, SendStream, Session}; +pub use adapter::{Error, RecvStream, SendStream, Session}; +use adapter::{Grouped, GroupedSend}; use crate::quic; @@ -17,8 +18,8 @@ impl GroupedSend for quic::web::SendStream { quic::web::SendStream::set_group(self, Some(group)); } - fn set_order(&mut self, order: i64) { - quic::web::SendStream::set_priority(self, order); + fn set_priority(&mut self, priority: i64) { + quic::web::SendStream::set_priority(self, priority); } } diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 1e073df907..3e5b01b059 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -46,6 +46,10 @@ impl moq::Stats for Stats { } /// A backend session whose send streams may join send groups, as native QUIC's do. /// +/// `pub` only because `pub(crate)` is refused (E0446): the adapter's `type Group` projects +/// through [`GroupedSend`]. The module is private and the trait is not re-exported, so +/// other crates can neither name nor implement it. +/// /// The backend interface has no groups, so moq-net's /// [`send_group`](poll::Session::send_group) reaches a backend through this. A backend /// without groups implements it empty. @@ -65,9 +69,11 @@ pub trait GroupedSend: backend::SendStream { /// Move the stream into `group`. fn set_group(&mut self, group: &Self::Group); - /// Set the stream's full-width priority, saturated into the backend's by default. - fn set_order(&mut self, order: i64) { - self.set_priority(order.clamp(i32::MIN.into(), i32::MAX.into()) as i32); + /// Set the priority within the stream's send group, or session-wide when ungrouped. + /// + /// Saturated into the backend's narrower priority by default. + fn set_priority(&mut self, priority: i64) { + backend::SendStream::set_priority(self, priority.clamp(i32::MIN.into(), i32::MAX.into()) as i32); } } @@ -143,8 +149,8 @@ impl poll::SendStream for SendStream { fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { backend::SendStream::poll_write_buf(&mut self.0, cx, buf).map_err(Error) } - fn set_priority(&mut self, order: i64) { - self.0.set_order(order) + fn set_priority(&mut self, priority: i64) { + GroupedSend::set_priority(&mut self.0, priority) } fn set_group(&mut self, group: &Self::Group) { GroupedSend::set_group(&mut self.0, group) From 14c7fab1a58a18231005e776d6e5ec882988a175 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:51:57 -0700 Subject: [PATCH 19/37] quest: claim m1/quic/datagram-priority Co-Authored-By: Claude Opus 5.5 From 188a508b9d2ef09ff4d4e7dca50f807e74a383a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:02:32 -0700 Subject: [PATCH 20/37] refactor(transport): send-group priority is i64, like stream priority moq-quic SendGroups::{open, set_priority, priority}, the tokio and uring driver handles and send_group, and moq-net's poll::Session::send_group and poll::SendGroup::set_priority now take i64. Group and stream priorities share one width, so no layer converts between them. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/transport/adapter.rs | 2 +- rs/moq-net/src/client.rs | 4 ++-- rs/moq-net/src/ietf/adapter.rs | 2 +- rs/moq-net/src/lite/test_transport.rs | 14 +++++++------- rs/moq-net/src/server.rs | 2 +- rs/moq-net/src/transport.rs | 6 +++--- rs/moq-net/src/transport/memory.rs | 2 +- rs/moq-net/tests/support/mock.rs | 2 +- rs/moq-quic/src/connection/streams/mod.rs | 6 +++--- rs/moq-quic/src/connection/streams/saturated.rs | 4 ++-- rs/moq-quic/src/connection/streams/schedule.rs | 14 +++++++------- rs/moq-tokio/src/quic/driver/connection.rs | 2 +- rs/moq-tokio/src/quic/driver/send_group.rs | 6 +++--- rs/moq-tokio/src/quic/web/session.rs | 2 +- rs/moq-tokio/src/transport.rs | 6 +++--- rs/moq-tokio/src/transport/grouped.rs | 2 +- rs/moq-tokio/src/transport/owned.rs | 2 +- rs/moq-uring/src/quic/driver/connection.rs | 2 +- rs/moq-uring/src/quic/driver/group.rs | 6 +++--- rs/moq-uring/src/quic/web.rs | 2 +- rs/moq-uring/src/transport.rs | 4 ++-- rs/moq-uring/src/transport/adapter.rs | 4 ++-- rs/moq-wasm/src/transport/adapter.rs | 2 +- 23 files changed, 49 insertions(+), 49 deletions(-) diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 6a3fb5b14e..bb2e27bfdf 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -77,7 +77,7 @@ impl + Clone + 'st .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 632f5bf09d..1a9d3a8b9e 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -620,7 +620,7 @@ mod tests { Poll::Pending } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } @@ -1015,7 +1015,7 @@ mod tests { }) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 64715ff93e..dc7a2d80aa 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -1172,7 +1172,7 @@ impl crate::transport::poll::Session for Con .map(|r| r.map(AdapterRecv::Real).map_err(|_| crate::Error::Closed)) } - fn send_group(&mut self, priority: i32) -> Option<::Group> { + fn send_group(&mut self, priority: i64) -> Option<::Group> { self.inner.send_group(priority) } diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 75a9e307ac..5c32cef64c 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -52,7 +52,7 @@ pub struct Log { priorities: Arc>>, /// Hands each send group its id. group_ids: Arc, - groups: Arc>>, + groups: Arc>>, joins: Arc>>, orders: Arc>>, } @@ -101,7 +101,7 @@ impl Log { /// Every send group's priority, as (group, priority) in call order: its opening, then /// each move. - pub fn groups(&self) -> Vec<(usize, i32)> { + pub fn groups(&self) -> Vec<(usize, i64)> { self.groups.lock().unwrap().clone() } @@ -157,7 +157,7 @@ pub struct SinkGroup { } impl SinkGroup { - fn open(log: Log, priority: i32) -> Self { + fn open(log: Log, priority: i64) -> Self { let id = log.group_ids.fetch_add(1, Ordering::Relaxed); log.groups.lock().unwrap().push((id, priority)); Self { id, log } @@ -165,7 +165,7 @@ impl SinkGroup { } impl poll::SendGroup for SinkGroup { - fn set_priority(&mut self, priority: i32) { + fn set_priority(&mut self, priority: i64) { self.log.groups.lock().unwrap().push((self.id, priority)); } } @@ -408,7 +408,7 @@ impl poll::Session for DeadStreamSession { Poll::Ready(Ok(SinkSend::new(self.log.clone()))) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } @@ -608,7 +608,7 @@ impl poll::Session for SinkSession { Poll::Ready(Ok(send)) } - fn send_group(&mut self, priority: i32) -> Option { + fn send_group(&mut self, priority: i64) -> Option { self.groups.then(|| SinkGroup::open(self.log.clone(), priority)) } @@ -933,7 +933,7 @@ impl poll::Session for ScriptedSession { Poll::Ready(Ok(SinkSend::new(self.log.clone()))) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 8bcc522988..4e3b2cdbfb 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1065,7 +1065,7 @@ mod tests { ) -> std::task::Poll> { std::task::Poll::Pending } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 288ab796ad..cdd084eee1 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -162,7 +162,7 @@ pub mod poll { /// priority get an equal share of bytes however many streams each has queued. Streams /// outside every group go before all of them. The group is released once its last /// handle is dropped and its streams finish. - fn send_group(&mut self, priority: i32) -> Option<::Group>; + fn send_group(&mut self, priority: i64) -> Option<::Group>; /// Poll to send a datagram over the network. /// @@ -333,7 +333,7 @@ pub mod poll { /// Clones share the group. pub trait SendGroup: Clone + 'static { /// Change the group's priority: higher preempts lower. - fn set_priority(&mut self, priority: i32); + fn set_priority(&mut self, priority: i64); } /// The send group of a transport that has none, which cannot be constructed. @@ -341,7 +341,7 @@ pub mod poll { pub enum Ungrouped {} impl SendGroup for Ungrouped { - fn set_priority(&mut self, _: i32) { + fn set_priority(&mut self, _: i64) { match *self {} } } diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 41ace4c5f5..c9229b31d8 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -294,7 +294,7 @@ impl poll::Session for Session { Poll::Ready(opened.map(|()| (send, recv))) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index d3153c2f2c..d98423ff31 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -596,7 +596,7 @@ impl poll::Session for MockSession { Poll::Ready(Ok(send)) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index dd4b6e1872..49322db340 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -433,7 +433,7 @@ impl SendGroups<'_> { /// /// # Panics /// - when `domain` was opened on another connection - pub fn open(&mut self, priority: i32, domain: Option<&SendDomain>) -> SendGroup { + pub fn open(&mut self, priority: i64, domain: Option<&SendDomain>) -> SendGroup { let tag = self.state.tag; let domain = domain.map(|d| d.index(tag)); SendGroup { @@ -455,7 +455,7 @@ impl SendGroups<'_> { /// /// # Panics /// - when `group` was opened on another connection - pub fn set_priority(&mut self, group: &SendGroup, priority: i32) { + pub fn set_priority(&mut self, group: &SendGroup, priority: i64) { let index = group.index(self.state.tag); self.state.pending.set_group_priority(index, priority); } @@ -464,7 +464,7 @@ impl SendGroups<'_> { /// /// # Panics /// - when `group` was opened on another connection - pub fn priority(&self, group: &SendGroup) -> i32 { + pub fn priority(&self, group: &SendGroup) -> i64 { self.state .pending .group_priority(group.index(self.state.tag)) diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs index 4a3c511de9..53ef61c856 100644 --- a/rs/moq-quic/src/connection/streams/saturated.rs +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -67,7 +67,7 @@ impl Sender { } } - fn group(&mut self, priority: i32) -> SendGroup { + fn group(&mut self, priority: i64) -> SendGroup { self.groups().open(priority, None) } @@ -345,7 +345,7 @@ fn retransmits_are_not_starved_by_equal_groups() { #[test] fn ungrouped_streams_preempt_groups() { let mut sender = Sender::new(); - let group = sender.group(i32::MAX); + let group = sender.group(i64::MAX); let grouped = sender.open(Some(&group), i64::MAX, 10_000); let control = sender.open(None, i64::MIN, 10_000); diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs index 1458cfa9de..7d31645298 100644 --- a/rs/moq-quic/src/connection/streams/schedule.rs +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -87,7 +87,7 @@ impl Queue { #[derive(Debug)] struct Group { domain: usize, - priority: i32, + priority: i64, queue: Queue, /// Bytes this group may still send in its current turn; negative after an overdraw. deficit: i64, @@ -108,7 +108,7 @@ struct Group { struct Domain { /// The round robin of groups at each priority. Only the highest level with a backlogged /// group is served. - levels: BTreeMap>, + levels: BTreeMap>, deficit: i64, /// Whether an entry for this domain sits in the domain round robin, as for groups. ringed: bool, @@ -194,7 +194,7 @@ impl Schedule { } /// Create a send group in `domain`, or the default domain. - pub(super) fn open_group(&mut self, priority: i32, domain: Option) -> usize { + pub(super) fn open_group(&mut self, priority: i64, domain: Option) -> usize { let domain = domain.unwrap_or(DEFAULT_DOMAIN); let state = &mut self.domains[domain]; assert!(!state.closed, "group opened in a closed domain"); @@ -217,12 +217,12 @@ impl Schedule { self.free_group(group); } - pub(super) fn group_priority(&self, group: usize) -> i32 { + pub(super) fn group_priority(&self, group: usize) -> i64 { self.groups[group].priority } /// Move a group to another priority level, keeping its credit. - pub(super) fn set_group_priority(&mut self, group: usize, priority: i32) { + pub(super) fn set_group_priority(&mut self, group: usize, priority: i64) { let state = &mut self.groups[group]; if state.priority == priority { return; @@ -497,7 +497,7 @@ impl Schedule { } } - fn unring_group(domain: &mut Domain, group: usize, priority: i32) { + fn unring_group(domain: &mut Domain, group: usize, priority: i64) { let level = domain .levels .get_mut(&priority) @@ -574,7 +574,7 @@ mod tests { #[test] fn ungrouped_streams_go_first() { let mut sim = Sim::new(); - let group = sim.schedule.open_group(i32::MAX, None); + let group = sim.schedule.open_group(i64::MAX, None); let grouped = sim.stream(Some(group), i64::MAX, 100); let ungrouped = sim.stream(None, i64::MIN, 100); assert_eq!(sim.step(1000), Some(ungrouped)); diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 55bfcbeec3..d5e80f4bfd 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -349,7 +349,7 @@ impl Connection { /// Open a send group at `priority`, for streams to join with [`SendStream::set_group`]. /// /// [`SendStream::set_group`]: crate::quic::driver::SendStream::set_group - pub fn send_group(&self, priority: i32) -> SendGroup { + pub fn send_group(&self, priority: i64) -> SendGroup { SendGroup::new(self.0.0.clone(), priority) } diff --git a/rs/moq-tokio/src/quic/driver/send_group.rs b/rs/moq-tokio/src/quic/driver/send_group.rs index c68b5c4acc..41cd2e0ab4 100644 --- a/rs/moq-tokio/src/quic/driver/send_group.rs +++ b/rs/moq-tokio/src/quic/driver/send_group.rs @@ -22,7 +22,7 @@ struct Inner { } impl SendGroup { - pub(crate) fn new(conn: Arc, priority: i32) -> Self { + pub(crate) fn new(conn: Arc, priority: i64) -> Self { let group = conn .state .lock("SendGroup::new") @@ -36,13 +36,13 @@ impl SendGroup { } /// Change the group's priority, moving any data it has queued along with it - pub fn set_priority(&self, priority: i32) { + pub fn set_priority(&self, priority: i64) { let mut conn = self.0.conn.state.lock("SendGroup::set_priority"); conn.inner.send_groups().set_priority(self.id(), priority); } /// Get the group's priority - pub fn priority(&self) -> i32 { + pub fn priority(&self) -> i64 { let mut conn = self.0.conn.state.lock("SendGroup::priority"); conn.inner.send_groups().priority(self.id()) } diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index b55cbe1437..c984e1b045 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -374,7 +374,7 @@ impl Session { /// /// See [`crate::quic::driver::SendGroup`]. The HTTP/3 streams stay outside every group, /// so they go first. - pub fn send_group(&self, priority: i32) -> crate::quic::driver::SendGroup { + pub fn send_group(&self, priority: i64) -> crate::quic::driver::SendGroup { self.conn.send_group(priority) } diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 33d7e27fdd..06e6b0652e 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -856,7 +856,7 @@ impl Drop for RecvStream { #[cfg(feature = "quic")] impl Grouped for crate::quic::web::Session { - fn send_group(&self, priority: i32) -> Option { + fn send_group(&self, priority: i64) -> Option { Some(Self::send_group(self, priority)) } } @@ -877,7 +877,7 @@ impl GroupedSend for crate::quic::web::SendStream { #[cfg(feature = "quic")] impl moq_net::transport::poll::SendGroup for crate::quic::driver::SendGroup { - fn set_priority(&mut self, priority: i32) { + fn set_priority(&mut self, priority: i64) { Self::set_priority(self, priority) } } @@ -996,7 +996,7 @@ mod tests { struct FakeGroup(u32); impl moq_net::transport::poll::SendGroup for FakeGroup { - fn set_priority(&mut self, _: i32) {} + fn set_priority(&mut self, _: i64) {} } impl GroupedSend for FakeSend { diff --git a/rs/moq-tokio/src/transport/grouped.rs b/rs/moq-tokio/src/transport/grouped.rs index 4273dc29da..acdf01cddf 100644 --- a/rs/moq-tokio/src/transport/grouped.rs +++ b/rs/moq-tokio/src/transport/grouped.rs @@ -11,7 +11,7 @@ /// this. A backend without groups implements it empty. pub trait Grouped: web_transport_trait::Session { /// Open a send group at `priority`, or `None` when the backend has none. - fn send_group(&self, priority: i32) -> Option<::Group> { + fn send_group(&self, priority: i64) -> Option<::Group> { let _ = priority; None } diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 1ae9d06b25..0c32203738 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -60,7 +60,7 @@ impl> poll::Session fn poll_open_bi(&mut self, cx: &mut Context<'_>) -> Poll, Self::Error>> { backend::Session::poll_open_bi(self, cx).map_err(Error) } - fn send_group(&mut self, priority: i32) -> Option<::Group> { + fn send_group(&mut self, priority: i64) -> Option<::Group> { self.session.send_group(priority) } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 75a8cb4bb0..6f3721eef9 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -317,7 +317,7 @@ impl Connection { /// Open a send group at `priority`, for streams to join with /// [`SendStream::set_group`](super::SendStream::set_group). - pub fn send_group(&self, priority: i32) -> super::SendGroup { + pub fn send_group(&self, priority: i64) -> super::SendGroup { super::SendGroup::new(self.shared.clone(), priority) } diff --git a/rs/moq-uring/src/quic/driver/group.rs b/rs/moq-uring/src/quic/driver/group.rs index 1a3bbe5d38..a90b1e2e97 100644 --- a/rs/moq-uring/src/quic/driver/group.rs +++ b/rs/moq-uring/src/quic/driver/group.rs @@ -22,7 +22,7 @@ struct Inner { } impl SendGroup { - pub(crate) fn new(shared: Shared, priority: i32) -> Self { + pub(crate) fn new(shared: Shared, priority: i64) -> Self { let group = shared.conn.borrow_mut().send_groups().open(priority, None); Self(Rc::new(Inner { shared, @@ -31,7 +31,7 @@ impl SendGroup { } /// Change the group's priority, moving any data it has queued along with it. - pub fn set_priority(&self, priority: i32) { + pub fn set_priority(&self, priority: i64) { self.0 .shared .conn @@ -41,7 +41,7 @@ impl SendGroup { } /// The group's priority. - pub fn priority(&self) -> i32 { + pub fn priority(&self) -> i64 { self.0.shared.conn.borrow_mut().send_groups().priority(self.id()) } diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 7f026caad0..1170e6bb56 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -442,7 +442,7 @@ impl Session { /// Open a send group at `priority`, for streams to join with /// [`SendStream::set_group`]. The HTTP/3 streams stay outside every /// group, so they go first. - pub fn send_group(&self, priority: i32) -> super::SendGroup { + pub fn send_group(&self, priority: i64) -> super::SendGroup { self.conn.send_group(priority) } diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 9df40de448..f4492304b3 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -6,7 +6,7 @@ use adapter::{Grouped, GroupedSend}; use crate::quic; impl Grouped for quic::web::Session { - fn send_group(&mut self, priority: i32) -> Option { + fn send_group(&mut self, priority: i64) -> Option { Some(quic::web::Session::send_group(self, priority)) } } @@ -24,7 +24,7 @@ impl GroupedSend for quic::web::SendStream { } impl moq_net::transport::poll::SendGroup for quic::SendGroup { - fn set_priority(&mut self, priority: i32) { + fn set_priority(&mut self, priority: i64) { quic::SendGroup::set_priority(self, priority) } } diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 3e5b01b059..a09b5e417a 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -55,7 +55,7 @@ impl moq::Stats for Stats { /// without groups implements it empty. pub trait Grouped: backend::Session { /// Open a send group at `priority`, or `None` when the backend has none. - fn send_group(&mut self, priority: i32) -> Option<::Group> { + fn send_group(&mut self, priority: i64) -> Option<::Group> { let _ = priority; None } @@ -110,7 +110,7 @@ impl + Clone + 'static> pol .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } - fn send_group(&mut self, priority: i32) -> Option<::Group> { + fn send_group(&mut self, priority: i64) -> Option<::Group> { Grouped::send_group(&mut self.0, priority) } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 6a3fb5b14e..bb2e27bfdf 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -77,7 +77,7 @@ impl + Clone + 'st .map_err(Error) .map(|res| res.map(|(send, recv)| (SendStream(send), RecvStream(recv)))) } - fn send_group(&mut self, _: i32) -> Option { + fn send_group(&mut self, _: i64) -> Option { None } fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { From 085a2c62dfca740cf28fc5c4419a3f4a58317e02 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:11:47 -0700 Subject: [PATCH 21/37] feat(quic)!: schedule datagrams in their send group Each send group queues datagrams beside its streams, ordered by the same priority and charged against the group's byte credit, so a datagram no longer goes ahead of every STREAM frame. Eviction is domain-aware and sending never blocks. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/datagrams.rs | 193 ++---- rs/moq-quic/src/connection/mod.rs | 37 +- rs/moq-quic/src/connection/streams/mod.rs | 43 +- .../src/connection/streams/saturated.rs | 251 +++++++- .../src/connection/streams/schedule.rs | 570 ++++++++++++++++-- rs/moq-quic/src/connection/streams/state.rs | 86 ++- rs/moq-quic/src/tests/mod.rs | 117 ++-- 7 files changed, 950 insertions(+), 347 deletions(-) diff --git a/rs/moq-quic/src/connection/datagrams.rs b/rs/moq-quic/src/connection/datagrams.rs index d116d8c81a..6cd3222f72 100644 --- a/rs/moq-quic/src/connection/datagrams.rs +++ b/rs/moq-quic/src/connection/datagrams.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use thiserror::Error; use tracing::{debug, trace}; -use super::{Connection, Event}; +use super::{Connection, SendGroup}; use crate::{ TransportError, frame::{Datagram, FrameStruct}, @@ -16,55 +16,57 @@ pub struct Datagrams<'a> { } impl Datagrams<'_> { - /// Queue an unreliable, unordered datagram for immediate transmission + /// Queue an unreliable, unordered datagram in `group` at `order` /// - /// If `drop` is true, previously queued datagrams which are still unsent may be discarded to - /// make space for this datagram, in order of oldest to newest. If `drop` is false, and there - /// isn't enough space due to previously queued datagrams, this function will return - /// `SendDatagramError::Blocked`. `Event::DatagramsUnblocked` will be emitted once datagrams - /// have been sent. + /// The datagram is scheduled like one of the group's streams at priority `order`: it waits + /// behind higher orders, goes ahead of a stream of equal order, and spends the group's + /// share of bytes. Without a group, it is ordered among the ungrouped streams, which go + /// before every group. + /// + /// Sending never blocks. While queued datagrams overflow + /// [`TransportConfig::datagram_send_buffer_size`](crate::TransportConfig::datagram_send_buffer_size), + /// the domain holding the most datagram bytes drops the oldest datagram of its + /// lowest-priority group, and this datagram is dropped instead once it is the one to go. /// /// Returns `Err` iff a `len`-byte datagram cannot currently be sent. - pub fn send(&mut self, data: Bytes, drop: bool) -> Result<(), SendDatagramError> { + /// + /// # Panics + /// - when `group` was opened on another connection + pub fn send( + &mut self, + data: Bytes, + group: Option<&SendGroup>, + order: i64, + ) -> Result<(), SendDatagramError> { if self.conn.config.datagram_receive_buffer_size.is_none() { return Err(SendDatagramError::Disabled); } let max = self .max_size() .ok_or(SendDatagramError::UnsupportedByPeer)?; - let send_buffer_size = self.conn.config.datagram_send_buffer_size; - if data.len() > Ord::min(max, send_buffer_size) { + let budget = self.conn.config.datagram_send_buffer_size; + let datagram = Datagram { data }; + if datagram.data.len() > max || datagram.size(true) > budget { return Err(SendDatagramError::TooLarge); } - if drop { - self.conn - .datagrams - .make_space_for(data.len(), send_buffer_size); - } else if !self + if !self .conn - .datagrams - .has_send_buffer_space(data.len(), send_buffer_size) + .streams + .queue_datagram(datagram, group, order, budget) { - self.conn.datagrams.send_blocked = true; - return Err(SendDatagramError::Blocked(data)); + trace!("dropping outgoing datagram"); } - self.conn.datagrams.outgoing.push_back(Datagram { data }); Ok(()) } - /// Discard queued datagrams that no longer fit the active path, waking blocked senders. + /// Discard queued datagrams that no longer fit the active path. pub(super) fn drop_oversized(&mut self) { let Some(max_datagram_size) = self.max_size() else { return; }; - if !self.conn.datagrams.drop_oversized(max_datagram_size) - || !self.conn.datagrams.send_blocked - { - return; - } - - self.conn.datagrams.send_blocked = false; - self.conn.events.push_back(Event::DatagramsUnblocked); + self.conn + .streams + .drop_oversized_datagrams(max_datagram_size); } /// Compute the maximum size of datagrams that may passed to `send_datagram` @@ -100,21 +102,20 @@ impl Datagrams<'_> { /// Bytes available in the outgoing datagram buffer /// /// When greater than zero, [`send`](Self::send)ing a datagram of at most this size is - /// guaranteed not to cause older datagrams to be dropped. + /// guaranteed not to cause queued datagrams to be dropped. pub fn send_buffer_space(&self) -> usize { self.conn .config .datagram_send_buffer_size - .saturating_sub(self.conn.datagrams.outgoing.payload_bytes) - .saturating_sub(size_of::()) + .saturating_sub(self.conn.streams.datagram_bytes()) + .saturating_sub(Datagram::SIZE_BOUND) } } +/// Incoming datagrams; outgoing ones are scheduled with the streams. #[derive(Default)] pub(super) struct DatagramState { pub(super) incoming: DatagramBuffer, - pub(super) outgoing: DatagramBuffer, - pub(super) send_blocked: bool, } impl DatagramState { @@ -148,68 +149,6 @@ impl DatagramState { Ok(was_empty) } - fn make_space_for(&mut self, datagram_len: usize, send_buffer_size: usize) { - while !self.has_send_buffer_space(datagram_len, send_buffer_size) { - let Some(prev) = self.outgoing.pop_front() else { - break; - }; - trace!(len = prev.data.len(), "dropping outgoing datagram"); - } - } - - fn has_send_buffer_space(&self, datagram_len: usize, send_buffer_size: usize) -> bool { - let Some(total) = self.outgoing.memory_used().checked_add(datagram_len) else { - return false; - }; - - total <= send_buffer_size - } - - /// Discard outgoing datagrams with a payload larger than `max_payload` bytes - /// - /// Returns whether any datagrams were dropped. - /// - /// Used to ensure that reductions in MTU don't get us stuck in a state where we have a datagram - /// queued but can't send it. - pub(super) fn drop_oversized(&mut self, max_payload: usize) -> bool { - let mut dropped_any = false; - self.outgoing.queue.retain(|datagram| { - let result = datagram.data.len() <= max_payload; - if !result { - trace!( - "dropping {} byte datagram violating {} byte limit", - datagram.data.len(), - max_payload - ); - self.outgoing.payload_bytes -= datagram.data.len(); - dropped_any = true; - } - result - }); - dropped_any - } - - /// Attempt to write a datagram frame into `buf`, consuming it from `self.outgoing` - /// - /// Returns whether a frame was written. At most `max_size` bytes will be written, including - /// framing. - pub(super) fn write(&mut self, buf: &mut Vec, max_size: usize) -> bool { - let Some(datagram) = self.outgoing.pop_front() else { - return false; - }; - - if buf.len() + datagram.size(true) > max_size { - // Future work: we could be more clever about cramming small datagrams into - // mostly-full packets when a larger one is queued first - self.outgoing.push_front(datagram); - return false; - } - - trace!(len = datagram.data.len(), "DATAGRAM"); - datagram.encode(true, buf); - true - } - pub(super) fn recv(&mut self) -> Option { let x = self.incoming.pop_front()?.data; Some(x) @@ -234,20 +173,11 @@ impl DatagramBuffer { Some(datagram) } - fn push_front(&mut self, datagram: Datagram) { - self.payload_bytes += datagram.data.len(); - self.queue.push_front(datagram); - } - fn memory_used(&self) -> usize { self.payload_bytes .saturating_add(self.queue.len() * size_of::()) } - pub(super) fn can_send_1rtt(&self, max_size: usize) -> bool { - self.queue.front().is_some_and(|x| x.size(true) <= max_size) - } - pub(super) fn is_empty(&self) -> bool { self.queue.is_empty() } @@ -257,37 +187,6 @@ impl DatagramBuffer { mod tests { use super::*; - #[test] - fn make_space_for_accounts_for_new_datagram() { - let mut state = DatagramState::default(); - state.outgoing.push_back(Datagram { - data: Bytes::from_static(&[0; 7]), - }); - state.outgoing.push_back(Datagram { - data: Bytes::from_static(&[0; 2]), - }); - - state.make_space_for(4, 10 + 2 * size_of::()); - - assert_eq!(state.outgoing.queue.len(), 1); - assert_eq!(state.outgoing.queue[0].data.len(), 2); - assert_eq!(state.outgoing.payload_bytes, 2); - } - - #[test] - fn make_space_for_handles_overflowing_capacity_check() { - let mut state = DatagramState::default(); - state.outgoing.queue.push_back(Datagram { - data: Bytes::from_static(&[0]), - }); - state.outgoing.payload_bytes = usize::MAX - 1; - - state.make_space_for(2, usize::MAX); - - assert!(state.outgoing.is_empty()); - assert_eq!(state.outgoing.payload_bytes, usize::MAX - 2); - } - #[test] fn empty_frame_flood_limit() { let mut state = DatagramState::default(); @@ -303,29 +202,12 @@ mod tests { } } } - - #[test] - fn drop_oversized_keeps_datagrams_at_limit() { - let mut state = DatagramState::default(); - state.outgoing.push_back(Datagram { - data: Bytes::from_static(&[0; 10]), - }); - state.outgoing.push_back(Datagram { - data: Bytes::from_static(&[0; 11]), - }); - - assert!(state.drop_oversized(10)); - - assert_eq!(state.outgoing.queue.len(), 1); - assert_eq!(state.outgoing.queue[0].data.len(), 10); - assert_eq!(state.outgoing.payload_bytes, 10); - } } /// Errors that can arise when sending a datagram #[derive(Debug, Error, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum SendDatagramError { - /// The peer does not support receiving datagram frames + /// The peer does not support receiving datagrams #[error("datagrams not supported by peer")] UnsupportedByPeer, /// Datagram support is disabled locally @@ -337,7 +219,4 @@ pub enum SendDatagramError { /// exceeded. #[error("datagram too large")] TooLarge, - /// Send would block - #[error("datagram send blocked")] - Blocked(Bytes), } diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index eea90958e9..eb8de7985e 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -23,7 +23,7 @@ use crate::{ config::{ServerConfig, TransportConfig}, connection::spaces::LostPacket, crypto::{self, KeyPair, Keys, PacketKey}, - frame::{self, Close, Datagram, FrameStruct, NewConnectionId, NewToken}, + frame::{self, Close, FrameStruct, NewConnectionId, NewToken}, packet::{ FixedLengthConnectionIdParser, Header, InitialHeader, InitialPacket, LongType, Packet, PacketNumber, PartialDecode, SpaceId, @@ -969,7 +969,7 @@ impl Connection { && !can_send.acks && can_send.other && (buf_capacity - builder.datagram_start) == self.path.current_mtu() as usize - && self.datagrams.outgoing.is_empty()), + && self.streams.datagram_bytes() == 0), "SendableFrames was {can_send:?}, but only ACKs have been written" ); pad_datagram |= sent.requires_padding; @@ -3611,23 +3611,6 @@ impl Connection { self.stats.frame_tx.retire_connection_id += 1; } - // DATAGRAM - let mut sent_datagrams = false; - while buf.len() + Datagram::SIZE_BOUND < max_size && space_id == SpaceId::Data { - match self.datagrams.write(buf, max_size) { - true => { - sent_datagrams = true; - sent.non_retransmits = true; - self.stats.frame_tx.datagram += 1; - } - false => break, - } - } - if self.datagrams.send_blocked && sent_datagrams { - self.events.push_back(Event::DatagramsUnblocked); - self.datagrams.send_blocked = false; - } - // NEW_TOKEN while let Some(remote_addr) = space.pending.new_tokens.pop() { debug_assert_eq!(space_id, SpaceId::Data); @@ -3667,12 +3650,15 @@ impl Connection { self.stats.frame_tx.new_token += 1; } - // STREAM + // STREAM and DATAGRAM, interleaved in schedule order if space_id == SpaceId::Data { - sent.stream_frames = - self.streams - .write_stream_frames(buf, max_size, self.config.send_fairness); + let written = self + .streams + .write_frames(buf, max_size, self.config.send_fairness); + sent.stream_frames = written.streams; + sent.non_retransmits |= written.datagrams > 0; self.stats.frame_tx.stream += sent.stream_frames.len() as u64; + self.stats.frame_tx.datagram += written.datagrams; } // Bundle ACK with other frames when there is room for them. @@ -3984,14 +3970,13 @@ impl Connection { /// /// See also `self.space(SpaceId::Data).can_send()` fn can_send_1rtt(&self, max_size: usize) -> bool { - self.streams.can_send_stream_data() + self.streams.can_send_frames(max_size) || self.path.challenge_pending || self .prev_path .as_ref() .is_some_and(|(_, x)| x.challenge_pending) || !self.path_responses.is_empty() - || self.datagrams.outgoing.can_send_1rtt(max_size) } /// Update counters to account for a packet becoming acknowledged, lost, or abandoned @@ -4344,8 +4329,6 @@ pub enum Event { Stream(StreamEvent), /// One or more application datagrams have been received DatagramReceived, - /// One or more application datagrams have been sent after blocking - DatagramsUnblocked, /// The currently active path was updated PathUpdated, } diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index dd4b6e1872..2e0f1494d0 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -18,7 +18,7 @@ use recv::Recv; mod saturated; mod schedule; pub use recv::{Chunks, Credit, ReadError, ReadableError}; -use schedule::Schedule; +use schedule::{Next, Schedule}; mod send; pub(crate) use send::{ByteSlice, BytesArray}; @@ -488,6 +488,47 @@ impl SendGroups<'_> { } } +impl StreamsState { + /// Queue a datagram in `group` at `order`, within `budget` bytes of queued datagrams. + /// + /// Returns whether it stayed queued rather than being evicted itself. + /// + /// # Panics + /// - when `group` was opened on another connection + pub(crate) fn queue_datagram( + &mut self, + datagram: frame::Datagram, + group: Option<&SendGroup>, + order: i64, + budget: usize, + ) -> bool { + let group = group.map(|g| g.index(self.tag)); + let frame = datagram.size(true); + self.pending + .push_datagram(group, order, datagram, frame, budget) + } + + /// Drop queued datagrams with a payload over `max_payload` bytes, returning whether any was. + pub(crate) fn drop_oversized_datagrams(&mut self, max_payload: usize) -> bool { + self.pending.retain_datagrams(|datagram| { + let keep = datagram.data.len() <= max_payload; + if !keep { + trace!( + "dropping {} byte datagram violating {} byte limit", + datagram.data.len(), + max_payload + ); + } + keep + }) + } + + /// Frame bytes queued datagrams hold against the send buffer. + pub(crate) fn datagram_bytes(&self) -> usize { + self.pending.datagram_bytes() + } +} + /// A send group, from [`SendGroups::open`] /// /// Release it with [`SendGroups::close`] on the connection that opened it. diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs index 4a3c511de9..16d27c7896 100644 --- a/rs/moq-quic/src/connection/streams/saturated.rs +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -1,18 +1,28 @@ -//! Scheduling proofs through the real send path: streams written into send groups and -//! drained into packets by `write_stream_frames`, with loss and resets applied the way +//! Scheduling proofs through the real send path: streams and datagrams queued into send +//! groups and drained into packets by `write_frames`, with loss and resets applied the way //! the connection applies them. use std::collections::HashMap; +use bytes::{BufMut, Bytes, BytesMut}; + use super::state::StreamsState; use super::{SendDomain, SendGroup, SendGroups, SendStream, Streams}; use crate::{ Dir, Side, StreamId, VarInt, connection::{State as ConnState, spaces::Retransmits}, - frame::StreamMeta, + frame::{self, Frame, StreamMeta}, transport_parameters::TransportParameters, }; +/// One frame of a packet, in the order it was written. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Sent { + Stream(StreamId), + /// A datagram, by the tag it was queued with. + Datagram(u64), +} + const PACKET: usize = 1200; /// How far two equally entitled groups may drift apart: each round robin may @@ -29,6 +39,12 @@ struct Sender { /// Bytes each stream's frames took on the wire, headers included, which is what /// the round robins share. wire: HashMap, + /// Bytes each datagram tag took on the wire, headers included. + datagram_wire: HashMap, + /// Every frame sent, in order. + order: Vec, + /// The next datagram tag. + tag: u64, } impl Sender { @@ -58,6 +74,9 @@ impl Sender { conn: ConnState::Established, sent: HashMap::new(), wire: HashMap::new(), + datagram_wire: HashMap::new(), + order: Vec::new(), + tag: 0, } } @@ -99,10 +118,52 @@ impl Sender { assert_eq!(self.send(id).write(&vec![0; bytes]), Ok(bytes)); } - /// Send one packet, returning its frames. + /// Queue a `bytes` payload datagram in `group` at `order`, returning its tag. + fn datagram(&mut self, group: Option<&SendGroup>, order: i64, bytes: usize) -> u64 { + let tag = self.tag; + self.tag += 1; + let mut data = BytesMut::with_capacity(bytes); + data.put_u64(tag); + data.resize(bytes, 0); + let datagram = frame::Datagram { data: data.freeze() }; + assert!( + self.state + .queue_datagram(datagram, group, order, usize::MAX) + ); + tag + } + + /// Send one packet, returning its stream frames. fn packet(&mut self) -> Vec { - let mut buf = Vec::with_capacity(PACKET); - let frames = self.state.write_stream_frames(&mut buf, PACKET, true); + self.packet_of(PACKET) + } + + /// Send one packet of `size` bytes, returning its stream frames. + fn packet_of(&mut self, size: usize) -> Vec { + let mut buf = Vec::with_capacity(size); + let written = self.state.write_frames(&mut buf, size, true); + let frames = written.streams; + + // The packet's frames in order, and the wire bytes its datagrams took. + let mut datagram_bytes = 0; + let frames_in = match buf.is_empty() { + true => None, + false => Some(frame::Iter::new(Bytes::from(buf.clone())).unwrap()), + }; + for frame in frames_in.into_iter().flatten() { + match frame.unwrap() { + Frame::Stream(stream) => self.order.push(Sent::Stream(stream.id)), + Frame::Datagram(datagram) => { + let wire = datagram.size(true) as u64; + let tag = u64::from_be_bytes(datagram.data[..8].try_into().unwrap()); + *self.datagram_wire.entry(tag).or_default() += wire; + datagram_bytes += wire; + self.order.push(Sent::Datagram(tag)); + } + frame => panic!("unexpected {frame:?}"), + } + } + let stream_bytes = buf.len() as u64 - datagram_bytes; // Every frame carries its length but perhaps the last, which can run to the // end of the packet instead. @@ -114,14 +175,11 @@ impl Sender { (header.len() as u64, frame.offsets.end - frame.offsets.start) }) .collect(); - if sizes.iter().map(|(h, p)| h + p).sum::() != buf.len() as u64 { + if sizes.iter().map(|(h, p)| h + p).sum::() != stream_bytes { let (header, payload) = sizes.last_mut().unwrap(); *header -= VarInt::from_u64(*payload).unwrap().size() as u64; } - assert_eq!( - sizes.iter().map(|(h, p)| h + p).sum::(), - buf.len() as u64 - ); + assert_eq!(sizes.iter().map(|(h, p)| h + p).sum::(), stream_bytes); for (frame, (header, payload)) in frames.iter().zip(sizes) { *self.sent.entry(frame.id).or_default() += payload; @@ -152,9 +210,25 @@ impl Sender { .sum() } + /// Send `packets` packets, returning every frame in order. + fn drain_frames(&mut self, packets: usize) -> Vec { + let start = self.order.len(); + for _ in 0..packets { + self.packet(); + } + self.order[start..].to_vec() + } + + fn datagram_wire(&self, tags: &[u64]) -> u64 { + tags.iter() + .map(|tag| self.datagram_wire.get(tag).copied().unwrap_or(0)) + .sum() + } + fn clear(&mut self) { self.sent.clear(); self.wire.clear(); + self.datagram_wire.clear(); } } @@ -439,3 +513,158 @@ fn streams_can_leave_their_group() { assert!(sender.state.pending.is_clean()); assert_eq!(sender.drain(1), [id]); } + +/// A low-priority datagram track waits behind a higher-priority stream rather than +/// going out ahead of every stream. +#[test] +fn low_priority_datagrams_wait_behind_streams() { + let mut sender = Sender::new(); + let high = sender.group(1); + let low = sender.group(0); + let tags: Vec<_> = (0..20) + .map(|seq| sender.datagram(Some(&low), seq, 1_000)) + .collect(); + let stream = sender.open(Some(&high), 0, 20_000); + + let order = sender.drain_frames(100); + let first = order + .iter() + .position(|sent| matches!(sent, Sent::Datagram(_))) + .unwrap(); + assert!(order[..first].iter().all(|&sent| sent == Sent::Stream(stream))); + assert!(order[first..].iter().all(|sent| matches!(sent, Sent::Datagram(_)))); + assert_eq!(sender.bytes(&[stream]), 20_000); + assert_eq!(order[first..].len(), tags.len()); +} + +/// A high-priority datagram track still goes ahead of lower-priority streams. +#[test] +fn high_priority_datagrams_preempt_streams() { + let mut sender = Sender::new(); + let high = sender.group(1); + let low = sender.group(0); + let stream = sender.open(Some(&low), 0, 20_000); + let tags: Vec<_> = (0..20) + .map(|seq| sender.datagram(Some(&high), seq, 1_000)) + .collect(); + + let order = sender.drain_frames(100); + // Newest first, as the group orders them. + let expected: Vec<_> = tags.iter().rev().map(|&tag| Sent::Datagram(tag)).collect(); + assert_eq!(order[..tags.len()], expected[..]); + assert!(order[tags.len()..].iter().all(|&sent| sent == Sent::Stream(stream))); +} + +/// Inside one group a datagram merges into the group order: it waits behind a stream +/// for a newer group and goes ahead of a stream of its own group. +#[test] +fn datagrams_merge_into_their_group_order() { + let mut sender = Sender::new(); + let group = sender.group(0); + let older = sender.datagram(Some(&group), 10, 100); + let newer = sender.open(Some(&group), 11, 3_000); + let tied = sender.open(Some(&group), 12, 3_000); + let ahead = sender.datagram(Some(&group), 12, 100); + + let order = sender.drain_frames(20); + let position = |sent| order.iter().position(|&x| x == sent).unwrap(); + let last = |sent| order.iter().rposition(|&x| x == sent).unwrap(); + assert_eq!(position(Sent::Datagram(ahead)), 0); + assert!(last(Sent::Stream(tied)) < position(Sent::Stream(newer))); + assert!(last(Sent::Stream(newer)) < position(Sent::Datagram(older))); +} + +/// A group's datagrams spend its fair share, so sending datagrams buys no extra bytes over +/// an equal group of streams. +#[test] +fn datagrams_and_streams_share_a_groups_credit() { + let mut sender = Sender::new(); + let datagrams = sender.group(0); + let streams = sender.group(0); + let tags: Vec<_> = (0..1_000) + .map(|seq| sender.datagram(Some(&datagrams), seq, 500)) + .collect(); + let stream = sender.open(Some(&streams), 0, 1_000_000); + + sender.drain(200); + assert_fair(sender.datagram_wire(&tags), sender.wire(&[stream])); +} + +/// With one domain per broadcast, two saturated datagram broadcasts both keep sending, +/// one of many small datagrams and one of few large ones, each getting half the bytes. +#[test] +fn saturated_datagram_domains_share_bytes() { + let mut sender = Sender::new(); + let small = sender.groups().open_domain(); + let large = sender.groups().open_domain(); + let small = sender.groups().open(0, Some(&small)); + let large = sender.groups().open(9, Some(&large)); + let small: Vec<_> = (0..2_000) + .map(|seq| sender.datagram(Some(&small), seq, 50)) + .collect(); + let large: Vec<_> = (0..200) + .map(|seq| sender.datagram(Some(&large), seq, 1_000)) + .collect(); + + sender.drain(100); + assert_fair(sender.datagram_wire(&small), sender.datagram_wire(&large)); +} + +/// A datagram outside every group goes ahead of every group, like a control stream. +#[test] +fn ungrouped_datagrams_preempt_groups() { + let mut sender = Sender::new(); + let group = sender.group(i32::MAX); + let stream = sender.open(Some(&group), i64::MAX, 3_000); + let tag = sender.datagram(None, 0, 100); + + let order = sender.drain_frames(5); + assert_eq!(order[0], Sent::Datagram(tag)); + assert!(order[1..].iter().all(|&sent| sent == Sent::Stream(stream))); +} + +/// A datagram too large for the room left in a packet ends it and leads the next one, +/// rather than letting what queues behind it go first. +#[test] +fn a_datagram_that_does_not_fit_leads_the_next_packet() { + let mut sender = Sender::new(); + let group = sender.group(0); + let stream = sender.open(Some(&group), 0, 100); + let small = sender.datagram(Some(&group), 2, 300); + let large = sender.datagram(Some(&group), 1, 1_000); + + assert_eq!(sender.drain_frames(1), [Sent::Datagram(small)]); + assert_eq!( + sender.drain_frames(1), + [Sent::Datagram(large), Sent::Stream(stream)] + ); + assert!(!sender.state.can_send_stream_data()); +} + +/// A packet clamped below a queued datagram, such as a loss probe, still carries what fits +/// behind it, and the datagram keeps its place. +#[test] +fn a_clamped_packet_sends_what_fits() { + let mut sender = Sender::new(); + let group = sender.group(0); + let large = sender.datagram(Some(&group), 1, 1_000); + assert!(!sender.state.can_send_frames(500)); + let stream = sender.open(Some(&group), 0, 2_000); + assert!(sender.state.can_send_frames(500)); + + sender.packet_of(500); + assert_eq!(sender.order, [Sent::Stream(stream)]); + assert_eq!(sender.drain_frames(1)[0], Sent::Datagram(large)); +} + +/// A released group keeps its queued datagrams and frees once they are sent. +#[test] +fn a_released_group_sends_its_datagrams() { + let mut sender = Sender::new(); + let group = sender.group(0); + let tag = sender.datagram(Some(&group), 0, 100); + sender.groups().close(group); + assert!(!sender.state.pending.is_clean()); + assert_eq!(sender.drain_frames(1), [Sent::Datagram(tag)]); + assert!(sender.state.pending.is_clean()); +} diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs index 1458cfa9de..898c2625ff 100644 --- a/rs/moq-quic/src/connection/streams/schedule.rs +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -1,20 +1,30 @@ -//! The order streams with queued data are sent in. +//! The order streams and datagrams with queued data are sent in. //! -//! Streams outside any send group go first, in quinn's order: highest priority first, round -//! robin between equal priorities. Everything else is a hierarchy: +//! Streams and datagrams outside any send group go first, in quinn's order: highest priority +//! first, round robin between equal priorities. Everything else is a hierarchy: //! //! 1. Domains share the remaining bytes by deficit round robin. //! 2. Inside a domain, a higher-priority send group preempts every lower one. //! 3. Send groups of equal priority share their domain's bytes by deficit round robin. -//! 4. Inside a group, streams go in their own priority order, round robin between ties. +//! 4. Inside a group, streams and datagrams go in their own order, round robin between tied +//! streams. A datagram goes ahead of a stream of equal order, and tied datagrams oldest +//! first. //! //! Both round robins charge the bytes actually written, so a group of many small streams and a -//! group of one large stream get the same rate. A turn may overdraw its credit by one frame; -//! the debt carries into the next turn, so the error stays bounded by a frame. +//! group of one large stream get the same rate, and a group's datagrams spend the same credit +//! as its streams. A turn may overdraw its credit by one frame; the debt carries into the next +//! turn, so the error stays bounded by a frame. +//! +//! Queued datagrams share one byte budget. When it overflows, the domain holding the most +//! datagram bytes gives up the oldest datagram of its lowest-priority group. Priorities are +//! only compared inside a domain, so one busy domain cannot evict another's datagrams. //! //! Self-contained (std and slab only) so the scheduler benchmark can compile it from source. -use std::collections::{BTreeMap, VecDeque}; +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet, VecDeque}, +}; use slab::Slab; @@ -42,65 +52,118 @@ enum Queued { At(Key), } -/// A queued stream's priority, then a decreasing counter so ties go round robin. -type Key = (i64, u64); +/// A queued item's order, then whether it is a datagram, which wins a tie, then a decreasing +/// counter so the rest of a tie goes oldest first: round robin for requeued streams. +type Key = (i64, bool, u64); + +/// What to send next, from [`Schedule::pop`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Next { + /// A stream, out of the schedule until [`Schedule::sent`] reports back. + Stream(K), + /// The datagram heading a group, or the ungrouped queue with `None`. It stays queued + /// until [`Schedule::sent_datagram`] takes it. + Datagram(Option), +} + +#[derive(Debug)] +enum Item { + Stream(K), + /// A datagram and the bytes of its frame, which it holds against the budget. + Datagram(D, usize), +} #[derive(Debug)] -struct Queue { - streams: BTreeMap, +struct Queue { + items: BTreeMap>, next: Option, + /// The order of every queued datagram by its counter, so the oldest is the last. + datagrams: BTreeMap, } -impl Queue { +impl Queue { fn new() -> Self { Self { - streams: BTreeMap::new(), + items: BTreeMap::new(), next: None, + datagrams: BTreeMap::new(), } } - fn pop(&mut self) -> Option { - self.next - .take() - .or_else(|| self.streams.pop_last().map(|(_, id)| id)) + fn pop(&mut self, group: Option) -> Option> { + if let Some(id) = self.next.take() { + return Some(Next::Stream(id)); + } + let head = self.items.last_entry()?; + if let Item::Datagram(..) = head.get() { + return Some(Next::Datagram(group)); + } + let Item::Stream(id) = head.remove() else { + unreachable!("checked for a stream"); + }; + Some(Next::Stream(id)) } fn remove(&mut self, queued: Queued) { match queued { Queued::Next => self.next = None, Queued::At(key) => { - self.streams.remove(&key); + self.items.remove(&key); } } } + /// The datagram heading the queue, its frame bytes, and its counter. + fn head_datagram(&self) -> (&D, usize, u64) { + match self.items.last_key_value() { + Some((&(_, _, recency), Item::Datagram(datagram, frame))) => { + (datagram, *frame, recency) + } + _ => panic!("no datagram heads the queue"), + } + } + + /// The counter of the oldest queued datagram. + fn oldest_datagram(&self) -> Option { + self.datagrams.last_key_value().map(|(&recency, _)| recency) + } + + fn take_datagram(&mut self, recency: u64) -> (D, usize) { + let order = self.datagrams.remove(&recency).expect("datagram is queued"); + match self.items.remove(&(order, true, recency)) { + Some(Item::Datagram(datagram, size)) => (datagram, size), + _ => unreachable!("an indexed datagram is queued"), + } + } + fn is_empty(&self) -> bool { - self.next.is_none() && self.streams.is_empty() + self.next.is_none() && self.items.is_empty() } #[cfg(test)] fn len(&self) -> usize { - self.streams.len() + self.next.is_some() as usize + self.items.len() + self.next.is_some() as usize } } #[derive(Debug)] -struct Group { +struct Group { domain: usize, priority: i32, - queue: Queue, + queue: Queue, /// Bytes this group may still send in its current turn; negative after an overdraw. deficit: i64, - /// Whether the group has queued streams, or is being served one. It stays set between + /// Whether the group has queued items, or is being served a stream. It stays set between /// [`Schedule::pop`] and [`Schedule::sent`], so requeueing the stream is not a new turn. active: bool, /// Whether an entry for this group sits in its level's round robin. Usually equal to - /// having queued streams, but a group that runs dry away from the front of the round + /// having queued items, but a group that runs dry away from the front of the round /// robin keeps its entry until the round robin reaches it. ringed: bool, /// Streams that name this group, queued or not. members: usize, - /// Whether the owner released the group, which is freed once no stream names it. + /// Whether the owner released the group, which is freed once no stream names it and its + /// datagrams are gone. closed: bool, } @@ -112,13 +175,24 @@ struct Domain { deficit: i64, /// Whether an entry for this domain sits in the domain round robin, as for groups. ringed: bool, - /// Groups in this domain with queued streams. + /// Groups in this domain with queued items. backlogged: usize, /// Groups in this domain that are not yet freed. groups: usize, closed: bool, + /// Datagram bytes queued in the domain. + datagram_bytes: usize, + /// The counter of every datagram queued in the domain, so the last is its oldest. + datagrams: BTreeSet, + /// The order the domain's queues give up datagrams in: groups before the ungrouped queue + /// (`None`, in the default domain), lowest priority first, then the oldest datagram. + victims: BTreeSet, } +/// A queue holding datagrams, keyed for eviction: whether it is the ungrouped queue, its +/// group's priority, its oldest datagram, and the group. +type Victim = (bool, i64, Reverse, Option); + impl Domain { fn new() -> Self { Self { @@ -128,15 +202,18 @@ impl Domain { backlogged: 0, groups: 0, closed: false, + datagram_bytes: 0, + datagrams: BTreeSet::new(), + victims: BTreeSet::new(), } } } -/// Every stream with data to send, in the order to send it. +/// Every stream and datagram with data to send, in the order to send it. #[derive(Debug)] -pub(super) struct Schedule { - ungrouped: Queue, - groups: Slab>, +pub(super) struct Schedule { + ungrouped: Queue, + groups: Slab>, domains: Slab, /// The domain round robin. ring: VecDeque, @@ -144,9 +221,20 @@ pub(super) struct Schedule { backlogged: usize, /// Decreases with every push, so a requeued stream goes behind its equal-priority peers. recency: u64, + /// Domains holding datagrams, by their datagram bytes then their oldest datagram, so the + /// last gives one up first. + hoards: BTreeSet<(usize, u64, usize)>, + /// Datagram bytes queued across every domain, held against the budget. + datagram_bytes: usize, + /// How many queued datagrams have each frame size, so the smallest is the first. + frames: BTreeMap, + /// Streams queued, not counting one between [`Self::pop`] and [`Self::sent`]. + streams: usize, + /// Datagrams [`Self::defer_datagram`] took out of line until [`Self::restore`]. + deferred: Vec<(Option, Key, Item)>, } -impl Schedule { +impl Schedule { pub(super) fn new() -> Self { let mut domains = Slab::new(); assert_eq!(domains.insert(Domain::new()), DEFAULT_DOMAIN); @@ -157,14 +245,34 @@ impl Schedule { ring: VecDeque::new(), backlogged: 0, recency: u64::MAX, + hoards: BTreeSet::new(), + datagram_bytes: 0, + frames: BTreeMap::new(), + streams: 0, + deferred: Vec::new(), } } - /// Whether no stream is queued. + /// Whether nothing is queued. pub(super) fn is_empty(&self) -> bool { self.ungrouped.is_empty() && self.backlogged == 0 } + /// Whether a packet with room for a `space`-byte frame would carry something: any stream, + /// which can be split to fit, or a datagram no larger than that. + pub(super) fn can_send(&self, space: usize) -> bool { + self.streams > 0 + || self + .frames + .first_key_value() + .is_some_and(|(&frame, _)| frame <= space) + } + + /// Frame bytes held by queued datagrams. + pub(super) fn datagram_bytes(&self) -> usize { + self.datagram_bytes + } + #[cfg(test)] pub(super) fn len(&self) -> usize { self.ungrouped.len() @@ -211,7 +319,7 @@ impl Schedule { }) } - /// Release a group; it is freed once no stream names it. + /// Release a group; it is freed once no stream names it and its datagrams are gone. pub(super) fn close_group(&mut self, group: usize) { self.groups[group].closed = true; self.free_group(group); @@ -223,11 +331,12 @@ impl Schedule { /// Move a group to another priority level, keeping its credit. pub(super) fn set_group_priority(&mut self, group: usize, priority: i32) { - let state = &mut self.groups[group]; - if state.priority == priority { + if self.groups[group].priority == priority { return; } + self.unindex(Some(group)); + let state = &mut self.groups[group]; let old = std::mem::replace(&mut state.priority, priority); let domain = state.domain; if state.ringed { @@ -242,6 +351,7 @@ impl Schedule { .push_back(group); } } + self.index(Some(group)); } /// Move a stream to `group`, or out of every group, keeping its place in line if queued. @@ -279,9 +389,10 @@ impl Schedule { return; } self.recency -= 1; - let key = (priority, self.recency); + let key = (priority, false, self.recency); slot.queued = Some(Queued::At(key)); - self.queue(slot.group).streams.insert(key, id); + self.queue(slot.group).items.insert(key, Item::Stream(id)); + self.streams += 1; self.activate(slot.group); } @@ -292,6 +403,7 @@ impl Schedule { assert!(queue.next.is_none()); queue.next = Some(id); slot.queued = Some(Queued::Next); + self.streams += 1; self.activate(slot.group); } @@ -301,16 +413,140 @@ impl Schedule { return; }; self.queue(slot.group).remove(queued); + self.streams -= 1; self.deactivate(slot.group); } - /// Take the next stream to send from. + /// Queue a datagram in `group`, or ungrouped, at `order` among that queue's streams and + /// datagrams. Its `frame` bytes are what it sends and holds of the `budget` every queued + /// datagram shares. /// - /// The caller writes from it and then reports back with [`Self::sent`], which requeues - /// it and charges the bytes to its group. - pub(super) fn pop(&mut self) -> Option { - if let Some(id) = self.ungrouped.pop() { - return Some(id); + /// Returns whether it stayed queued: while the budget overflows, the domain holding the + /// most datagram bytes evicts the oldest datagram of its lowest-priority group, and this + /// datagram is dropped once it is the one to go. + pub(super) fn push_datagram( + &mut self, + group: Option, + order: i64, + datagram: D, + frame: usize, + budget: usize, + ) -> bool { + if let Some(group) = group { + assert!(!self.groups[group].closed, "datagram sent on a closed group"); + } + + self.recency -= 1; + let recency = self.recency; + self.unindex(group); + let queue = self.queue(group); + queue + .items + .insert((order, true, recency), Item::Datagram(datagram, frame)); + queue.datagrams.insert(recency, order); + let domain = self.domain_of(group); + let domain = &mut self.domains[domain]; + domain.datagram_bytes += frame; + domain.datagrams.insert(recency); + self.datagram_bytes += frame; + *self.frames.entry(frame).or_default() += 1; + self.index(group); + self.activate(group); + + while self.datagram_bytes > budget { + let &(_, _, domain) = self.hoards.last().expect("queued bytes are indexed"); + let &(_, _, Reverse(oldest), victim) = self.domains[domain] + .victims + .first() + .expect("a domain holding datagram bytes has a victim"); + self.remove_datagram(victim, oldest); + if oldest == recency { + return false; + } + } + true + } + + /// The datagram [`Self::pop`] returned as [`Next::Datagram`], and its frame bytes. + pub(super) fn datagram(&self, group: Option) -> (&D, usize) { + let (datagram, frame, _) = self.head(group); + (datagram, frame) + } + + /// Take the datagram [`Self::pop`] returned as [`Next::Datagram`], charging its frame to + /// its group. + pub(super) fn sent_datagram(&mut self, group: Option) -> D { + let (_, frame, recency) = self.head(group); + if let Some(group) = group { + let state = &mut self.groups[group]; + state.deficit -= frame as i64; + self.domains[state.domain].deficit -= frame as i64; + } + self.remove_datagram(group, recency) + } + + /// Take the datagram [`Self::pop`] returned out of line, so what queues behind it can fill + /// a packet it does not fit. [`Self::restore`] puts it back in its place. + /// + /// A group this empties restarts its turn when restored, gaining up to a quantum of credit, + /// so this is for the rare packet clamped below the path MTU, not every full one. + pub(super) fn defer_datagram(&mut self, group: Option) { + let (key, item) = self.queue(group).items.pop_last().expect("a datagram heads"); + self.deferred.push((group, key, item)); + // Its group stays allocated: the datagram is still indexed for eviction. + self.deactivate(group); + } + + /// Requeue every datagram [`Self::defer_datagram`] took out of line. + pub(super) fn restore(&mut self) { + for (group, key, item) in std::mem::take(&mut self.deferred) { + self.queue(group).items.insert(key, item); + self.activate(group); + } + } + + fn head(&self, group: Option) -> (&D, usize, u64) { + match group { + Some(group) => self.groups[group].queue.head_datagram(), + None => self.ungrouped.head_datagram(), + } + } + + /// Drop every queued datagram `keep` refuses, returning whether any was. + pub(super) fn retain_datagrams(&mut self, mut keep: impl FnMut(&D) -> bool) -> bool { + let groups = self.groups.iter().map(|(index, g)| (Some(index), &g.queue)); + let mut doomed = Vec::new(); + for (group, queue) in std::iter::once((None, &self.ungrouped)).chain(groups) { + for (&recency, &order) in &queue.datagrams { + if let Some(Item::Datagram(datagram, _)) = queue.items.get(&(order, true, recency)) + && !keep(datagram) + { + doomed.push((group, recency)); + } + } + } + for &(group, recency) in &doomed { + self.remove_datagram(group, recency); + } + !doomed.is_empty() + } + + /// Take the next stream or datagram to send. + /// + /// The caller writes from a stream and then reports back with [`Self::sent`], which + /// requeues it and charges the bytes to its group. A datagram stays queued until + /// [`Self::sent_datagram`] takes it, so one that does not fit waits for the next packet. + pub(super) fn pop(&mut self) -> Option> { + let next = self.pop_next(); + if let Some(Next::Stream(_)) = next { + self.streams -= 1; + } + next + } + + fn pop_next(&mut self) -> Option> { + if let Some(next) = self.ungrouped.pop(None) { + return Some(next); } loop { @@ -347,7 +583,12 @@ impl Schedule { continue; } - return Some(group_state.queue.pop().expect("checked non-empty")); + return Some( + group_state + .queue + .pop(Some(group)) + .expect("checked non-empty"), + ); } } @@ -379,14 +620,78 @@ impl Schedule { } } - fn queue(&mut self, group: Option) -> &mut Queue { + fn queue(&mut self, group: Option) -> &mut Queue { match group { Some(group) => &mut self.groups[group].queue, None => &mut self.ungrouped, } } - /// Give a group that just got its first queued stream a turn. + fn domain_of(&self, group: Option) -> usize { + group.map_or(DEFAULT_DOMAIN, |group| self.groups[group].domain) + } + + /// Dequeue a datagram, leaving the round robin with the group's last item. + fn remove_datagram(&mut self, group: Option, recency: u64) -> D { + self.unindex(group); + let (datagram, frame) = self.queue(group).take_datagram(recency); + let domain = self.domain_of(group); + let domain = &mut self.domains[domain]; + domain.datagram_bytes -= frame; + domain.datagrams.remove(&recency); + self.datagram_bytes -= frame; + if let std::collections::btree_map::Entry::Occupied(mut count) = self.frames.entry(frame) { + *count.get_mut() -= 1; + if *count.get() == 0 { + count.remove(); + } + } + self.index(group); + + self.deactivate(group); + if let Some(group) = group { + self.free_group(group); + } + datagram + } + + /// Withdraw a queue and its domain from the eviction order, ahead of changing its + /// datagrams or priority; [`Self::index`] puts them back. + fn unindex(&mut self, group: Option) { + self.reindex(group, false); + } + + fn index(&mut self, group: Option) { + self.reindex(group, true); + } + + fn reindex(&mut self, group: Option, insert: bool) { + let (queue, priority) = match group { + Some(index) => { + let state = &self.groups[index]; + (&state.queue, i64::from(state.priority)) + } + None => (&self.ungrouped, i64::MAX), + }; + let index = self.domain_of(group); + let domain = &mut self.domains[index]; + if let Some(oldest) = queue.oldest_datagram() { + let victim = (group.is_none(), priority, Reverse(oldest), group); + match insert { + true => domain.victims.insert(victim), + false => domain.victims.remove(&victim), + }; + } + if let Some(&oldest) = domain.datagrams.last() { + let hoard = (domain.datagram_bytes, oldest, index); + match insert { + true => self.hoards.insert(hoard), + false => self.hoards.remove(&hoard), + }; + } + } + + /// Give a group that just got its first queued item a turn. fn activate(&mut self, group: Option) { let Some(group) = group else { return; @@ -470,11 +775,11 @@ impl Schedule { fn free_group(&mut self, group: usize) { let state = &self.groups[group]; - if !state.closed || state.members > 0 { + if !state.closed || state.members > 0 || !state.queue.datagrams.is_empty() { return; } - // No members means nothing queued, but a stale round-robin entry may remain. + // Nothing names or fills the group, but a stale round-robin entry may remain. let state = self.groups.remove(group); let domain = &mut self.domains[state.domain]; if state.ringed { @@ -516,9 +821,10 @@ mod tests { use super::*; - /// Streams with a fixed number of bytes each, drained a frame at a time. + /// Streams with a fixed number of bytes each, drained a frame at a time, and datagrams + /// sent whole. Datagrams carry their id and size. struct Sim { - schedule: Schedule, + schedule: Schedule, streams: HashMap, next: u64, fair: bool, @@ -554,9 +860,30 @@ mod tests { id } - /// Send one frame of up to `frame` bytes, returning the stream it came from. + /// Queue a datagram of `bytes` within `budget`, returning its id and whether it stayed. + fn datagram( + &mut self, + group: Option, + order: i64, + bytes: usize, + budget: usize, + ) -> (u64, bool) { + let id = self.next; + self.next += 1; + let queued = self + .schedule + .push_datagram(group, order, (id, bytes), bytes, budget); + (id, queued) + } + + /// Send one frame of up to `frame` bytes or one datagram, returning its id. fn step(&mut self, frame: usize) -> Option { - let id = self.schedule.pop()?; + let id = match self.schedule.pop()? { + Next::Stream(id) => id, + Next::Datagram(group) => { + return Some(self.schedule.sent_datagram(group).0); + } + }; let stream = self.streams.get_mut(&id).unwrap(); let bytes = frame.min(stream.remaining); stream.remaining -= bytes; @@ -569,6 +896,11 @@ mod tests { } Some(id) } + + /// Send everything left, returning the ids in order. + fn drain(&mut self) -> Vec { + std::iter::from_fn(|| self.step(1000)).collect() + } } #[test] @@ -657,4 +989,138 @@ mod tests { let ids: Vec<_> = (0..4).map(|_| sim.step(1000).unwrap()).collect(); assert!(ids.contains(&first) && ids.contains(&second), "{ids:?}"); } + + #[test] + fn datagrams_follow_their_order_and_win_a_tie() { + let mut sim = Sim::new(); + let group = sim.schedule.open_group(0, None); + let older = sim.datagram(Some(group), 10, 100, usize::MAX).0; + let stream = sim.stream(Some(group), 11, 100); + let tied = sim.stream(Some(group), 10, 100); + let first = sim.datagram(Some(group), 10, 100, usize::MAX).0; + let second = sim.datagram(Some(group), 10, 100, usize::MAX).0; + + // The newer group first, then its tie: datagrams oldest first, ahead of the stream. + assert_eq!(sim.drain(), [stream, older, first, second, tied]); + assert!(sim.schedule.is_empty()); + assert_eq!(sim.schedule.datagram_bytes(), 0); + } + + #[test] + fn a_closed_group_sends_its_datagrams_then_frees() { + let mut sim = Sim::new(); + let group = sim.schedule.open_group(0, None); + let id = sim.datagram(Some(group), 0, 100, usize::MAX).0; + sim.schedule.close_group(group); + assert!(!sim.schedule.is_clean()); + + assert_eq!(sim.drain(), [id]); + assert!(sim.schedule.is_clean()); + assert!(sim.schedule.groups.is_empty()); + } + + #[test] + fn eviction_takes_the_lowest_priority_oldest_datagram() { + let mut sim = Sim::new(); + let high = sim.schedule.open_group(1, None); + let low = sim.schedule.open_group(0, None); + let low_old = sim.datagram(Some(low), 0, 100, 400).0; + let high_old = sim.datagram(Some(high), 0, 100, 400).0; + // Newer but higher order: eviction goes by age, not order. + let low_new = sim.datagram(Some(low), 5, 100, 400).0; + let ungrouped = sim.datagram(None, 0, 100, 400).0; + + // Full: the next evicts the low group's oldest, then its newest. + let (high_new, queued) = sim.datagram(Some(high), 0, 100, 400); + assert!(queued); + assert_eq!(sim.datagram(Some(high), 0, 100, 400).1, true); + assert_eq!(sim.drain(), [ungrouped, high_old, high_new, high_new + 1]); + let _ = (low_old, low_new); + } + + #[test] + fn the_new_datagram_drops_when_it_is_the_victim() { + let mut sim = Sim::new(); + let high = sim.schedule.open_group(1, None); + let low = sim.schedule.open_group(0, None); + let kept = sim.datagram(Some(high), 0, 100, 100).0; + assert_eq!(sim.datagram(Some(low), 0, 100, 100).1, false); + assert_eq!(sim.schedule.datagram_bytes(), 100); + assert!(sim.schedule.is_clean()); + assert_eq!(sim.drain(), [kept]); + } + + #[test] + fn eviction_stays_inside_the_busiest_domain() { + let mut sim = Sim::new(); + let quiet = sim.schedule.open_domain(); + let busy = sim.schedule.open_domain(); + let quiet_low = sim.schedule.open_group(0, Some(quiet)); + let busy_high = sim.schedule.open_group(9, Some(busy)); + + // The quiet domain's low-priority datagram survives a busy high-priority domain. + let survivor = sim.datagram(Some(quiet_low), 0, 100, 500).0; + let mut busy_ids = Vec::new(); + for _ in 0..10 { + busy_ids.push(sim.datagram(Some(busy_high), 0, 100, 500).0); + } + let sent = sim.drain(); + assert!(sent.contains(&survivor), "{sent:?}"); + // The busy domain kept its newest four. + assert_eq!(sent.len(), 5); + assert!(busy_ids[6..].iter().all(|id| sent.contains(id)), "{sent:?}"); + } + + #[test] + fn domains_tied_on_bytes_evict_from_the_older() { + let mut sim = Sim::new(); + let groups: Vec<_> = (0..3) + .map(|_| { + let domain = sim.schedule.open_domain(); + sim.schedule.open_group(0, Some(domain)) + }) + .collect(); + let ids: Vec<_> = groups + .iter() + .map(|&group| sim.datagram(Some(group), 0, 100, 250)) + .collect(); + // Each domain holds 100 bytes, so the one with the oldest datagram gives it up. + assert!(ids.iter().all(|&(_, queued)| queued)); + let sent = sim.drain(); + assert!(!sent.contains(&ids[0].0), "{sent:?}"); + assert_eq!(sent.len(), 2); + } + + #[test] + fn a_deferred_datagram_keeps_its_place() { + let mut sim = Sim::new(); + let group = sim.schedule.open_group(0, None); + let large = sim.datagram(Some(group), 2, 1000, usize::MAX).0; + let small = sim.datagram(Some(group), 1, 10, usize::MAX).0; + let stream = sim.stream(Some(group), 0, 100); + + // Only 100 bytes of room: the large datagram steps aside for what fits behind it. + assert_eq!(sim.schedule.pop(), Some(Next::Datagram(Some(group)))); + sim.schedule.defer_datagram(Some(group)); + assert_eq!(sim.step(1000), Some(small)); + assert_eq!(sim.step(1000), Some(stream)); + assert_eq!(sim.step(1000), None); + sim.schedule.restore(); + assert!(!sim.schedule.can_send(999)); + assert!(sim.schedule.can_send(1000)); + assert_eq!(sim.drain(), [large]); + } + + #[test] + fn retained_datagrams_drop_what_no_longer_fits() { + let mut sim = Sim::new(); + let group = sim.schedule.open_group(0, None); + let small = sim.datagram(Some(group), 0, 10, usize::MAX).0; + sim.datagram(Some(group), 0, 1000, usize::MAX); + sim.datagram(None, 0, 1000, usize::MAX); + assert!(sim.schedule.retain_datagrams(|&(_, bytes)| bytes <= 100)); + assert!(!sim.schedule.retain_datagrams(|&(_, bytes)| bytes <= 100)); + assert_eq!(sim.schedule.datagram_bytes(), 10); + assert_eq!(sim.drain(), [small]); + } } diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 4a9fa2c081..367da550ce 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -9,8 +9,8 @@ use rustc_hash::FxHashMap; use tracing::{debug, trace}; use super::{ - Credit, Recv, Retransmits, Schedule, Send, SendState, ShouldTransmit, StreamEvent, StreamHalf, - ThinRetransmits, + Credit, Next, Recv, Retransmits, Schedule, Send, SendState, ShouldTransmit, StreamEvent, + StreamHalf, ThinRetransmits, }; use crate::{ Dir, MAX_STREAM_COUNT, Side, StreamId, TransportError, VarInt, @@ -21,6 +21,15 @@ use crate::{ transport_parameters::TransportParameters, }; +/// The frames [`StreamsState::write_frames`] wrote into a packet. +#[derive(Debug, Default)] +pub(crate) struct WrittenFrames { + /// The STREAM frames, which are retransmitted when lost. + pub(crate) streams: StreamMetaVec, + /// How many DATAGRAM frames, which are not. + pub(crate) datagrams: u64, +} + /// Wrapper around `Recv` that facilitates reusing `Recv` instances #[derive(Debug)] pub(super) enum StreamRecv { @@ -101,8 +110,8 @@ pub struct StreamsState { /// This differs from `self.send.len()` in that it does not include streams that the peer is /// permitted to open but which have not yet been opened. pub(super) send_streams: usize, - /// Streams with outgoing data queued, in the order to send them - pub(super) pending: Schedule, + /// Streams with outgoing data queued and outgoing datagrams, in the order to send them + pub(super) pending: Schedule, events: VecDeque, /// Streams blocked on connection-level flow control or stream window space @@ -400,11 +409,18 @@ impl StreamsState { } } - /// Whether any stream data is queued, regardless of control frames + /// Whether any stream data or datagram is queued, regardless of control frames + #[cfg(test)] pub(crate) fn can_send_stream_data(&self) -> bool { !self.pending.is_empty() } + /// Whether [`Self::write_frames`] would write a frame into `space` bytes: any queued + /// stream, or a queued datagram whose frame fits + pub(crate) fn can_send_frames(&self, space: usize) -> bool { + self.pending.can_send(space) + } + /// Whether MAX_STREAM_DATA frames could be sent for stream `id` pub(crate) fn can_send_flow_control(&self, id: StreamId) -> bool { self.recv @@ -612,25 +628,40 @@ impl StreamsState { } } - pub(crate) fn write_stream_frames( + /// Write STREAM and DATAGRAM frames in schedule order until the packet is full. + /// + /// A datagram cannot be split, so one too large for the room left ends the packet and + /// leads the next one. Only when it would leave the packet empty, which a packet clamped + /// below the path MTU can, does it step aside for what fits behind it, since + /// [`Self::can_send_frames`] promised a frame. + pub(crate) fn write_frames( &mut self, buf: &mut Vec, max_buf_size: usize, fair: bool, - ) -> StreamMetaVec { - let mut stream_frames = StreamMetaVec::new(); + ) -> WrittenFrames { + let mut written = WrittenFrames::default(); while buf.len() + frame::Stream::SIZE_BOUND < max_buf_size { - if max_buf_size - .checked_sub(buf.len() + frame::Stream::SIZE_BOUND) - .is_none() - { - break; - } - // Pop the next stream to send from. Reset streams leave the schedule, so every // stream in it has data or a FIN to send. - let Some(id) = self.pending.pop() else { - break; + let id = match self.pending.pop() { + None => break, + Some(Next::Stream(id)) => id, + Some(Next::Datagram(group)) => { + let (_, frame) = self.pending.datagram(group); + if buf.len() + frame > max_buf_size { + if written.datagrams > 0 || !written.streams.is_empty() { + break; + } + self.pending.defer_datagram(group); + continue; + } + let datagram = self.pending.sent_datagram(group); + trace!(len = datagram.data.len(), "DATAGRAM"); + datagram.encode(true, buf); + written.datagrams += 1; + continue; + } }; let stream = self .send @@ -669,10 +700,11 @@ impl StreamsState { let requeue = stream.is_pending().then_some(stream.priority); self.pending .sent(id, &mut stream.slot, buf.len() - start, requeue, fair); - stream_frames.push(meta); + written.streams.push(meta); } - stream_frames + self.pending.restore(); + written } pub(crate) fn received_ack_of(&mut self, frame: frame::StreamMeta) { @@ -1624,7 +1656,7 @@ mod tests { high.write(b"high").unwrap(); let mut buf = Vec::with_capacity(40); - let meta = server.write_stream_frames(&mut buf, 40, true); + let meta = server.write_frames(&mut buf, 40, true).streams; assert_eq!(meta[0].id, id_high); assert_eq!(meta[1].id, id_mid); assert_eq!(meta[2].id, id_low); @@ -1662,7 +1694,7 @@ mod tests { assert_eq!(low.write(b"low").unwrap(), 3); let mut buf = Vec::with_capacity(40); - let mut sent = server.write_stream_frames(&mut buf, 40, fair); + let mut sent = server.write_frames(&mut buf, 40, fair).streams; assert_eq!(sent.len(), 1); let lost = sent.pop().unwrap(); assert_eq!(lost.id, id_low); @@ -1684,7 +1716,7 @@ mod tests { assert_eq!(high.write(b"high").unwrap(), 4); buf.clear(); - let sent = server.write_stream_frames(&mut buf, 40, fair); + let sent = server.write_frames(&mut buf, 40, fair).streams; assert_eq!(sent.len(), 2); assert_eq!(sent[0].id, id_high); assert_eq!(sent[0].offsets, 0..4); @@ -1745,7 +1777,7 @@ mod tests { high.set_priority(-1).unwrap(); let mut buf = Vec::with_capacity(1000); - let meta = server.write_stream_frames(&mut buf, 40, true); + let meta = server.write_frames(&mut buf, 40, true).streams; assert_eq!(meta.len(), 1); assert_eq!(meta[0].id, id_high); @@ -1753,7 +1785,7 @@ mod tests { assert_eq!(server.pending.len(), 2); // Send the remaining data. The initial mid priority one should go first now - let meta = server.write_stream_frames(&mut buf, 1000, true); + let meta = server.write_frames(&mut buf, 1000, true).streams; assert_eq!(meta.len(), 2); assert_eq!(meta[0].id, id_mid); assert_eq!(meta[1].id, id_high); @@ -1814,7 +1846,7 @@ mod tests { // loop until all the streams are written loop { let buf_len = buf.len(); - let meta = server.write_stream_frames(&mut buf, buf_len + 40, fair); + let meta = server.write_frames(&mut buf, buf_len + 40, fair).streams; if meta.is_empty() { break; } @@ -1885,7 +1917,7 @@ mod tests { // Write the first chunk of stream_a let buf_len = buf.len(); - let meta = server.write_stream_frames(&mut buf, buf_len + 40, false); + let meta = server.write_frames(&mut buf, buf_len + 40, false).streams; assert!(!meta.is_empty()); metas.extend(meta); @@ -1902,7 +1934,7 @@ mod tests { // loop until all the streams are written loop { let buf_len = buf.len(); - let meta = server.write_stream_frames(&mut buf, buf_len + 40, false); + let meta = server.write_frames(&mut buf, buf_len + 40, false).streams; if meta.is_empty() { break; } diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 60ff3f285e..42e438e836 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -2837,7 +2837,7 @@ fn tail_loss_small_segment_size() { info!("Sending datagram batch"); for _ in 0..DGRAM_NUM { pair.client_datagrams(client_ch) - .send(vec![0; DGRAM_LEN].into(), false) + .send(vec![0; DGRAM_LEN].into(), None, 0) .unwrap(); } @@ -2890,7 +2890,7 @@ fn tail_loss_probe_keeps_ping_when_datagram_does_not_fit() { let datagram_len = pair.client_datagrams(client_ch).max_size().unwrap(); assert!(datagram_len > INITIAL_MTU as usize); pair.client_datagrams(client_ch) - .send(vec![0; datagram_len].into(), false) + .send(vec![0; datagram_len].into(), None, 0) .unwrap(); pair.drive(); @@ -2937,7 +2937,7 @@ fn tail_loss_respect_max_datagrams() { info!("Sending datagram batch"); for _ in 0..DGRAM_NUM { pair.client_datagrams(client_ch) - .send(vec![0; DGRAM_LEN].into(), false) + .send(vec![0; DGRAM_LEN].into(), None, 0) .unwrap(); } @@ -2958,7 +2958,7 @@ fn datagram_send_recv() { const DATA: &[u8] = b"whee"; pair.client_datagrams(client_ch) - .send(DATA.into(), true) + .send(DATA.into(), None, 0) .unwrap(); pair.drive(); assert_matches!( @@ -2995,13 +2995,13 @@ fn datagram_recv_buffer_overflow() { const DATA2: &[u8] = &[0xBC; (PAYLOAD_WINDOW / 3) + 1]; const DATA3: &[u8] = &[0xCD; (PAYLOAD_WINDOW / 3) + 1]; pair.client_datagrams(client_ch) - .send(DATA1.into(), true) + .send(DATA1.into(), None, 0) .unwrap(); pair.client_datagrams(client_ch) - .send(DATA2.into(), true) + .send(DATA2.into(), None, 0) .unwrap(); pair.client_datagrams(client_ch) - .send(DATA3.into(), true) + .send(DATA3.into(), None, 0) .unwrap(); pair.drive(); assert_matches!( @@ -3013,7 +3013,7 @@ fn datagram_recv_buffer_overflow() { assert_matches!(pair.server_datagrams(server_ch).recv(), None); pair.client_datagrams(client_ch) - .send(DATA1.into(), true) + .send(DATA1.into(), None, 0) .unwrap(); pair.drive(); assert_eq!(pair.server_datagrams(server_ch).recv().unwrap(), DATA1); @@ -3032,12 +3032,12 @@ fn datagram_larger_than_send_buffer_is_too_large() { assert_matches!( pair.client_datagrams(client_ch) - .send(Bytes::from_static(&[0; 2]), true), + .send(Bytes::from_static(&[0; 2]), None, 0), Err(SendDatagramError::TooLarge) ); assert_matches!( pair.client_datagrams(client_ch) - .send(Bytes::from_static(&[0; 2]), false), + .send(Bytes::from_static(&[0; 2]), None, 0), Err(SendDatagramError::TooLarge) ); } @@ -3057,7 +3057,7 @@ fn datagram_unsupported() { assert_matches!(pair.server_conn_mut(server_ch).poll(), None); assert_matches!(pair.client_datagrams(client_ch).max_size(), None); - match pair.client_datagrams(client_ch).send(Bytes::new(), true) { + match pair.client_datagrams(client_ch).send(Bytes::new(), None, 0) { Err(SendDatagramError::UnsupportedByPeer) => {} Err(e) => panic!("unexpected error: {e}"), Ok(_) => panic!("unexpected success"), @@ -4350,7 +4350,7 @@ fn pure_sender_voluntarily_acks() { for _ in 0..100 { const MSG: &[u8] = b"hello"; pair.client_datagrams(client_ch) - .send(Bytes::from_static(MSG), true) + .send(Bytes::from_static(MSG), None, 0) .unwrap(); pair.drive(); assert_eq!(pair.server_datagrams(server_ch).recv().unwrap(), MSG); @@ -4481,7 +4481,7 @@ fn datagram_gso() { // Sending ack-eliciting packet from server let client send ACK, which prevents // sending bundled ACK for a while. pair.server_datagrams(server_ch) - .send(Bytes::new(), false) + .send(Bytes::new(), None, 0) .unwrap(); pair.drive(); @@ -4495,7 +4495,7 @@ fn datagram_gso() { const DATAGRAMS: usize = 10; for _ in 0..DATAGRAMS { pair.client_datagrams(client_ch) - .send(Bytes::from_static(&[0; DATAGRAM_LEN]), false) + .send(Bytes::from_static(&[0; DATAGRAM_LEN]), None, 0) .unwrap(); } pair.drive(); @@ -4524,7 +4524,7 @@ fn gso_truncation() { const SIZES: [usize; 3] = [1024, 768, 768]; for len in SIZES { pair.client_datagrams(client_ch) - .send(vec![0; len].into(), false) + .send(vec![0; len].into(), None, 0) .unwrap(); } pair.drive(); @@ -4589,7 +4589,7 @@ fn min_mtu_does_not_pad_application_data() { let message = Bytes::from_static(b"hello"); pair.client_datagrams(client_ch) - .send(message.clone(), false) + .send(message.clone(), None, 0) .unwrap(); pair.client.drive(pair.time, pair.server.addr); assert_eq!(pair.client.outbound.len(), 1); @@ -4774,10 +4774,10 @@ fn pad_to_mtu() { const LEN_1: usize = 800; const LEN_2: usize = 600; pair.client_datagrams(client_ch) - .send(vec![0; LEN_1].into(), false) + .send(vec![0; LEN_1].into(), None, 0) .unwrap(); pair.client_datagrams(client_ch) - .send(vec![0; LEN_2].into(), false) + .send(vec![0; LEN_2].into(), None, 0) .unwrap(); pair.client.drive(pair.time, pair.server.addr); @@ -4833,7 +4833,7 @@ fn large_datagram_with_acks() { let max_size = pair.client_datagrams(client_ch).max_size().unwrap(); let msg = Bytes::from(vec![0; max_size]); pair.client_datagrams(client_ch) - .send(msg.clone(), true) + .send(msg.clone(), None, 0) .unwrap(); let initial_datagrams = pair.client_conn_mut(client_ch).stats().udp_tx.datagrams; pair.drive(); @@ -4858,7 +4858,7 @@ fn voluntary_ack_with_large_datagrams() { for _ in 0..COUNT { let max_size = pair.client_datagrams(client_ch).max_size().unwrap(); pair.client_datagrams(client_ch) - .send(vec![0; max_size].into(), true) + .send(vec![0; max_size].into(), None, 0) .unwrap(); pair.drive(); } @@ -4880,7 +4880,7 @@ fn ack_bundled_with_datagrams() { // Send packet from client and then send from server. the packet from server should include ACKs pair.client_datagrams(client_ch) - .send(vec![0; 1].into(), false) + .send(vec![0; 1].into(), None, 0) .unwrap(); pair.drive_client(); pair.drive_server(); @@ -4890,7 +4890,7 @@ fn ack_bundled_with_datagrams() { pair.server_conn_mut(server_ch).stats().udp_tx.datagrams; pair.server_datagrams(server_ch) - .send(vec![0; 1].into(), false) + .send(vec![0; 1].into(), None, 0) .unwrap(); pair.drive_server(); @@ -4928,12 +4928,10 @@ fn path_changes_unblock_oversized_datagrams() { assert!(old_max > 1200); let empty_space = pair.server_datagrams(server_ch).send_buffer_space(); let data = Bytes::from(vec![42; old_max]); - loop { - match pair.server_datagrams(server_ch).send(data.clone(), false) { - Ok(()) => {} - Err(SendDatagramError::Blocked(_)) => break, - Err(error) => panic!("unexpected send error: {error}"), - } + while pair.server_datagrams(server_ch).send_buffer_space() >= old_max { + pair.server_datagrams(server_ch) + .send(data.clone(), None, 0) + .unwrap(); } while pair.server_conn_mut(server_ch).poll().is_some() {} @@ -4970,14 +4968,10 @@ fn path_changes_unblock_oversized_datagrams() { pair.server_datagrams(server_ch).send_buffer_space(), empty_space ); - assert!( - iter::from_fn(|| pair.server_conn_mut(server_ch).poll()) - .any(|event| matches!(event, Event::DatagramsUnblocked)) - ); let small = Bytes::from_static(b"small"); pair.server_datagrams(server_ch) - .send(small.clone(), false) + .send(small.clone(), None, 0) .unwrap(); pair.drive(); assert_eq!(pair.client_datagrams(client_ch).recv(), Some(small)); @@ -4985,9 +4979,9 @@ fn path_changes_unblock_oversized_datagrams() { } } -/// Verify that dropping oversized datagrams will trigger a DatagramsUnblocked event. +/// Verify that datagrams that no longer fit the path MTU are dropped from the send buffer. #[test] -fn oversized_datagrams_trigger_unblock() { +fn oversized_datagrams_are_dropped() { let _guard = subscribe(); let mut pair = Pair::default(); // Start the connection with a large MTU. @@ -5005,44 +4999,27 @@ fn oversized_datagrams_trigger_unblock() { // Send datagrams until the send buffer is full. let max_size = pair.client_datagrams(client_ch).max_size().unwrap(); let data = vec![0; max_size]; - loop { - match pair - .client_datagrams(client_ch) - .send(data.clone().into(), false) - { - Ok(_) => {} - Err(SendDatagramError::Blocked(_)) => { - break; - } - Err(e) => panic!("unexpected error: {e}"), - } + while pair.client_datagrams(client_ch).send_buffer_space() >= max_size { + pair.client_datagrams(client_ch) + .send(data.clone().into(), None, 0) + .unwrap(); } // Set the MTU to a smaller value so the queued datagrams cannot be sent. pair.mtu = 1200; // Drive the pair until black hole detection kicks in and the path MTU is adjusted. while pair.step() { - let err = loop { - if let Err(e) = pair + // Keep the buffer full until the MTU is adjusted and the datagram no longer fits. + let mut sent = Ok(()); + while sent.is_ok() && pair.client_datagrams(client_ch).send_buffer_space() >= max_size { + sent = pair .client_datagrams(client_ch) - .send(data.clone().into(), false) - { - break e; - } - }; - match err { - SendDatagramError::Blocked(_) => { - // continue with the next step but drain the DatagramsUnblocked events - // emitted datagrams were sent out. - while let Some(event) = pair.client_conn_mut(client_ch).poll() { - tracing::info!("ignoring connection event: {event:?}"); - } - } - SendDatagramError::TooLarge => { - // mtu adjusted, break the loop - break; - } - _ => panic!("unexpected error: {err}"), + .send(data.clone().into(), None, 0); + } + match sent { + Ok(()) => {} + Err(SendDatagramError::TooLarge) => break, + Err(err) => panic!("unexpected error: {err}"), } } @@ -5057,13 +5034,9 @@ fn oversized_datagrams_trigger_unblock() { assert_eq!( pair.client_datagrams(client_ch).send_buffer_space(), - send_buffer_size - size_of::(), + send_buffer_size - Datagram::SIZE_BOUND, "expected the send buffer to be empty after too large datagrams were dropped", ); - match pair.client_conn_mut(client_ch).poll() { - Some(Event::DatagramsUnblocked) => {} - _ => panic!("expected DatagramsUnblocked event"), - } } #[test] @@ -5963,7 +5936,7 @@ fn stream_starvation_reported_before_resumed_send() { fn datagram_starvation_reported_before_resumed_send() { check_starvation_reported_before_resumed_send(|pair, ch| { pair.client_datagrams(ch) - .send(Bytes::from_static(&[42; 1000]), true) + .send(Bytes::from_static(&[42; 1000]), None, 0) .unwrap(); }); } From 074bee29231b51b31e1619287e6c261ef026f9b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:24:03 -0700 Subject: [PATCH 22/37] feat(net)!: send each datagram in its subscription's send group The poll transport's poll_send_datagram becomes a synchronous send_datagram taking a send group and an order. The lite and IETF publishers pass the subscription's group and the group's in-group order, and every transport follows: moq-tokio and moq-uring rank it on moq-quic, the rest ignore it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/transport/adapter.rs | 10 ++- rs/moq-net/benches/schedule.rs | 86 ++++++++++++++++-- rs/moq-net/src/client.rs | 18 +++- rs/moq-net/src/ietf/adapter.rs | 11 ++- rs/moq-net/src/ietf/publisher.rs | 48 +++++++++- rs/moq-net/src/lite/publisher.rs | 37 +++++++- rs/moq-net/src/lite/test_transport.rs | 60 ++++++++++--- rs/moq-net/src/server.rs | 11 +-- rs/moq-net/src/transport.rs | 33 +++---- rs/moq-net/src/transport/memory.rs | 18 ++-- rs/moq-net/tests/support/mock.rs | 6 +- rs/moq-quic/README.md | 1 + rs/moq-quic/src/config/transport.rs | 7 +- rs/moq-quic/src/connection/streams/mod.rs | 3 +- .../src/connection/streams/schedule.rs | 1 + rs/moq-tokio/src/quic/driver/connection.rs | 87 +++---------------- rs/moq-tokio/src/quic/driver/mod.rs | 2 +- rs/moq-tokio/src/quic/driver/tests.rs | 4 +- rs/moq-tokio/src/quic/web/session.rs | 40 +++------ rs/moq-tokio/src/transport.rs | 14 ++- rs/moq-tokio/src/transport/grouped.rs | 11 +++ rs/moq-tokio/src/transport/owned.rs | 9 +- rs/moq-uring/src/quic/driver/connection.rs | 46 +++++----- rs/moq-uring/src/quic/web.rs | 23 +++-- rs/moq-uring/src/transport.rs | 9 ++ rs/moq-uring/src/transport/adapter.rs | 25 +++++- rs/moq-uring/tests/web.rs | 2 +- rs/moq-wasm/src/transport/adapter.rs | 10 ++- 28 files changed, 415 insertions(+), 217 deletions(-) diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 6a3fb5b14e..d24ace8108 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -80,8 +80,14 @@ impl + Clone + 'st fn send_group(&mut self, _: i32) -> Option { None } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) + /// The backend cannot rank a datagram against a stream, so the group and order are ignored, + /// and a datagram it has no room for is dropped. + fn send_datagram(&mut self, payload: Bytes, _: Option<&poll::Ungrouped>, _: i64) -> Result<(), Self::Error> { + let mut cx = Context::from_waker(std::task::Waker::noop()); + match backend::Session::poll_send_datagram(&mut self.0, &mut cx, &payload) { + Poll::Ready(res) => res.map_err(Error), + Poll::Pending => Ok(()), + } } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { backend::Session::poll_recv_datagram(&mut self.0, cx).map_err(Error) diff --git a/rs/moq-net/benches/schedule.rs b/rs/moq-net/benches/schedule.rs index 0a531ce542..e02c647e6c 100644 --- a/rs/moq-net/benches/schedule.rs +++ b/rs/moq-net/benches/schedule.rs @@ -14,6 +14,8 @@ //! //! `fanout` sweeps subscriptions against groups in flight per subscription, timing one frame, //! so a scheduling cost that grows with the table rather than the touched path is a slope. +//! `datagram` sweeps send groups, domains, and queued datagrams per group against one +//! datagram sent and one evicted, for the same reason. use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -38,7 +40,7 @@ mod track { mod schedule; use priority::{Priority, PriorityHandle, PriorityQueue}; -use schedule::{Schedule, Slot}; +use schedule::{Next, Schedule, Slot}; /// The most a frame carries, one packet. const PACKET: usize = 1200; @@ -188,7 +190,7 @@ struct Sim { kind: Kind, tracks: Vec, capacity: usize, - schedule: Schedule, + schedule: Schedule, groups: Vec, queue: PriorityQueue, moved: Arc>>, @@ -324,7 +326,7 @@ impl Sim { let mut budget = self.capacity; while budget > 0 { - let Some(id) = self.schedule.pop() else { + let Some(Next::Stream(id)) = self.schedule.pop() else { break; }; let stream = self.streams.get_mut(&id).unwrap(); @@ -420,7 +422,7 @@ fn bench_workloads(c: &mut Criterion) { } /// A saturated schedule: `subscriptions` groups at one priority, each with `streams` queued. -fn saturated(subscriptions: usize, streams: usize) -> (Schedule, Vec) { +fn saturated(subscriptions: usize, streams: usize) -> (Schedule, Vec) { let mut schedule = Schedule::new(); let mut slots = Vec::with_capacity(subscriptions * streams); for _ in 0..subscriptions { @@ -449,7 +451,9 @@ fn bench_fanout(c: &mut Criterion) { &(), |b, ()| { b.iter(|| { - let id = schedule.pop().unwrap(); + let Some(Next::Stream(id)) = schedule.pop() else { + unreachable!("only streams are queued"); + }; let slot = &mut slots[id as usize]; schedule.sent(id, slot, PACKET, Some((id % streams as u64) as i64), true); }); @@ -485,5 +489,75 @@ fn bench_fanout(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_workloads, bench_fanout); +/// A full datagram budget: `groups` send groups at one priority, each in its own domain when +/// `domains`, with `datagrams` queued in each, one packet apiece. +fn datagrams(groups: usize, datagrams: usize, domains: bool) -> (Schedule, Vec) { + let mut schedule = Schedule::new(); + let ids: Vec<_> = (0..groups) + .map(|_| { + let domain = domains.then(|| schedule.open_domain()); + schedule.open_group(0, domain) + }) + .collect(); + let budget = groups * datagrams * PACKET; + for seq in 0..datagrams { + for &group in &ids { + assert!(schedule.push_datagram(Some(group), seq as i64, (), PACKET, budget)); + } + } + (schedule, ids) +} + +fn bench_datagram(c: &mut Criterion) { + for (name, domains) in [("groups", false), ("domains", true)] { + // A datagram into a full budget, which evicts the oldest of the busiest domain's + // lowest-priority group. + let mut group = c.benchmark_group(format!("schedule_datagram_evict_{name}")); + for groups in [1, 16, 256, 4_096] { + for queued in [1, 16, 256] { + if groups * queued > 100_000 { + continue; + } + let (mut schedule, ids) = datagrams(groups, queued, domains); + let budget = schedule.datagram_bytes(); + let mut seq = queued as i64; + let mut next = 0; + group.bench_with_input(BenchmarkId::new(format!("{name}_{groups}"), queued), &(), |b, ()| { + b.iter(|| { + // Round robin, so every group keeps its share of the budget. + schedule.push_datagram(Some(ids[next]), seq, (), PACKET, budget); + next = (next + 1) % ids.len(); + seq += 1; + }); + }); + } + } + group.finish(); + } + + // A datagram sent and another queued in its place, through every round robin. + let mut group = c.benchmark_group("schedule_datagram_send"); + for groups in [1, 16, 256, 4_096] { + for queued in [1, 16, 256] { + if groups * queued > 100_000 { + continue; + } + let (mut schedule, _) = datagrams(groups, queued, false); + let mut seq = queued as i64; + group.bench_with_input(BenchmarkId::new(format!("groups_{groups}"), queued), &(), |b, ()| { + b.iter(|| { + let Some(Next::Datagram(at)) = schedule.pop() else { + unreachable!("only datagrams are queued"); + }; + schedule.sent_datagram(at); + schedule.push_datagram(at, seq, (), PACKET, usize::MAX); + seq += 1; + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_workloads, bench_fanout, bench_datagram); criterion_main!(benches); diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 632f5bf09d..fc9568de2f 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -624,8 +624,13 @@ mod tests { None } - fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { - Poll::Ready(Ok(())) + fn send_datagram( + &mut self, + _: Bytes, + _: Option<&crate::transport::poll::Ungrouped>, + _: i64, + ) -> Result<(), Self::Error> { + Ok(()) } fn poll_recv_datagram(&mut self, _cx: &mut Context<'_>) -> Poll> { @@ -1019,8 +1024,13 @@ mod tests { None } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - self.inner.poll_send_datagram(cx, payload) + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&crate::transport::poll::Ungrouped>, + order: i64, + ) -> Result<(), Self::Error> { + self.inner.send_datagram(payload, group, order) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 64715ff93e..ecf2089469 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -1176,10 +1176,15 @@ impl crate::transport::poll::Session for Con self.inner.send_group(priority) } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error> { self.inner - .poll_send_datagram(cx, payload) - .map(|r| r.map_err(|_| crate::Error::Closed)) + .send_datagram(payload, group, order) + .map_err(|_| crate::Error::Closed) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index fac84917f3..68ba1f152b 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -3920,7 +3920,10 @@ impl TrackServe { ); continue; } - let _ = self.session.send_datagram(&body); + // Ranked like a stream carrying the group from its start, so it merges into the + // subscription's group order and wins a tie with that group's stream. + let order = crate::priority::in_group(sequence, 0, self.ranking.order); + let _ = self.session.send_datagram(body, self.ranking.group.as_ref(), order); } Ok(()) } @@ -5106,6 +5109,49 @@ mod group_priority_test { assert_eq!(log.orders().len(), 2, "the streams keep their place in the group"); } + /// An OBJECT_DATAGRAM goes in its subscription's send group, ranked like the stream + /// carrying its group from the start, so the transport merges it into the group order. + #[test] + fn datagrams_rank_within_their_subscription() { + let log = crate::lite::test_transport::Log::default(); + let session = SinkSession::new(log.clone()) + .with_groups() + .with_datagrams(1200) + .with_unacked_fin(); + let mut track = track::Producer::new( + std::sync::Arc::new(crate::broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe( + track::Subscription::default() + .with_priority(3) + .with_max_delay(std::time::Duration::from_secs(3600)), + ); + let mut serve = TrackServe::new( + session, + subscriber, + RequestId(0), + Version::Draft14, + ServeRange::default(), + None, + ); + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group + .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"x".as_slice()) + .unwrap(); + group.finish().unwrap(); + let sequence = track + .append_datagram(crate::Timestamp::from_millis(1).unwrap(), b"d".as_slice()) + .unwrap(); + + let waiter = kio::Waiter::noop(); + assert!(serve.poll(&waiter).is_pending()); + let order = crate::priority::in_group(sequence, 0, Order::Descending); + assert_eq!(log.datagram_ranks(), [(Some(0), order)]); + assert_eq!(log.orders(), [(0, 1)], "the stream for group 0 ranks below"); + } + /// A group with more objects ready than one poll's budget drains over several polls. #[test] fn a_long_group_drains_within_the_budget() { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 2af6c0377e..3e40a1d64a 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3179,6 +3179,8 @@ struct Opens { impl Subscription { /// Send one datagram best-effort over a QUIC datagram (lite-05 §6.4). /// + /// It goes in the subscription's send group, ranked like a stream carrying its group, so + /// it merges into the subscription's group order and wins a tie with that group's stream. /// The datagram is dropped (there is no group fallback) if the encoded body doesn't fit the /// transport's datagram limit or the send fails (congestion / no capacity right now). /// Returns whether it was handed to the transport. @@ -3213,7 +3215,8 @@ impl Subscription { return false; } - let _ = self.session.send_datagram(&body); + let order = crate::priority::in_group(datagram.sequence, 0, self.steer_seen.order); + let _ = self.session.send_datagram(body, self.group.as_ref(), order); true } @@ -4582,6 +4585,38 @@ mod serve_group_test { assert_eq!(flipped, [(0, 1), (1, -1)], "older groups first"); } + /// A datagram goes in its subscription's send group, ranked like the stream carrying + /// its group from the start, so the transport merges it into the group order. + #[moq_net_sim::test] + async fn datagrams_rank_within_their_subscription() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(track::Subscription::default().with_max_delay(Duration::from_secs(3600))); + let mut session = SinkSession::new(Log::default()) + .with_groups() + .with_datagrams(1200) + .with_unacked_fin(); + let log = session.log.clone(); + let (mut run, mut writer, _) = lite_run(Version::Lite06, session.clone(), subscriber); + run.ctx.group = crate::transport::poll::Session::send_group(&mut session, 3); + + write_group(&mut track, 0, 0); + let sequence = track + .append_datagram(Timestamp::from_millis(1).unwrap(), b"d".as_slice()) + .unwrap(); + { + let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); + assert!(futures::poll!(poll.as_mut()).is_pending()); + } + let order = crate::priority::in_group(sequence, 0, Order::Descending); + assert_eq!(log.datagram_ranks(), [(Some(0), order)]); + assert_eq!(log.orders(), [(0, 1)], "the stream for group 0 ranks below"); + assert!(order > 1); + } + /// A group with more frames ready than one poll's budget drains over several polls, /// so a sibling (the transport's driver, say) runs in between. #[test] diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 75a9e307ac..cbd9945e75 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -40,6 +40,8 @@ impl crate::transport::Error for SinkError { #[derive(Clone, Default)] pub struct Log { pub datagrams: Arc>>, + /// The send group and order each datagram was sent with, beside [`Self::datagrams`]. + datagram_ranks: Arc, i64)>>>, pub writes: Arc>>, pub resets: Arc>>, /// Every write, FIN and reset on every send stream, in call order. See [`Self::trail`]. @@ -116,6 +118,19 @@ impl Log { self.orders.lock().unwrap().clone() } + /// The send group and order of every datagram, as (group, order) in call order. + pub fn datagram_ranks(&self) -> Vec<(Option, i64)> { + self.datagram_ranks.lock().unwrap().clone() + } + + fn record_datagram(&self, payload: bytes::Bytes, group: Option<&SinkGroup>, order: i64) { + self.datagrams.lock().unwrap().push(payload); + self.datagram_ranks + .lock() + .unwrap() + .push((group.map(|group| group.id), order)); + } + /// The session-level closes, as (code, reason). A list rather than a last-value so /// a second close racing the first is visible. pub fn closes(&self) -> Vec<(u32, String)> { @@ -412,8 +427,14 @@ impl poll::Session for DeadStreamSession { None } - fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { - Poll::Ready(Ok(())) + fn send_datagram( + &mut self, + payload: bytes::Bytes, + group: Option<&SinkGroup>, + order: i64, + ) -> Result<(), Self::Error> { + self.log.record_datagram(payload, group, order); + Ok(()) } fn poll_recv_datagram(&mut self, _cx: &mut Context<'_>) -> Poll> { @@ -464,6 +485,8 @@ pub struct SinkSession { unacked_fin: bool, /// Set by [`Self::with_groups`]. groups: bool, + /// Set by [`Self::with_datagrams`]; zero disables datagrams. + datagram_size: usize, /// The ALPN to report, for a test that needs a specific negotiated version rather /// than the SETUP-negotiated fallback an absent one selects. protocol: Option<&'static str>, @@ -494,6 +517,12 @@ impl SinkSession { self } + /// Accept datagrams up to `max_size` bytes, recording them in the log. + pub fn with_datagrams(mut self, max_size: usize) -> Self { + self.datagram_size = max_size; + self + } + /// Report these connection statistics, as a real transport would. pub fn with_stats(self, stats: SinkStats) -> Self { self.set_stats(stats); @@ -612,8 +641,14 @@ impl poll::Session for SinkSession { self.groups.then(|| SinkGroup::open(self.log.clone(), priority)) } - fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, _payload: &[u8]) -> Poll> { - Poll::Ready(Ok(())) + fn send_datagram( + &mut self, + payload: bytes::Bytes, + group: Option<&SinkGroup>, + order: i64, + ) -> Result<(), Self::Error> { + self.log.record_datagram(payload, group, order); + Ok(()) } fn poll_recv_datagram(&mut self, _cx: &mut Context<'_>) -> Poll> { @@ -621,7 +656,7 @@ impl poll::Session for SinkSession { } fn max_datagram_size(&self) -> usize { - 0 + self.datagram_size } fn binding(&self) -> crate::transport::Binding { @@ -937,13 +972,14 @@ impl poll::Session for ScriptedSession { None } - fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - self.log - .datagrams - .lock() - .unwrap() - .push(bytes::Bytes::copy_from_slice(payload)); - Poll::Ready(Ok(())) + fn send_datagram( + &mut self, + payload: bytes::Bytes, + group: Option<&SinkGroup>, + order: i64, + ) -> Result<(), Self::Error> { + self.log.record_datagram(payload, group, order); + Ok(()) } fn poll_recv_datagram(&mut self, _cx: &mut Context<'_>) -> Poll> { diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 8bcc522988..5a3a341e28 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1069,12 +1069,13 @@ mod tests { None } - fn poll_send_datagram( + fn send_datagram( &mut self, - _cx: &mut std::task::Context<'_>, - _payload: &[u8], - ) -> std::task::Poll> { - std::task::Poll::Ready(Ok(())) + _: Bytes, + _: Option<&crate::transport::poll::Ungrouped>, + _: i64, + ) -> Result<(), Self::Error> { + Ok(()) } fn poll_recv_datagram( &mut self, diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 288ab796ad..0e5b9b3b94 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -164,24 +164,27 @@ pub mod poll { /// handle is dropped and its streams finish. fn send_group(&mut self, priority: i32) -> Option<::Group>; - /// Poll to send a datagram over the network. + /// Send a datagram in `group` at `order`, best-effort. /// - /// Returns [`Poll::Pending`] while the transport has no room for it, so a caller - /// can wait for capacity rather than having the payload dropped underneath it. + /// `order` ranks the datagram among the group's streams like a stream's own + /// [priority](SendStream::set_priority), and a datagram goes ahead of a stream of equal + /// order. Without a group it is ranked among the ungrouped streams. A transport that + /// cannot rank datagrams against streams ignores both. /// - /// `payload` is taken by reference, not by value or as a [`Buf`]: a - /// [`Poll::Pending`] return means the caller retries with the same datagram, and - /// both of those would have consumed it. (A datagram also needs *contiguous* - /// bytes, and the only way to get those from a generic [`Buf`] is - /// [`Buf::copy_to_bytes`], which consumes.) + /// Never blocks: a transport with no room drops a queued datagram, the lowest priority + /// first where it can tell, exactly as the network is allowed to. /// /// Accepting a datagram is not delivery. QUIC datagrams may still be dropped: /// - Network congestion. /// - Random packet loss. /// - Payload is larger than `max_datagram_size()` /// - Peer is not receiving datagrams. - /// - ??? - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll>; + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error>; /// Poll for a datagram from the network. fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll>; @@ -242,16 +245,6 @@ pub mod poll { RecvDatagram(self) } - /// Send a datagram, best-effort: if the transport has no room for it right - /// now, the datagram is dropped, exactly as the network is allowed to do. - fn send_datagram(&mut self, payload: &[u8]) -> Result<(), Self::Error> { - let mut cx = Context::from_waker(std::task::Waker::noop()); - match self.poll_send_datagram(&mut cx, payload) { - Poll::Ready(res) => res, - Poll::Pending => Ok(()), - } - } - /// Wait until the session is closed by either side, returning the reason. fn closed(&mut self) -> SessionClosed<'_, Self> { SessionClosed(self) diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 41ace4c5f5..62b94e041a 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -13,8 +13,8 @@ //! with a code and drops what it queued. Closing the session keeps queued messages //! readable, and leaves streams as they were, so the peer reads them, then the close. //! -//! Datagrams are copied, since [`poll::Session::poll_send_datagram`] borrows them, and -//! dropped when oversized or when the peer's queue is full, as a network would. +//! Datagrams ignore their send group and order, and are dropped when oversized or when the +//! peer's queue is full, as a network would. use std::{ collections::VecDeque, @@ -298,15 +298,15 @@ impl poll::Session for Session { None } - fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { + fn send_datagram(&mut self, payload: Bytes, _: Option<&poll::Ungrouped>, _: i64) -> Result<(), Error> { if let Some(err) = closed(&self.conn().close) { - return Poll::Ready(Err(err)); + return Err(err); } // An oversized datagram, or one the peer has no room for, is lost as on a network. if payload.len() <= MAX_DATAGRAM_SIZE { - let _ = self.side.outgoing().datagrams.try_push(Bytes::copy_from_slice(payload)); + let _ = self.side.outgoing().datagrams.try_push(payload); } - Poll::Ready(Ok(())) + Ok(()) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { @@ -877,12 +877,12 @@ mod tests { #[test] fn datagrams() { let (mut a, mut b) = pair(Config::default()); - ready(a.poll_send_datagram(&mut noop(), &[0; MAX_DATAGRAM_SIZE + 1])).unwrap(); + a.send_datagram(vec![0; MAX_DATAGRAM_SIZE + 1].into(), None, 0).unwrap(); for _ in 0..DATAGRAM_QUEUE { - ready(a.poll_send_datagram(&mut noop(), b"one")).unwrap(); + a.send_datagram("one".into(), None, 0).unwrap(); } // The peer's queue is full, so this one is lost, like the oversized one. - ready(a.poll_send_datagram(&mut noop(), b"two")).unwrap(); + a.send_datagram("two".into(), None, 0).unwrap(); for _ in 0..DATAGRAM_QUEUE { assert_eq!(ready(b.poll_recv_datagram(&mut noop())).unwrap(), "one"); } diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index d3153c2f2c..efb63c3f6f 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -600,11 +600,11 @@ impl poll::Session for MockSession { None } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { + fn send_datagram(&mut self, payload: Bytes, group: Option<&poll::Ungrouped>, order: i64) -> Result<(), Self::Error> { if self.side.lossy.load(Ordering::Relaxed) { - return Poll::Ready(Ok(())); + return Ok(()); } - self.inner.poll_send_datagram(cx, payload) + self.inner.send_datagram(payload, group, order) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index b7ed475a02..61fff81933 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -67,6 +67,7 @@ Changes on top of the upstream commit, besides the renames: quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. - **Instants on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. - **Send groups** (`streams/schedule.rs`, `Connection::send_groups`): streams outside every group go first, in quinn's order; groups in one scheduling domain are served strictly by priority, with equal priorities sharing bytes by deficit round robin; domains share bytes the same way. A stream's priority is an `i64` ordering it only within its group. This replaces quinn's single pending-stream heap, so a cherry-pick touching `PendingStreamsQueue` has to be redone against the schedule. moq-tokio's `quic::driver` and moq-uring's driver wrap a group in a shared `SendGroup` handle that closes on drop. +- **Scheduled datagrams** (`Datagrams::send`): a datagram is queued in a send group (or ungrouped) at an `i64` order and scheduled with the group's streams, winning a tie with a stream of equal order and spending the group's byte credit, instead of quinn's datagram-first FIFO in `populate_packet`. Sending never blocks: `datagram_send_buffer_size` is shared, and an overflow evicts the oldest datagram of the lowest-priority group in the domain holding the most datagram bytes. `SendDatagramError::Blocked` and `Event::DatagramsUnblocked` are gone, and a cherry-pick touching the `// DATAGRAM` loop has to be redone against `StreamsState::write_frames`. - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/transport.rs b/rs/moq-quic/src/config/transport.rs index 48e26c5b66..a0e5428987 100644 --- a/rs/moq-quic/src/config/transport.rs +++ b/rs/moq-quic/src/config/transport.rs @@ -324,9 +324,12 @@ impl TransportConfig { /// While datagrams are sent ASAP, it is possible for an application to generate data faster /// than the link, or even the underlying hardware, can transmit them. This limits the amount of /// memory that may be consumed in that case. When the send buffer is full and a new datagram is - /// sent, older datagrams are dropped until sufficient space is available. + /// sent, the domain holding the most datagram bytes drops the oldest datagram of its + /// lowest-priority send group until the new one fits; see + /// [`Datagrams::send`](crate::Datagrams::send). /// - /// The amount of payload data buffered may be smaller than `value` due to overhead. + /// Counts each datagram's frame bytes, so the payload buffered is a little smaller than + /// `value`. pub fn datagram_send_buffer_size(&mut self, value: usize) -> &mut Self { self.datagram_send_buffer_size = value; self diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index 2e0f1494d0..96bb98447c 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -423,7 +423,8 @@ impl<'a> SendStream<'a> { /// A send group shares bytes with the other groups of its domain: a higher priority group /// preempts every lower one, and groups of equal priority get an equal share of bytes no /// matter how many streams each has queued. Domains share bytes equally whatever their -/// groups' priorities. Streams outside any group go before all of it. +/// groups' priorities. Streams outside any group go before all of it. A group's datagrams, +/// from [`Datagrams::send`](crate::Datagrams::send), share its bytes in the same order. pub struct SendGroups<'a> { pub(super) state: &'a mut StreamsState, } diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs index 898c2625ff..8e39202a1a 100644 --- a/rs/moq-quic/src/connection/streams/schedule.rs +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -254,6 +254,7 @@ impl Schedule { } /// Whether nothing is queued. + #[cfg(test)] pub(super) fn is_empty(&self) -> bool { self.ungrouped.is_empty() && self.backlogged == 0 } diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 55bfcbeec3..f17f2d822f 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -521,27 +521,32 @@ impl Connection { } } - /// Transmit `data` as an unreliable, unordered application datagram + /// Transmit `data` as an unreliable, unordered application datagram in `group` at `order` /// /// Application datagrams are a low-level primitive. They may be lost or delivered out of order, /// and `data` must both fit inside a single QUIC packet and be smaller than the maximum /// dictated by the peer. /// - /// Previously queued datagrams which are still unsent may be discarded to make space for this - /// datagram, in order of oldest to newest. - pub fn send_datagram(&self, data: Bytes) -> Result<(), SendDatagramError> { + /// The datagram is scheduled like one of the group's streams at priority `order`, or among + /// the ungrouped streams without a group. Sending never blocks: a full buffer drops the + /// oldest datagram of the lowest-priority group, see [`moq_quic::Datagrams::send`]. + pub fn send_datagram( + &self, + data: Bytes, + group: Option<&SendGroup>, + order: i64, + ) -> Result<(), SendDatagramError> { let conn = &mut *self.0.state.lock("send_datagram"); if let Some(x) = &conn.error { return Err(SendDatagramError::ConnectionLost(x.clone())); } use moq_quic::SendDatagramError::*; - match conn.inner.datagrams().send(data, true) { + match conn.inner.datagrams().send(data, group.map(SendGroup::id), order) { Ok(()) => { conn.wake(); Ok(()) } Err(e) => Err(match e { - Blocked(..) => unreachable!(), UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, Disabled => SendDatagramError::Disabled, TooLarge => SendDatagramError::TooLarge, @@ -549,22 +554,6 @@ impl Connection { } } - /// Transmit `data` as an unreliable, unordered application datagram - /// - /// Unlike [`send_datagram()`], this method will wait for buffer space during congestion - /// conditions, which effectively prioritizes old datagrams over new datagrams. - /// - /// See [`send_datagram()`] for details. - /// - /// [`send_datagram()`]: Connection::send_datagram - pub fn send_datagram_wait(&self, data: Bytes) -> SendDatagram<'_> { - SendDatagram { - conn: &self.0, - data: Some(data), - notify: self.0.shared.datagrams_unblocked.notified(), - } - } - /// Compute the maximum size of datagrams that may be passed to [`send_datagram()`]. /// /// Returns `None` if datagrams are unsupported by the peer or disabled locally. @@ -971,55 +960,6 @@ impl Future for ReadDatagram<'_> { } } -pin_project! { - /// Future produced by [`Connection::send_datagram_wait`] - pub struct SendDatagram<'a> { - conn: &'a ConnectionRef, - data: Option, - #[pin] - notify: Notified<'a>, - } -} - -impl Future for SendDatagram<'_> { - type Output = Result<(), SendDatagramError>; - fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { - let mut this = self.project(); - let mut state = this.conn.state.lock("SendDatagram::poll"); - if let Some(e) = &state.error { - return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); - } - use moq_quic::SendDatagramError::*; - match state - .inner - .datagrams() - .send(this.data.take().unwrap(), false) - { - Ok(()) => { - state.wake(); - Poll::Ready(Ok(())) - } - Err(e) => Poll::Ready(Err(match e { - Blocked(data) => { - this.data.replace(data); - loop { - match this.notify.as_mut().poll(ctx) { - Poll::Pending => return Poll::Pending, - // Spurious wakeup, get a new future - Poll::Ready(()) => this - .notify - .set(this.conn.shared.datagrams_unblocked.notified()), - } - } - } - UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, - Disabled => SendDatagramError::Disabled, - TooLarge => SendDatagramError::TooLarge, - })), - } - } -} - /// Watcher produced by [`Connection::watch_remote_address`] pub struct RemoteAddressWatcher { receiver: watch::Receiver, @@ -1095,7 +1035,6 @@ pub(crate) struct Shared { /// Notified when the peer has initiated a new stream stream_incoming: [Notify; 2], datagram_received: Notify, - datagrams_unblocked: Notify, closed: Notify, connected: Arc, /// Number of live handles that can used to initiate or handle I/O; excludes the driver @@ -1299,9 +1238,6 @@ impl State { DatagramReceived => { shared.datagram_received.notify_waiters(); } - DatagramsUnblocked => { - shared.datagrams_unblocked.notify_waiters(); - } Stream(StreamEvent::Readable { id }) => wake_stream(id, &mut self.blocked_readers), Stream(StreamEvent::Available { dir }) => { // Might mean any number of streams are ready, so we wake up everyone @@ -1383,7 +1319,6 @@ impl State { shared.stream_incoming[Dir::Uni as usize].notify_waiters(); shared.stream_incoming[Dir::Bi as usize].notify_waiters(); shared.datagram_received.notify_waiters(); - shared.datagrams_unblocked.notify_waiters(); shared.handshake_confirmed.notify_waiters(); wake_all_notify(&mut self.stopped); shared.closed.notify_waiters(); diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index af3e46f00e..531cc8331d 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -69,7 +69,7 @@ pub use moq_quic::{QlogConfig, QlogStream}; pub use crate::quic::driver::connection::{ AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, - RemoteAddressWatcher, SendDatagram, SendDatagramError, + RemoteAddressWatcher, SendDatagramError, }; pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index de22289b70..d7846d30e9 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1570,9 +1570,9 @@ async fn two_datagram_readers() { x }, async { - server.send_datagram(b"one"[..].into()).unwrap(); + server.send_datagram(b"one"[..].into(), None, 0).unwrap(); done.notified().await; - server.send_datagram_wait(b"two"[..].into()).await.unwrap(); + server.send_datagram(b"two"[..].into(), None, 0).unwrap(); } ); assert!(*a == *b"one" || *b == *b"one"); diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index b55cbe1437..caee77d9b0 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -324,34 +324,18 @@ impl Session { Ok(datagram) } - /// Sends an application datagram to the remote peer. + /// Sends an application datagram to the remote peer in `group` at `order`. /// /// Datagrams are unreliable and may be dropped or delivered out of order. /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the driver API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram(buf.into()) - } else { - self.conn.send_datagram(data) - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Sends an application datagram to the remote peer, waiting for buffer space if needed. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + /// See [`Connection::send_datagram`](crate::quic::driver::Connection::send_datagram) for how + /// `group` and `order` schedule it. + pub fn send_datagram( + &self, + data: Bytes, + group: Option<&crate::quic::driver::SendGroup>, + order: i64, + ) -> Result<(), SessionError> { let result = if !self.header_datagram.is_empty() { // Unfortunately, we need to allocate/copy each datagram because of the driver API. // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 @@ -361,9 +345,9 @@ impl Session { buf.extend_from_slice(&self.header_datagram); buf.extend_from_slice(&data); - self.conn.send_datagram_wait(buf.into()).await + self.conn.send_datagram(buf.into(), group, order) } else { - self.conn.send_datagram_wait(data).await + self.conn.send_datagram(data, group, order) }; result.map_err(|e| self.map_error(e))?; @@ -1020,7 +1004,7 @@ impl web_transport_trait::Session for Session { } fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { - Self::send_datagram(self, data) + Self::send_datagram(self, data, None, 0) } async fn recv_datagram(&self) -> Result { diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 33d7e27fdd..da7482dc86 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -206,7 +206,10 @@ where fn poll_send_datagram(&mut self, _cx: &mut Context<'_>, payload: &[u8]) -> Poll> { // The async interface's send is non-blocking (the transport queues or // drops), so there is never a reason to return Pending. - Poll::Ready(self.session.send_datagram(Bytes::copy_from_slice(payload))) + Poll::Ready(web_transport_trait::Session::send_datagram( + &self.session, + Bytes::copy_from_slice(payload), + )) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { @@ -859,6 +862,15 @@ impl Grouped for crate::quic::web::Session { fn send_group(&self, priority: i32) -> Option { Some(Self::send_group(self, priority)) } + + fn send_datagram( + &self, + payload: Bytes, + group: Option<&crate::quic::driver::SendGroup>, + order: i64, + ) -> Result<(), Self::Error> { + Self::send_datagram(self, payload, group, order) + } } #[cfg(feature = "quic")] diff --git a/rs/moq-tokio/src/transport/grouped.rs b/rs/moq-tokio/src/transport/grouped.rs index 4273dc29da..de706c47f4 100644 --- a/rs/moq-tokio/src/transport/grouped.rs +++ b/rs/moq-tokio/src/transport/grouped.rs @@ -15,6 +15,17 @@ pub trait Grouped: web_transport_trait::Session { let _ = priority; None } + + /// Send a datagram in `group` at `order`; a backend without groups ignores both. + fn send_datagram( + &self, + payload: bytes::Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error> { + let _ = (group, order); + web_transport_trait::Session::send_datagram(self, payload) + } } /// A backend send stream that may join its [`Grouped`] session's send groups. diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 1ae9d06b25..12f69c5459 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -63,8 +63,13 @@ impl> poll::Session fn send_group(&mut self, priority: i32) -> Option<::Group> { self.session.send_group(priority) } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - backend::Session::poll_send_datagram(self, cx, payload).map_err(Error) + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error> { + super::Grouped::send_datagram(&self.session, payload, group, order).map_err(Error) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { backend::Session::poll_recv_datagram(self, cx).map_err(Error) diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 75a8cb4bb0..c452f3dcc8 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -72,7 +72,6 @@ pub(crate) struct State { sends: FxHashMap>, datagram_recv_waiters: kio::WaiterList, - datagram_send_waiters: kio::WaiterList, /// The terminal error, set exactly once; everything fails with it after. closed: Option, @@ -103,7 +102,6 @@ impl State { finishing: FxHashMap::default(), sends: FxHashMap::default(), datagram_recv_waiters: kio::WaiterList::new(), - datagram_send_waiters: kio::WaiterList::new(), closed: None, closed_waiters: kio::WaiterList::new(), dead: false, @@ -139,7 +137,6 @@ impl State { self.accept_uni_waiters.wake(); self.open_waiters.wake(); self.datagram_recv_waiters.wake(); - self.datagram_send_waiters.wake(); for waiters in self.readable.values_mut() { waiters.wake(); } @@ -321,6 +318,25 @@ impl Connection { super::SendGroup::new(self.shared.clone(), priority) } + /// Send a datagram in `group` at `order`, scheduled like one of the group's streams at + /// priority `order`, or among the ungrouped streams without a group. + /// + /// Never blocks: a full buffer drops the oldest datagram of the lowest-priority group, + /// see [`moq_quic::Datagrams::send`]. + pub fn send_datagram(&self, payload: Bytes, group: Option<&super::SendGroup>, order: i64) -> Result<(), Error> { + if let Some(err) = self.shared.closed() { + return Err(err); + } + self.shared + .conn + .borrow_mut() + .datagrams() + .send(payload, group.map(super::SendGroup::id), order) + .map_err(|err| Error::Quic(err.to_string()))?; + self.shared.kick(); + Ok(()) + } + /// The peer's certificate chain in DER, leaf first, or `None` if it /// presented none. /// @@ -570,25 +586,8 @@ impl web_transport_trait::poll::Session for Connection { } } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - let waiter = self.park.hold(cx); - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - let payload = Bytes::copy_from_slice(payload); - match self.shared.conn.borrow_mut().datagrams().send(payload, false) { - Ok(()) => { - self.shared.kick(); - Poll::Ready(Ok(())) - } - // The send queue is full; a flush frees space. - Err(moq_quic::SendDatagramError::Blocked(_)) => { - let mut state = self.shared.state.borrow_mut(); - waiter.register(&mut state.datagram_send_waiters); - Poll::Pending - } - Err(err) => Poll::Ready(Err(Error::Quic(err.to_string()))), - } + fn poll_send_datagram(&mut self, _: &mut Context<'_>, payload: &[u8]) -> Poll> { + Poll::Ready(Self::send_datagram(self, Bytes::copy_from_slice(payload), None, 0)) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { @@ -813,7 +812,6 @@ impl Driver { } moq_quic::Event::ConnectionLost { reason } => state.fail(reason.into()), moq_quic::Event::DatagramReceived => state.datagram_recv_waiters.wake(), - moq_quic::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), moq_quic::Event::HandshakeDataReady => {} moq_quic::Event::Stream(event) => sweep_stream(&mut state, event), // Non-exhaustive upstream; nothing else needs a wakeup. @@ -898,8 +896,6 @@ impl Driver { return Poll::Ready(Err(Error::Io(err.to_string()))); } self.close_staged |= closing; - // A flush frees datagram-send queue space. - self.shared.state.borrow_mut().datagram_send_waiters.wake(); Poll::Ready(Ok(())) } } diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 7f026caad0..47a700b500 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -446,6 +446,19 @@ impl Session { self.conn.send_group(priority) } + /// Send a datagram in `group` at `order`, scheduled like one of the group's streams. + /// + /// See [`Connection::send_datagram`]. + pub fn send_datagram(&self, payload: Bytes, group: Option<&super::SendGroup>, order: i64) -> Result<(), Error> { + let Some(web) = &self.web else { + return self.conn.send_datagram(payload, group, order); + }; + let mut framed = BytesMut::with_capacity(web.header_datagram.len() + payload.len()); + framed.extend_from_slice(&web.header_datagram); + framed.extend_from_slice(&payload); + self.conn.send_datagram(framed.freeze(), group, order).map_err(unmap_err) + } + /// Wrap a raw-QUIC connection in a session, with no /// WebTransport layering: streams and datagrams pass through untouched. pub fn raw(conn: Connection) -> Self { @@ -663,14 +676,8 @@ impl web_transport_trait::poll::Session for Session { ))) } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - let Some(web) = &self.web else { - return web_transport_trait::poll::Session::poll_send_datagram(&mut self.conn, cx, payload); - }; - let mut framed = Vec::with_capacity(web.header_datagram.len() + payload.len()); - framed.extend_from_slice(&web.header_datagram); - framed.extend_from_slice(payload); - web_transport_trait::poll::Session::poll_send_datagram(&mut self.conn, cx, &framed).map_err(unmap_err) + fn poll_send_datagram(&mut self, _: &mut Context<'_>, payload: &[u8]) -> Poll> { + Poll::Ready(Self::send_datagram(self, Bytes::copy_from_slice(payload), None, 0)) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 9df40de448..75e9ac70d1 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -9,6 +9,15 @@ impl Grouped for quic::web::Session { fn send_group(&mut self, priority: i32) -> Option { Some(quic::web::Session::send_group(self, priority)) } + + fn send_datagram( + &mut self, + payload: bytes::Bytes, + group: Option<&quic::SendGroup>, + order: i64, + ) -> Result<(), Self::Error> { + quic::web::Session::send_datagram(self, payload, group, order) + } } impl GroupedSend for quic::web::SendStream { diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 3e5b01b059..94840b63cb 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -59,6 +59,22 @@ pub trait Grouped: backend::Session { let _ = priority; None } + + /// Send a datagram in `group` at `order`; a backend without groups ignores both and + /// drops the datagram when it has no room. + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error> { + let _ = (group, order); + let mut cx = Context::from_waker(std::task::Waker::noop()); + match backend::Session::poll_send_datagram(self, &mut cx, &payload) { + Poll::Ready(res) => res, + Poll::Pending => Ok(()), + } + } } /// A backend send stream that may join its [`Grouped`] session's send groups. @@ -113,8 +129,13 @@ impl + Clone + 'static> pol fn send_group(&mut self, priority: i32) -> Option<::Group> { Grouped::send_group(&mut self.0, priority) } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&::Group>, + order: i64, + ) -> Result<(), Self::Error> { + Grouped::send_datagram(&mut self.0, payload, group, order).map_err(Error) } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { backend::Session::poll_recv_datagram(&mut self.0, cx).map_err(Error) diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index ffdd71de50..03150e22fe 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -142,7 +142,7 @@ fn webtransport_echo_end_to_end() { assert_eq!(read_all(&mut recv).await, PAYLOAD, "uni echo"); // Datagram echo. - session.send_datagram(PAYLOAD.to_vec().into()).expect("send datagram"); + session.send_datagram(PAYLOAD.to_vec().into(), None, 0).expect("send datagram"); let echoed = session.read_datagram().await.expect("read datagram"); assert_eq!(&echoed[..], PAYLOAD, "datagram echo"); diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 6a3fb5b14e..d24ace8108 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -80,8 +80,14 @@ impl + Clone + 'st fn send_group(&mut self, _: i32) -> Option { None } - fn poll_send_datagram(&mut self, cx: &mut Context<'_>, payload: &[u8]) -> Poll> { - backend::Session::poll_send_datagram(&mut self.0, cx, payload).map_err(Error) + /// The backend cannot rank a datagram against a stream, so the group and order are ignored, + /// and a datagram it has no room for is dropped. + fn send_datagram(&mut self, payload: Bytes, _: Option<&poll::Ungrouped>, _: i64) -> Result<(), Self::Error> { + let mut cx = Context::from_waker(std::task::Waker::noop()); + match backend::Session::poll_send_datagram(&mut self.0, &mut cx, &payload) { + Poll::Ready(res) => res.map_err(Error), + Poll::Pending => Ok(()), + } } fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { backend::Session::poll_recv_datagram(&mut self.0, cx).map_err(Error) From 9448b2dad4798c7f421ca2006dfa44c9e306234b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:29:17 -0700 Subject: [PATCH 23/37] style: just fix Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/support/mock.rs | 7 +++++- .../src/connection/streams/saturated.rs | 22 +++++++++++++++---- .../src/connection/streams/schedule.rs | 15 +++++++++---- rs/moq-tokio/src/quic/driver/connection.rs | 6 ++++- rs/moq-uring/src/quic/web.rs | 4 +++- rs/moq-uring/tests/web.rs | 4 +++- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index 0f962cec2b..d5ca709ac4 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -600,7 +600,12 @@ impl poll::Session for MockSession { None } - fn send_datagram(&mut self, payload: Bytes, group: Option<&poll::Ungrouped>, order: i64) -> Result<(), Self::Error> { + fn send_datagram( + &mut self, + payload: Bytes, + group: Option<&poll::Ungrouped>, + order: i64, + ) -> Result<(), Self::Error> { if self.side.lossy.load(Ordering::Relaxed) { return Ok(()); } diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs index 402df2e7ab..b2669ee105 100644 --- a/rs/moq-quic/src/connection/streams/saturated.rs +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -125,7 +125,9 @@ impl Sender { let mut data = BytesMut::with_capacity(bytes); data.put_u64(tag); data.resize(bytes, 0); - let datagram = frame::Datagram { data: data.freeze() }; + let datagram = frame::Datagram { + data: data.freeze(), + }; assert!( self.state .queue_datagram(datagram, group, order, usize::MAX) @@ -531,8 +533,16 @@ fn low_priority_datagrams_wait_behind_streams() { .iter() .position(|sent| matches!(sent, Sent::Datagram(_))) .unwrap(); - assert!(order[..first].iter().all(|&sent| sent == Sent::Stream(stream))); - assert!(order[first..].iter().all(|sent| matches!(sent, Sent::Datagram(_)))); + assert!( + order[..first] + .iter() + .all(|&sent| sent == Sent::Stream(stream)) + ); + assert!( + order[first..] + .iter() + .all(|sent| matches!(sent, Sent::Datagram(_))) + ); assert_eq!(sender.bytes(&[stream]), 20_000); assert_eq!(order[first..].len(), tags.len()); } @@ -552,7 +562,11 @@ fn high_priority_datagrams_preempt_streams() { // Newest first, as the group orders them. let expected: Vec<_> = tags.iter().rev().map(|&tag| Sent::Datagram(tag)).collect(); assert_eq!(order[..tags.len()], expected[..]); - assert!(order[tags.len()..].iter().all(|&sent| sent == Sent::Stream(stream))); + assert!( + order[tags.len()..] + .iter() + .all(|&sent| sent == Sent::Stream(stream)) + ); } /// Inside one group a datagram merges into the group order: it waits behind a stream diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs index 819784c692..a4c2499e67 100644 --- a/rs/moq-quic/src/connection/streams/schedule.rs +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -434,7 +434,10 @@ impl Schedule { budget: usize, ) -> bool { if let Some(group) = group { - assert!(!self.groups[group].closed, "datagram sent on a closed group"); + assert!( + !self.groups[group].closed, + "datagram sent on a closed group" + ); } self.recency -= 1; @@ -492,7 +495,11 @@ impl Schedule { /// A group this empties restarts its turn when restored, gaining up to a quantum of credit, /// so this is for the rare packet clamped below the path MTU, not every full one. pub(super) fn defer_datagram(&mut self, group: Option) { - let (key, item) = self.queue(group).items.pop_last().expect("a datagram heads"); + let (key, item) = self + .queue(group) + .items + .pop_last() + .expect("a datagram heads"); self.deferred.push((group, key, item)); // Its group stays allocated: the datagram is still indexed for eviction. self.deactivate(group); @@ -1034,7 +1041,7 @@ mod tests { // Full: the next evicts the low group's oldest, then its newest. let (high_new, queued) = sim.datagram(Some(high), 0, 100, 400); assert!(queued); - assert_eq!(sim.datagram(Some(high), 0, 100, 400).1, true); + assert!(sim.datagram(Some(high), 0, 100, 400).1); assert_eq!(sim.drain(), [ungrouped, high_old, high_new, high_new + 1]); let _ = (low_old, low_new); } @@ -1045,7 +1052,7 @@ mod tests { let high = sim.schedule.open_group(1, None); let low = sim.schedule.open_group(0, None); let kept = sim.datagram(Some(high), 0, 100, 100).0; - assert_eq!(sim.datagram(Some(low), 0, 100, 100).1, false); + assert!(!sim.datagram(Some(low), 0, 100, 100).1); assert_eq!(sim.schedule.datagram_bytes(), 100); assert!(sim.schedule.is_clean()); assert_eq!(sim.drain(), [kept]); diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 2bd31a65c6..8de19c7613 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -541,7 +541,11 @@ impl Connection { return Err(SendDatagramError::ConnectionLost(x.clone())); } use moq_quic::SendDatagramError::*; - match conn.inner.datagrams().send(data, group.map(SendGroup::id), order) { + match conn + .inner + .datagrams() + .send(data, group.map(SendGroup::id), order) + { Ok(()) => { conn.wake(); Ok(()) diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index af3cc0b6b5..fce701fe2f 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -456,7 +456,9 @@ impl Session { let mut framed = BytesMut::with_capacity(web.header_datagram.len() + payload.len()); framed.extend_from_slice(&web.header_datagram); framed.extend_from_slice(&payload); - self.conn.send_datagram(framed.freeze(), group, order).map_err(unmap_err) + self.conn + .send_datagram(framed.freeze(), group, order) + .map_err(unmap_err) } /// Wrap a raw-QUIC connection in a session, with no diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index 03150e22fe..15de7c3efd 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -142,7 +142,9 @@ fn webtransport_echo_end_to_end() { assert_eq!(read_all(&mut recv).await, PAYLOAD, "uni echo"); // Datagram echo. - session.send_datagram(PAYLOAD.to_vec().into(), None, 0).expect("send datagram"); + session + .send_datagram(PAYLOAD.to_vec().into(), None, 0) + .expect("send datagram"); let echoed = session.read_datagram().await.expect("read datagram"); assert_eq!(&echoed[..], PAYLOAD, "datagram echo"); From e83baf028d1d3e4911535aaaf9f2b3e7fa16baa0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:30:24 -0700 Subject: [PATCH 24/37] test(quic): datagram priority holds across a congestion-limited flight Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/tests/mod.rs | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 42e438e836..08dbcadeb0 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -2969,6 +2969,54 @@ fn datagram_send_recv() { assert_matches!(pair.server_datagrams(server_ch).recv(), None); } +/// Writes more than a congestion window to a stream in a group at `stream`, queues datagrams in +/// a group at `datagram`, and returns how many datagrams the first flight carried. +fn first_flight_datagrams(stream: i64, datagram: i64) -> usize { + const DATAGRAMS: usize = 8; + let _guard = subscribe(); + let mut pair = Pair::default(); + let (client_ch, server_ch) = pair.connect(); + let conn = pair.server_conn_mut(server_ch); + let streams = conn.send_groups().open(stream, None); + let datagrams = conn.send_groups().open(datagram, None); + let s = conn.streams().open(Dir::Uni).unwrap(); + conn.send_stream(s).set_group(Some(&streams)).unwrap(); + conn.send_stream(s).write(&[0; 256 * 1024]).unwrap(); + for _ in 0..DATAGRAMS { + conn.datagrams() + .send(vec![1; 100].into(), Some(&datagrams), 0) + .unwrap(); + } + + // One congestion-limited flight. + pair.drive_server(); + pair.drive_client(); + assert!(pair.client_streams(client_ch).accept(Dir::Uni).is_some()); + let mut first = 0; + while pair.client_datagrams(client_ch).recv().is_some() { + first += 1; + } + + // Nothing is dropped, only ordered. + pair.drive(); + let mut rest = 0; + while pair.client_datagrams(client_ch).recv().is_some() { + rest += 1; + } + assert_eq!(first + rest, DATAGRAMS); + first +} + +#[test] +fn low_priority_datagrams_wait_behind_streams() { + assert_eq!(first_flight_datagrams(1, 0), 0); +} + +#[test] +fn high_priority_datagrams_preempt_streams() { + assert_eq!(first_flight_datagrams(0, 1), 8); +} + #[test] fn datagram_recv_buffer_overflow() { let _guard = subscribe(); From 39dbc17e7b4ff76de12a992468eced40b39412fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:40:20 -0700 Subject: [PATCH 25/37] quest(m2): plan dropping web-transport-trait from the Rust stack Custom transports implement moq-net's transport::poll traits, which already carry send groups. The generic web_transport_trait adapters, whose bound #5711 sealed to the crates' own backends, are deleted along with the sealed Grouped/GroupedSend pair. quic-qmux's port now targets moq-net's traits. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 + quest/m2/drop-web-transport-trait.md | 47 ++++++++++++++++++++++++++++ quest/m2/quic-qmux.md | 3 +- 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 quest/m2/drop-web-transport-trait.md diff --git a/quest/m2/README.md b/quest/m2/README.md index b955da7d25..0ab57c8fbc 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -119,6 +119,7 @@ ranked just below deterministic TS legs, which it requires. become resets, and a tail loss probe fires early while there is still time - [qmux on the QUIC stream state machine](/quest/m2/quic-qmux.md) - qmux is a first-class crate over `moq-quic`'s shared stream state machine +- [Drop web-transport-trait](/quest/m2/drop-web-transport-trait.md) - every backend implements moq-net's `transport::poll` traits, and the generic `web_transport_trait` adapters are deleted - [Align BBR loss handling with draft-06](/quest/m2/quic-bbr-loss-parity.md) - losses use their own sample and the draft-05 links go, keeping the spurious-loss undo that already re-enters ProbeUp through Refill - [Measure ECN on the backbone](/quest/m2/quic-ecn-measure.md) - a written verdict on marking versus dropping, and whether Linode and OVH keep marks diff --git a/quest/m2/drop-web-transport-trait.md b/quest/m2/drop-web-transport-trait.md new file mode 100644 index 0000000000..3c05dd0b47 --- /dev/null +++ b/quest/m2/drop-web-transport-trait.md @@ -0,0 +1,47 @@ +# [L] Drop web-transport-trait from the Rust stack + +## Goal + +moq-tokio, moq-uring, moq-relay, moq-ffi and moq-wasm no longer depend on +`web-transport-trait`. The qmux, iroh and wasm backends implement moq-net's +`transport::poll` traits, either directly or through a thin adapter per +backend. The generic adapters over `web_transport_trait` are deleted: +moq-tokio's `transport::Session` with its async-to-poll emulation, and +moq-uring's `transport::Session`. The sealed `Grouped`/`GroupedSend` pair +goes with them. A custom transport implements moq-net's traits, which are +already generic and send-group-aware (`Session::send_group`, +`SendStream::Group`, `Ungrouped`). + +## Plan + +Decided 2026-10-16 in the review of #5711: the extension point for custom +transports is moq-net's `transport::poll`, not `web_transport_trait`. #5711 +sealed the adapters' bound to the crate's own backends, so a foreign +`web_transport_trait::Session` no longer fits them. This quest removes the +layer instead of reopening it. + +- Native QUIC (`moq_tokio::quic::web`, `moq_uring::quic::web`) implements + `poll::Session` itself, keeping its real send groups. It is a candidate for + the first step, since its `web_transport_trait` impl only feeds the adapter. +- qmux and iroh get their own impls. Flat backends use `Ungrouped`. +- The browser backend in moq-wasm and the ffi backend implement the traits + directly. Their adapters already clamp priorities themselves. +- Report the public API impact per crate: deleted generic `Session` + adapters, and new impls on concrete session types. + +Notes from related quests: +- [qmux on moq-quic](/quest/m2/quic-qmux.md) planned to port qmux's + `web-transport-trait` impl. If it lands first, its impl targets moq-net's + `transport::poll` instead. +- [poll_acked](/quest/m2/quic-ack-hook.md) already puts its hook on moq-net's + trait because moq-net has no `web-transport-trait` dependency. That holds + here. +- [Rust request credit](/quest/m1/rs-request-credit.md) prefers adding a + credit query to qmux and iroh upstream in moq-dev/web-transport. Once this + lands, that query belongs on each backend's own moq-net impl. + +## Related + +- [qmux on the QUIC stream state machine](/quest/m2/quic-qmux.md) - the in-tree qmux implements the moq-net traits directly +- [poll_acked on moq-net's send stream](/quest/m2/quic-ack-hook.md) - another hook that lives only on moq-net's traits +- [Rust requests fail fast without stream credit](/quest/m1/rs-request-credit.md) - its credit query moves to each backend's moq-net impl diff --git a/quest/m2/quic-qmux.md b/quest/m2/quic-qmux.md index 716cd8fd5f..ec3ca9fb20 100644 --- a/quest/m2/quic-qmux.md +++ b/quest/m2/quic-qmux.md @@ -16,7 +16,8 @@ Decided in the 2026-09-30 plan: - **Scope is the Rust core plus adapters.** Port the prototype core and moq-dev/web-transport's TCP, TLS, WebSocket, and UDS adapters, ALPN negotiation, byte-stream framing, and socket stats (about 2.3k lines) onto - it, along with the `web-transport-trait` impl. The TypeScript `@moq/qmux` + it. Its session implements moq-net's `transport::poll` traits instead of + `web-transport-trait` ([Drop web-transport-trait](/quest/m2/drop-web-transport-trait.md)). The TypeScript `@moq/qmux` and `@moq/web-socket-stream` stay in moq-dev/web-transport with their own state machine and become the interop reference. - **All four wire versions stay**: draft-02, draft-01, draft-00, and the From 3a4a040d25061147d90f98982f190b3793c94c5d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:12:15 -0700 Subject: [PATCH 26/37] docs: send groups schedule datagrams too Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/test_transport.rs | 7 +++++-- rs/moq-tokio/src/quic/driver/send_group.rs | 9 +++++---- rs/moq-uring/src/quic/driver/group.rs | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 99b7d878d9..4772814717 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -35,13 +35,16 @@ impl crate::transport::Error for SinkError { } } +/// A datagram's send group id and in-group order. +pub type Rank = (Option, i64); + /// What the session's send streams recorded. Resets are a list, not a last-value, so a /// stray second reset (a drop fallback firing behind an abort) is visible. #[derive(Clone, Default)] pub struct Log { pub datagrams: Arc>>, /// The send group and order each datagram was sent with, beside [`Self::datagrams`]. - datagram_ranks: Arc, i64)>>>, + datagram_ranks: Arc>>, pub writes: Arc>>, pub resets: Arc>>, /// Every write, FIN and reset on every send stream, in call order. See [`Self::trail`]. @@ -119,7 +122,7 @@ impl Log { } /// The send group and order of every datagram, as (group, order) in call order. - pub fn datagram_ranks(&self) -> Vec<(Option, i64)> { + pub fn datagram_ranks(&self) -> Vec { self.datagram_ranks.lock().unwrap().clone() } diff --git a/rs/moq-tokio/src/quic/driver/send_group.rs b/rs/moq-tokio/src/quic/driver/send_group.rs index 41cd2e0ab4..9e8adc793f 100644 --- a/rs/moq-tokio/src/quic/driver/send_group.rs +++ b/rs/moq-tokio/src/quic/driver/send_group.rs @@ -7,11 +7,12 @@ use crate::quic::driver::connection::ConnectionInner; /// Streams that [join](crate::quic::driver::SendStream::set_group) a group share bytes with /// the connection's other groups: a higher priority group preempts every lower one, and groups /// of equal priority get an equal share of bytes however many streams each has queued. A -/// stream's own priority orders it only against the group's other streams. Streams outside any -/// group go before every group. See [`moq_quic::SendGroups`]. +/// stream's own priority orders it only against the group's other streams, and a datagram sent +/// in the group is ordered among them by its `order`. Streams and datagrams outside any group go +/// before every group. See [`moq_quic::SendGroups`]. /// -/// Clones share one group, released once the last clone is dropped and its streams finish. A -/// group does not keep its connection open. +/// Clones share one group, released once the last clone is dropped and its streams finish +/// and its datagrams are sent. A group does not keep its connection open. #[derive(Debug, Clone)] pub struct SendGroup(Arc); diff --git a/rs/moq-uring/src/quic/driver/group.rs b/rs/moq-uring/src/quic/driver/group.rs index a90b1e2e97..bbe81976a8 100644 --- a/rs/moq-uring/src/quic/driver/group.rs +++ b/rs/moq-uring/src/quic/driver/group.rs @@ -9,10 +9,12 @@ use super::Shared; /// Streams that [join](super::SendStream::set_group) a group share bytes with the /// connection's other groups: a higher priority group preempts every lower one, and groups of /// equal priority get an equal share of bytes however many streams each has queued. A stream's -/// own priority orders it only against the group's other streams. Streams outside any group go +/// own priority orders it only against the group's other streams, and a datagram sent in the +/// group is ordered among them by its `order`. Streams and datagrams outside any group go /// before every group. See [`moq_quic::SendGroups`]. /// -/// Clones share one group, released once the last clone is dropped and its streams finish. +/// Clones share one group, released once the last clone is dropped and its streams finish +/// and its datagrams are sent. #[derive(Clone)] pub struct SendGroup(Rc); From ade4a19aba6a5e02ef12d7dda6fe23787ea96889 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:21:14 -0700 Subject: [PATCH 27/37] docs(lite): order a datagram with its subscription's streams Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 5 +++++ drafts/draft-lcurley-moq-lite.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 55b74b56bd..6e9236321d 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -254,6 +254,11 @@ datagram: unreliable, unordered, under about 1200 bytes, and never retransmitted. It suits real-time audio and sensor data. There is no stream fallback, so a datagram that doesn't fit isn't delivered that way. +A datagram shares its subscription's priority and group order with streams, +so a low-priority datagram waits behind higher-priority groups instead of +jumping the queue. Sent from Rust over QUIC or WebTransport (iroh excepted), +a full send buffer drops the lowest-priority subscription's oldest datagram. + Nothing caches a datagram. A subscription gets the ones inside its group range, and a new one may get the few still in the publisher's short send buffer, but `FETCH` never returns one. A publisher that knows the sequence was diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 3f475ef672..d1687c14e8 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -596,6 +596,7 @@ The subscriber's `Group Order` determines which Group within the Track is transm A publisher SHOULD attempt to transmit streams based on these rules. This depends on the QUIC implementation and it may not be possible to get fine-grained control. +A publisher SHOULD order a [datagram](#datagrams) among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. ### Priority The `Subscriber Priority` is scoped to the connection and MAY change over the life of the subscription via SUBSCRIBE_UPDATE. @@ -1580,6 +1581,7 @@ The `Message Length` describes the payload size on the wire. - SUBSCRIBE_OK `Group` names where delivery starts when the subscription resolves, which need not be the first group sent. It is not a new floor. A subscriber does not wait for a group between the requested floor and it, but a publisher still delivers one that arrives later within Subscriber Max Age. - Corrected the SUBSCRIBE note that offset `Group Start` by 1. Only `Group End` and `Frame End` are offset so 0 can mean absent. `Group Start` is an absolute sequence, and 0 is group 0. - A subscription's range bounds datagrams like groups, and FETCH never returns a datagram. +- A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. - Assigned 0x3A NOT_FETCHABLE in the stream error table: a FETCH for a group delivered only as a datagram. - The subscriber FINs its Subscribe Stream after settling its tail; graceful session close waits for that FIN or reset. - A subscriber MAY cancel by FIN on its Subscribe Stream, and the publisher then FINs without SUBSCRIBE_END. From 8836fc43dab92cd133d8babda7508030b5e71fb0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:38:47 -0700 Subject: [PATCH 28/37] doc(moq-lite): scope equal-priority fairness to native stream data Browser WebTransport from Wasm has no send groups, and Lite datagrams bypass the stream scheduler, so neither gets the byte-fair split. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 55b74b56bd..d76cfec67e 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -197,7 +197,7 @@ Each subscription carries the knobs that decide behavior under congestion: | Knob | Effect | | --- | --- | -| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. Sent from Rust over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split bandwidth evenly by bytes, however many groups each has queued. | +| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. From a native Rust publisher over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split stream bandwidth evenly by bytes, however many groups each has queued. Datagrams are not part of that split. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. | From ddeb5232ff49a0a1ef9a14515aa8cbe8312c303b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:39:57 -0700 Subject: [PATCH 29/37] doc(moq-lite): datagrams join the fair split later Datagram priority will move datagrams into the send-group hierarchy; its quest now owns updating the Priority row when that lands. Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 2 +- quest/m1/quic/datagram-priority.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index d76cfec67e..e27b5ba15d 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -197,7 +197,7 @@ Each subscription carries the knobs that decide behavior under congestion: | Knob | Effect | | --- | --- | -| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. From a native Rust publisher over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split stream bandwidth evenly by bytes, however many groups each has queued. Datagrams are not part of that split. | +| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. From a native Rust publisher over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split stream bandwidth evenly by bytes, however many groups each has queued. Datagrams are not part of that split yet. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. | diff --git a/quest/m1/quic/datagram-priority.md b/quest/m1/quic/datagram-priority.md index ccf6e545a8..55b2d76ed5 100644 --- a/quest/m1/quic/datagram-priority.md +++ b/quest/m1/quic/datagram-priority.md @@ -103,6 +103,9 @@ on the browser's WebTransport, which cannot rank a datagram against a stream, so it ignores the handle and order and keeps its current best-effort send. Wire: none. +Docs: the Priority row in `doc/concept/moq-lite.md` says datagrams are not +part of the equal-priority byte split yet; drop that once they are. + ## Related - [Scope track priority](/quest/m1/track-priority-scope.md) - owns what a From 6b0f6084f8635de63516a81bad76033feeef5500 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:44:22 -0700 Subject: [PATCH 30/37] docs(lite): datagrams share the equal-priority split Co-Authored-By: Claude Opus 5.5 --- doc/concept/moq-lite.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 89d1030c69..27a09b7578 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -197,7 +197,7 @@ Each subscription carries the knobs that decide behavior under congestion: | Knob | Effect | | --- | --- | -| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. From a native Rust publisher over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split stream bandwidth evenly by bytes, however many groups each has queued. Datagrams are not part of that split yet. | +| **Priority** (0..255) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. From a native Rust publisher over QUIC or WebTransport (iroh excepted), subscriptions at the same priority split bandwidth evenly by bytes, streams and datagrams alike, however many groups each has queued. A browser publisher and other transports without send groups are not covered. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. | @@ -256,8 +256,9 @@ fallback, so a datagram that doesn't fit isn't delivered that way. A datagram shares its subscription's priority and group order with streams, so a low-priority datagram waits behind higher-priority groups instead of -jumping the queue. Sent from Rust over QUIC or WebTransport (iroh excepted), -a full send buffer drops the lowest-priority subscription's oldest datagram. +jumping the queue. From a native Rust publisher over QUIC or WebTransport +(iroh excepted), a full send buffer drops the lowest-priority subscription's +oldest datagram. Nothing caches a datagram. A subscription gets the ones inside its group range, and a new one may get the few still in the publisher's short send From 2f6964796048a74aebe7a538fec1b092a7fc3b52 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:17:51 -0700 Subject: [PATCH 31/37] docs(lite): drop a datagram the transport has no room for Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index d1687c14e8..adbdbc10e3 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -725,7 +725,7 @@ A publisher sends a datagram to a subscription only when its Group Sequence is i A datagram is never cached or retransmitted, and FETCH never returns one. A publisher MAY send a new subscription the datagrams still in a short send buffer, so a subscriber cannot assume a datagram was sent after it subscribed. A publisher that knows a FETCH names a group delivered only as a datagram resets the stream with NOT_FETCHABLE; otherwise it answers as for a group that does not exist, with NOT_FOUND. -A publisher SHOULD only send a datagram if the congestion controller can transmit it immediately. +A publisher SHOULD NOT wait to send a datagram: one the transport has no room for is dropped. There is no separate subscription for datagram delivery: datagrams are routed to existing subscriptions via the Subscribe ID, and a subscriber receiving the same group via both a stream and a datagram MUST deduplicate by group sequence. Each datagram body has the following encoding (note: there is no message length prefix; the QUIC datagram boundary delimits the payload): @@ -1581,7 +1581,7 @@ The `Message Length` describes the payload size on the wire. - SUBSCRIBE_OK `Group` names where delivery starts when the subscription resolves, which need not be the first group sent. It is not a new floor. A subscriber does not wait for a group between the requested floor and it, but a publisher still delivers one that arrives later within Subscriber Max Age. - Corrected the SUBSCRIBE note that offset `Group Start` by 1. Only `Group End` and `Frame End` are offset so 0 can mean absent. `Group Start` is an absolute sequence, and 0 is group 0. - A subscription's range bounds datagrams like groups, and FETCH never returns a datagram. -- A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. +- A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. It SHOULD NOT wait to send a datagram and drops one the transport has no room for, where it previously sent one only if the congestion controller could transmit it immediately. - Assigned 0x3A NOT_FETCHABLE in the stream error table: a FETCH for a group delivered only as a datagram. - The subscriber FINs its Subscribe Stream after settling its tail; graceful session close waits for that FIN or reset. - A subscriber MAY cancel by FIN on its Subscribe Stream, and the publisher then FINs without SUBSCRIBE_END. From df8148bc7776ce9086243e11e5a8eaa1f4b9149b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:32:38 -0700 Subject: [PATCH 32/37] docs(lite): a datagram is dropped rather than blocked on Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-lite.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index adbdbc10e3..9481b69f4a 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -725,7 +725,7 @@ A publisher sends a datagram to a subscription only when its Group Sequence is i A datagram is never cached or retransmitted, and FETCH never returns one. A publisher MAY send a new subscription the datagrams still in a short send buffer, so a subscriber cannot assume a datagram was sent after it subscribed. A publisher that knows a FETCH names a group delivered only as a datagram resets the stream with NOT_FETCHABLE; otherwise it answers as for a group that does not exist, with NOT_FOUND. -A publisher SHOULD NOT wait to send a datagram: one the transport has no room for is dropped. +A publisher SHOULD drop a datagram rather than block when the transport has no room for it. There is no separate subscription for datagram delivery: datagrams are routed to existing subscriptions via the Subscribe ID, and a subscriber receiving the same group via both a stream and a datagram MUST deduplicate by group sequence. Each datagram body has the following encoding (note: there is no message length prefix; the QUIC datagram boundary delimits the payload): @@ -1581,7 +1581,7 @@ The `Message Length` describes the payload size on the wire. - SUBSCRIBE_OK `Group` names where delivery starts when the subscription resolves, which need not be the first group sent. It is not a new floor. A subscriber does not wait for a group between the requested floor and it, but a publisher still delivers one that arrives later within Subscriber Max Age. - Corrected the SUBSCRIBE note that offset `Group Start` by 1. Only `Group End` and `Frame End` are offset so 0 can mean absent. `Group Start` is an absolute sequence, and 0 is group 0. - A subscription's range bounds datagrams like groups, and FETCH never returns a datagram. -- A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. It SHOULD NOT wait to send a datagram and drops one the transport has no room for, where it previously sent one only if the congestion controller could transmit it immediately. +- A publisher SHOULD order a datagram among the subscription's streams as if it were its Group's stream, ahead of that stream when both exist. It SHOULD drop a datagram rather than block when the transport has no room for it, where it previously sent one only if the congestion controller could transmit it immediately. - Assigned 0x3A NOT_FETCHABLE in the stream error table: a FETCH for a group delivered only as a datagram. - The subscriber FINs its Subscribe Stream after settling its tail; graceful session close waits for that FIN or reset. - A subscriber MAY cancel by FIN on its Subscribe Stream, and the publisher then FINs without SUBSCRIBE_END. From 9ea98fb6166b0adee80abec121c1fc25688ecc1e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 18:05:49 -0700 Subject: [PATCH 33/37] fix(quic): the fuzz datagram op sends at an order Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/fuzz/connection.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-quic/src/fuzz/connection.rs b/rs/moq-quic/src/fuzz/connection.rs index 63cdd15a40..6c419ad0bc 100644 --- a/rs/moq-quic/src/fuzz/connection.rs +++ b/rs/moq-quic/src/fuzz/connection.rs @@ -138,7 +138,7 @@ enum Op { Datagram { who: Who, len: u16, - drop: bool, + order: i8, }, Ping { who: Who, @@ -554,7 +554,7 @@ impl Pair { conn.set_send_window(window.into()); } } - Op::Datagram { who, len, drop } => { + Op::Datagram { who, len, order } => { let peer = self.peer(who); let seq = peer.datagrams_sent.len() as u64; let Some(conn) = peer.conn() else { @@ -566,7 +566,7 @@ impl Pair { .chain((0..len).map(|i| datagram_byte(seq, i.into()))) .collect(); let size = data.len(); - if conn.datagrams().send(data, drop).is_ok() { + if conn.datagrams().send(data, None, order.into()).is_ok() { peer.datagrams_sent.push(size); } } @@ -820,7 +820,7 @@ pub(super) fn datagrams() -> Vec> { Op::Datagram { who: Who::Server, len: 10, - drop: false, + order: 0, }, Op::KeyUpdate { who: Who::Client }, Op::Ping { who: Who::Client }, From 06b61b06c662e0c167b9e9c4241682c7e4a80f63 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:38:49 -0700 Subject: [PATCH 34/37] fix(net): rank a lite datagram by the current group order A SUBSCRIBE_UPDATE refreshed the steer only when a group was served next, so datagrams before then kept the old order. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 49 +++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index ad4a1f7ed5..c9665cb4c4 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -3225,7 +3225,8 @@ impl Subscription { return false; } - let order = crate::priority::in_group(datagram.sequence, 0, self.steer_seen.order); + // The run loop may not have served a group since the last SUBSCRIBE_UPDATE. + let order = crate::priority::in_group(datagram.sequence, 0, self.steer_current().order); let _ = self.session.send_datagram(body, self.group.as_ref(), order); true } @@ -4627,6 +4628,52 @@ mod serve_group_test { assert!(order > 1); } + /// A SUBSCRIBE_UPDATE that flips the group order reranks the next datagram, even with no + /// group served in between. + #[moq_net_sim::test] + async fn datagrams_follow_an_updated_group_order() { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe(track::Subscription::default().with_max_delay(Duration::from_secs(3600))); + let mut session = SinkSession::new(Log::default()).with_groups().with_datagrams(1200); + let log = session.log.clone(); + let (mut run, mut writer, _) = lite_run(Version::Lite06, session.clone(), subscriber); + run.ctx.group = crate::transport::poll::Session::send_group(&mut session, 3); + + let first = track + .append_datagram(Timestamp::from_millis(1).unwrap(), b"a".as_slice()) + .unwrap(); + { + let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); + assert!(futures::poll!(poll.as_mut()).is_pending()); + } + + run.update(lite::SubscribeUpdate { + order: Order::Ascending, + priority: 3, + max_delay: Duration::from_secs(3600), + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, + }); + let second = track + .append_datagram(Timestamp::from_millis(2).unwrap(), b"b".as_slice()) + .unwrap(); + { + let mut poll = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); + assert!(futures::poll!(poll.as_mut()).is_pending()); + } + + let descending = crate::priority::in_group(first, 0, Order::Descending); + let ascending = crate::priority::in_group(second, 0, Order::Ascending); + assert_ne!(ascending, crate::priority::in_group(second, 0, Order::Descending)); + assert_eq!(log.datagram_ranks(), [(Some(0), descending), (Some(0), ascending)]); + } + /// A group with more frames ready than one poll's budget drains over several polls, /// so a sibling (the transport's driver, say) runs in between. #[test] From 33fc34e3505b016c09ed87f1fb0d93431f35aea8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:42:37 -0700 Subject: [PATCH 35/37] fix(quic): the wire bench sends ungrouped datagrams Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/benches/wire.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-quic/benches/wire.rs b/rs/moq-quic/benches/wire.rs index cf167d371b..a651516fe9 100644 --- a/rs/moq-quic/benches/wire.rs +++ b/rs/moq-quic/benches/wire.rs @@ -50,7 +50,7 @@ fn queue(pair: &mut Pair, packets: usize, frames: usize) -> usize { let mut datagrams = conn.datagrams(); let empty = datagrams.send_buffer_space(); for _ in 0..packets * frames { - datagrams.send(data.clone(), false).unwrap(); + datagrams.send(data.clone(), None, 0).unwrap(); } empty } From c798282f1abd5649168c1bdcd093afe30475e3ea Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 19:58:00 -0700 Subject: [PATCH 36/37] fix(quic): a packet holding control frames is not empty for a datagram write_frames judged emptiness by its own frames, so a top-ranked datagram that did not fit behind an ACK or other control frame stepped aside and let lower-ranked data take the packet. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 8 ++-- .../src/connection/streams/saturated.rs | 31 ++++++++++++++- rs/moq-quic/src/connection/streams/state.rs | 39 ++++++++++++------- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 7c68679e5e..c1869ecb9d 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -3667,9 +3667,11 @@ impl Connection { // STREAM and DATAGRAM, interleaved in schedule order if space_id == SpaceId::Data { - let written = self - .streams - .write_frames(buf, max_size, self.config.send_fairness); + let written = self.streams.write_frames( + buf, + pre_payload_len..max_size, + self.config.send_fairness, + ); sent.stream_frames = written.streams; sent.non_retransmits |= written.datagrams > 0; self.stats.frame_tx.stream += sent.stream_frames.len() as u64; diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs index 9619b6501d..0ae5297667 100644 --- a/rs/moq-quic/src/connection/streams/saturated.rs +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -29,6 +29,9 @@ const PACKET: usize = 1200; /// overdraw by a frame, at both the group and the domain tier. const SLACK: u64 = 2 * PACKET as u64; +/// A PING frame's one byte, standing in for the control frames ahead of the streams. +const PING: u8 = 0x01; + /// A sender with more queued than the path takes, drained a packet at a time. struct Sender { state: StreamsState, @@ -142,8 +145,15 @@ impl Sender { /// Send one packet of `size` bytes, returning its stream frames. fn packet_of(&mut self, size: usize) -> Vec { + self.packet_after(&[], size) + } + + /// Send one packet of `size` bytes that already holds the control frames `prefix`, + /// returning its stream frames. + fn packet_after(&mut self, prefix: &[u8], size: usize) -> Vec { let mut buf = Vec::with_capacity(size); - let written = self.state.write_frames(&mut buf, size, true); + buf.extend_from_slice(prefix); + let written = self.state.write_frames(&mut buf, 0..size, true); let frames = written.streams; // The packet's frames in order, and the wire bytes its datagrams took. @@ -162,10 +172,11 @@ impl Sender { datagram_bytes += wire; self.order.push(Sent::Datagram(tag)); } + Frame::Ping => {} frame => panic!("unexpected {frame:?}"), } } - let stream_bytes = buf.len() as u64 - datagram_bytes; + let stream_bytes = (buf.len() - prefix.len()) as u64 - datagram_bytes; // Every frame carries its length but perhaps the last, which can run to the // end of the packet instead. @@ -725,6 +736,22 @@ fn a_clamped_packet_sends_what_fits() { assert_eq!(sender.drain_frames(1)[0], Sent::Datagram(large)); } +/// A packet already holding a control frame is not empty, so a datagram that no longer +/// fits ends it rather than letting lower-ranked data ahead. +#[test] +fn a_datagram_behind_control_frames_ends_the_packet() { + let mut sender = Sender::new(); + let group = sender.group(0); + let large = sender.datagram(Some(&group), 1, 1_000); + sender.open(Some(&group), 0, 2_000); + + // The DATAGRAM frame (type, two-byte length, payload) would fill the packet alone, + // but a PING is already in it. + sender.packet_after(&[PING], 1 + 2 + 1_000); + assert_eq!(sender.order, []); + assert_eq!(sender.drain_frames(1)[0], Sent::Datagram(large)); +} + /// A released group keeps its queued datagrams and frees once they are sent. #[test] fn a_released_group_sends_its_datagrams() { diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 373b57b337..736d0b439a 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -2,6 +2,7 @@ use std::{ collections::{VecDeque, hash_map}, convert::TryFrom, mem, + ops::Range, }; use bytes::BufMut; @@ -670,6 +671,7 @@ impl StreamsState { /// Write STREAM and DATAGRAM frames in schedule order until the packet is full. /// + /// `payload` is the packet's frame space in `buf`, which may already hold control frames. /// A datagram cannot be split, so one too large for the room left ends the packet and /// leads the next one. Only when it would leave the packet empty, which a packet clamped /// below the path MTU can, does it step aside for what fits behind it, since @@ -677,9 +679,10 @@ impl StreamsState { pub(crate) fn write_frames( &mut self, buf: &mut Vec, - max_buf_size: usize, + payload: Range, fair: bool, ) -> WrittenFrames { + let max_buf_size = payload.end; let mut written = WrittenFrames::default(); while buf.len() + frame::Stream::SIZE_BOUND < max_buf_size { // Pop the next frame to send. A stream leaves the schedule once it has nothing @@ -690,7 +693,7 @@ impl StreamsState { Some(Next::Datagram(group)) => { let (_, frame) = self.pending.datagram(group); if buf.len() + frame > max_buf_size { - if written.datagrams > 0 || !written.streams.is_empty() { + if buf.len() > payload.start { break; } self.pending.defer_datagram(group); @@ -1707,7 +1710,7 @@ mod tests { high.write(b"high").unwrap(); let mut buf = Vec::with_capacity(40); - let meta = server.write_frames(&mut buf, 40, true).streams; + let meta = server.write_frames(&mut buf, 0..40, true).streams; assert_eq!(meta[0].id, id_high); assert_eq!(meta[1].id, id_mid); assert_eq!(meta[2].id, id_low); @@ -1745,7 +1748,7 @@ mod tests { assert_eq!(low.write(b"low").unwrap(), 3); let mut buf = Vec::with_capacity(40); - let mut sent = server.write_frames(&mut buf, 40, fair).streams; + let mut sent = server.write_frames(&mut buf, 0..40, fair).streams; assert_eq!(sent.len(), 1); let lost = sent.pop().unwrap(); assert_eq!(lost.id, id_low); @@ -1767,7 +1770,7 @@ mod tests { assert_eq!(high.write(b"high").unwrap(), 4); buf.clear(); - let sent = server.write_frames(&mut buf, 40, fair).streams; + let sent = server.write_frames(&mut buf, 0..40, fair).streams; assert_eq!(sent.len(), 2); assert_eq!(sent[0].id, id_high); assert_eq!(sent[0].offsets, 0..4); @@ -1828,7 +1831,7 @@ mod tests { high.set_priority(-1).unwrap(); let mut buf = Vec::with_capacity(1000); - let meta = server.write_frames(&mut buf, 40, true).streams; + let meta = server.write_frames(&mut buf, 0..40, true).streams; assert_eq!(meta.len(), 1); assert_eq!(meta[0].id, id_high); @@ -1836,7 +1839,7 @@ mod tests { assert_eq!(server.pending.len(), 2); // Send the remaining data. The initial mid priority one should go first now - let meta = server.write_frames(&mut buf, 1000, true).streams; + let meta = server.write_frames(&mut buf, 0..1000, true).streams; assert_eq!(meta.len(), 2); assert_eq!(meta[0].id, id_mid); assert_eq!(meta[1].id, id_high); @@ -1897,7 +1900,9 @@ mod tests { // loop until all the streams are written loop { let buf_len = buf.len(); - let meta = server.write_frames(&mut buf, buf_len + 40, fair).streams; + let meta = server + .write_frames(&mut buf, buf_len..buf_len + 40, fair) + .streams; if meta.is_empty() { break; } @@ -1968,7 +1973,9 @@ mod tests { // Write the first chunk of stream_a let buf_len = buf.len(); - let meta = server.write_frames(&mut buf, buf_len + 40, false).streams; + let meta = server + .write_frames(&mut buf, buf_len..buf_len + 40, false) + .streams; assert!(!meta.is_empty()); metas.extend(meta); @@ -1985,7 +1992,9 @@ mod tests { // loop until all the streams are written loop { let buf_len = buf.len(); - let meta = server.write_frames(&mut buf, buf_len + 40, false).streams; + let meta = server + .write_frames(&mut buf, buf_len..buf_len + 40, false) + .streams; if meta.is_empty() { break; } @@ -2908,7 +2917,7 @@ mod tests { }; stream.write(b"hello world").unwrap(); let mut buf = Vec::new(); - let sent = stream.state.write_frames(&mut buf, 100, true).streams; + let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; stream.reset_to(7u32.into(), 4).unwrap(); assert!(pending.reset_stream.is_empty()); assert_eq!(server.buffered_data, 4); @@ -2916,7 +2925,7 @@ mod tests { // The prefix is lost and resent, without what followed it server.retransmit(sent[0].clone()); buf.clear(); - let resent = server.write_frames(&mut buf, 100, true).streams; + let resent = server.write_frames(&mut buf, 0..100, true).streams; assert_eq!(resent[0].offsets, 0..4); assert_eq!(server.received_ack_of(resent[0].clone()), Some(4)); assert_eq!(server.buffered_data, 0); @@ -2981,7 +2990,7 @@ mod tests { }; stream.write(b"hello world").unwrap(); let mut buf = Vec::new(); - let sent = stream.state.write_frames(&mut buf, 100, true).streams; + let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; stream.reset_to(7u32.into(), 4).unwrap(); server.reset_acked(id, 4); @@ -3057,7 +3066,7 @@ mod tests { }; stream.write(b"hello world").unwrap(); let mut buf = Vec::new(); - let sent = stream.state.write_frames(&mut buf, 100, true).streams; + let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; assert_eq!(sent[0].offsets, 0..11); stream.reset_to(7u32.into(), 4).unwrap(); assert_eq!(server.buffered_data, 4); @@ -3065,7 +3074,7 @@ mod tests { // Only the lost prefix is retransmitted, without the FIN of a stream that was never finished server.retransmit(sent[0].clone()); buf.clear(); - let resent = server.write_frames(&mut buf, 100, true).streams; + let resent = server.write_frames(&mut buf, 0..100, true).streams; assert_eq!(resent.len(), 1); assert_eq!(resent[0].offsets, 0..4); assert!(!resent[0].fin); From 2ad974dc7e2c6720c4d171f324fdb3fcbe495a24 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 20:02:15 -0700 Subject: [PATCH 37/37] style(quic): drop a path qualification Range now covers Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/streams/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 736d0b439a..6c1f57c8d9 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -2662,7 +2662,7 @@ mod tests { } } - fn receive(state: &mut StreamsState, id: StreamId, data: std::ops::Range) { + fn receive(state: &mut StreamsState, id: StreamId, data: Range) { let len = (data.end - data.start) as usize; let _ = state .received(