From a62b26dc8ed4917362737e7d9ebc81145b77dac3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:26:58 -0700 Subject: [PATCH 01/51] quest: claim quest/m1/quic/fork/switch Co-Authored-By: Claude Opus 5.5 From 590d323102f2dd9b7630077175b05ae164aa9b6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:32:13 -0700 Subject: [PATCH 02/51] feat(moq-tokio): import quinn's async layer and web-transport-moq verbatim Unmodified copies, not yet compiled, so a reviewer can diff the adaptation that follows against upstream: - quinn-rs/quinn 7616e6b2782722f3ce4a1b181ef08817d7f545e4 quinn/src -> rs/moq-tokio/src/quic/driver (lib.rs -> mod.rs), the commit moq-quic forked from. - moq-dev/noq e27c0ce00741d81c8620b98e345d4f2c88e0768c web-transport-moq/src -> rs/moq-tokio/src/quic/web (lib.rs -> mod.rs). Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1467 +++++++++++++ rs/moq-tokio/src/quic/driver/endpoint.rs | 1020 +++++++++ rs/moq-tokio/src/quic/driver/incoming.rs | 152 ++ rs/moq-tokio/src/quic/driver/mod.rs | 134 ++ rs/moq-tokio/src/quic/driver/mutex.rs | 163 ++ rs/moq-tokio/src/quic/driver/recv_stream.rs | 823 ++++++++ rs/moq-tokio/src/quic/driver/runtime/mod.rs | 330 +++ rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 + rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 111 + rs/moq-tokio/src/quic/driver/send_stream.rs | 454 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 1871 +++++++++++++++++ rs/moq-tokio/src/quic/driver/work_limiter.rs | 224 ++ rs/moq-tokio/src/quic/web/client.rs | 378 ++++ rs/moq-tokio/src/quic/web/connect.rs | 158 ++ rs/moq-tokio/src/quic/web/crypto.rs | 67 + rs/moq-tokio/src/quic/web/error.rs | 405 ++++ rs/moq-tokio/src/quic/web/mod.rs | 71 + rs/moq-tokio/src/quic/web/recv.rs | 159 ++ rs/moq-tokio/src/quic/web/send.rs | 197 ++ rs/moq-tokio/src/quic/web/server.rs | 200 ++ rs/moq-tokio/src/quic/web/session.rs | 1331 ++++++++++++ rs/moq-tokio/src/quic/web/settings.rs | 70 + 22 files changed, 9890 insertions(+) create mode 100644 rs/moq-tokio/src/quic/driver/connection.rs create mode 100644 rs/moq-tokio/src/quic/driver/endpoint.rs create mode 100644 rs/moq-tokio/src/quic/driver/incoming.rs create mode 100644 rs/moq-tokio/src/quic/driver/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/mutex.rs create mode 100644 rs/moq-tokio/src/quic/driver/recv_stream.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/tokio.rs create mode 100644 rs/moq-tokio/src/quic/driver/send_stream.rs create mode 100755 rs/moq-tokio/src/quic/driver/tests.rs create mode 100644 rs/moq-tokio/src/quic/driver/work_limiter.rs create mode 100644 rs/moq-tokio/src/quic/web/client.rs create mode 100644 rs/moq-tokio/src/quic/web/connect.rs create mode 100644 rs/moq-tokio/src/quic/web/crypto.rs create mode 100644 rs/moq-tokio/src/quic/web/error.rs create mode 100644 rs/moq-tokio/src/quic/web/mod.rs create mode 100644 rs/moq-tokio/src/quic/web/recv.rs create mode 100644 rs/moq-tokio/src/quic/web/send.rs create mode 100644 rs/moq-tokio/src/quic/web/server.rs create mode 100644 rs/moq-tokio/src/quic/web/session.rs create mode 100644 rs/moq-tokio/src/quic/web/settings.rs diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs new file mode 100644 index 00000000000..6ce968db476 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -0,0 +1,1467 @@ +use std::{ + any::Any, + fmt, + future::Future, + io, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, Waker, ready}, +}; + +use bytes::Bytes; +use pin_project_lite::pin_project; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tokio::sync::{ + Notify, + futures::{Notified, OwnedNotified}, + mpsc, oneshot, watch, +}; +use tracing::{Instrument, Span, debug_span}; + +use crate::{ + ConnectionEvent, Duration, Instant, VarInt, + mutex::Mutex, + recv_stream::RecvStream, + runtime::{AsyncTimer, Runtime, UdpSender}, + send_stream::SendStream, + udp_transmit, +}; +use proto::{ + ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, + StreamId, TransportError, TransportErrorCode, congestion::Controller, +}; + +/// In-progress connection attempt future +/// +/// Resolves into a [`Connection`] once the TLS handshake completes. In TLS 1.3, peer +/// certificates arrive with handshake completion, so they are only available via +/// [`Connection::peer_identity()`]. [`Connection`] is a lightweight handle, so completing the +/// handshake imposes no extra cost; use [`Connection::close()`] to reject peers that fail +/// custom validation. +#[derive(Debug)] +pub struct Connecting { + conn: Option, + connected: Pin>, + handshake_data_ready: Option>, +} + +impl Connecting { + pub(crate) fn new( + handle: ConnectionHandle, + conn: proto::Connection, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + sender: Pin>, + runtime: Arc, + ) -> Self { + let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); + + let conn = ConnectionRef(Arc::new(ConnectionInner { + state: Mutex::new(State::new( + conn, + handle, + endpoint_events, + conn_events, + on_handshake_data_send, + sender, + runtime.clone(), + )), + shared: Shared::default(), + })); + let connected = Box::pin(conn.shared.connected.clone().notified_owned()); + + let driver = ConnectionDriver::new(conn.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {e}"); + } + } + .instrument(Span::current()), + )); + + Self { + conn: Some(conn), + connected, + handshake_data_ready: Some(on_handshake_data_recv), + } + } + + /// Convert into a 0-RTT or 0.5-RTT connection at the cost of weakened security + /// + /// Returns `Ok` immediately if the local endpoint is able to attempt sending 0/0.5-RTT data. + /// If so, the returned [`Connection`] can be used to send application data without waiting for + /// the rest of the handshake to complete, at the cost of weakened cryptographic security + /// guarantees. The [`Connection::authenticated`] future resolves when the handshake does + /// complete, at which point subsequently opened streams and written data will have full + /// cryptographic protection. + /// + /// ## Outgoing + /// + /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually + /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This + /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may + /// arise when the handshake completes. If 0-RTT was rejected, the existence of streams opened + /// and other application data sent by a client prior to the handshake completing will not be + /// conveyed to the remote application. Clients which need to e.g. re-transmit rejected stream + /// data must take care to detect `ZeroRttRejected` errors even on finished streams opened + /// during 0-RTT, e.g. by awaiting an application-layer response on bidirectional streams, or + /// awaiting [`SendStream::stopped`] on unidirectional streams. + /// + /// A server may reject 0-RTT data at its discretion, but accepting 0-RTT data requires the + /// relevant resumption state to be stored in the server, which servers may limit or lose for + /// various reasons including not persisting resumption state across server restarts. + /// + /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Incoming + /// + /// For incoming connections, conversion to 0.5-RTT will always succeed. + /// + /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Security + /// + /// On outgoing connections, this enables transmission of 0-RTT data, which is vulnerable to + /// replay attacks, and should therefore never invoke non-idempotent operations. + /// + /// On incoming connections, this enables transmission of 0.5-RTT data, which may be sent + /// before TLS client authentication has occurred, and should therefore not be used to send + /// data for which client authentication is being used. + pub fn into_0rtt(mut self) -> Result { + // This lock borrows `self` and would normally be dropped at the end of this scope, so we'll + // have to release it explicitly before returning `self` by value. + let conn = (self.conn.as_mut().unwrap()).state.lock("into_0rtt"); + + let is_ok = conn.inner.has_0rtt() || conn.inner.side().is_server(); + drop(conn); + + if is_ok { + let conn = self.conn.take().unwrap(); + Ok(Connection(conn)) + } else { + Err(self) + } + } + + /// Parameters negotiated during the handshake + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a + /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// + /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. + /// + /// This operation is cancel-safe. + pub async fn handshake_data(&mut self) -> Result, ConnectionError> { + // Taking &mut self allows us to use a single oneshot channel rather than dealing with + // potentially many tasks waiting on the same event. It's a bit of a hack, but keeps things + // simple. + if let Some(x) = self.handshake_data_ready.as_mut() { + let _ = x.await; + // Once the data is ready, apply state changes. This prevents a panic due to + // inconsistent state when retrying a call to `handshake_data`. + self.handshake_data_ready = None; + } + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("handshake"); + inner + .inner + .crypto_session() + .handshake_data() + .ok_or_else(|| { + inner + .error + .clone() + .expect("spurious handshake data ready notification") + }) + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn local_ip(&self) -> Option { + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("local_ip"); + + inner.inner.local_ip() + } + + /// The peer's UDP address + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn remote_address(&self) -> SocketAddr { + let conn_ref: &ConnectionRef = self.conn.as_ref().expect("used after yielding Ready"); + conn_ref.state.lock("remote_address").inner.remote_address() + } +} + +impl Future for Connecting { + type Output = Result; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.connected).poll(cx).map(|_| { + let conn = self.conn.take().unwrap(); + let inner = conn.state.lock("connecting"); + if inner.connected { + drop(inner); + Ok(Connection(conn)) + } else { + Err(inner + .error + .clone() + .expect("connected signaled without connection success or error")) + } + }) + } +} + +/// A future that drives protocol logic for a connection +/// +/// This future handles the protocol logic for a single connection, routing events from the +/// `Connection` API object to the `Endpoint` task and the related stream-related interfaces. +/// It also keeps track of outstanding timeouts for the `Connection`. +/// +/// If the connection encounters an error condition, this future will yield an error. It will +/// terminate (yielding `Ok(())`) if the connection was closed without error. Unlike other +/// connection-related futures, this waits for the draining period to complete to ensure that +/// packets still in flight from the peer are handled gracefully. +#[must_use = "connection drivers must be spawned for their connections to function"] +#[derive(Debug)] +struct ConnectionDriver { + conn: ConnectionRef, + span: Span, +} + +impl ConnectionDriver { + fn new(conn: ConnectionRef) -> Self { + let span = { + let conn = &mut conn.state.lock("poll"); + debug_span!("drive", id = conn.handle.0) + }; + + Self { conn, span } + } +} + +impl Future for ConnectionDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let conn = &mut *self.conn.state.lock("poll"); + let _guard = self.span.enter(); + + if let Err(e) = conn.process_conn_events(&self.conn.shared, cx) { + conn.terminate(e, &self.conn.shared); + return Poll::Ready(Ok(())); + } + let mut keep_going = conn.drive_transmit(cx).inspect_err(|_| { + // Transmit failed, so close the connection and clean state before returning the error. + if !conn.inner.is_closed() { + conn.implicit_close(&self.conn.shared); + } + })?; + // If a timer expires, there might be more to transmit. When we transmit something, we + // might need to reset a timer. Hence, we must loop until neither happens. + keep_going |= conn.drive_timer(cx); + conn.forward_endpoint_events(); + conn.forward_app_events(&self.conn.shared); + + if !conn.inner.is_drained() { + if keep_going { + // If the connection hasn't processed all tasks, schedule it again + cx.waker().wake_by_ref(); + } else { + match conn.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => conn.driver = Some(cx.waker().clone()), + }; + } + return Poll::Pending; + } + if conn.error.is_none() { + unreachable!("drained connections always have an error"); + } + Poll::Ready(Ok(())) + } +} + +/// A QUIC connection. +/// +/// If all references to a connection (including every clone of the `Connection` handle, streams of +/// incoming streams, and the various stream types) have been dropped, then the connection will be +/// automatically closed with an `error_code` of 0 and an empty `reason`. You can also close the +/// connection explicitly by calling [`Connection::close()`]. +/// +/// Closing the connection immediately abandons efforts to deliver data to the peer. Upon +/// receiving CONNECTION_CLOSE the peer *may* drop any stream data not yet delivered to the +/// application. [`Connection::close()`] describes in more detail how to gracefully close a +/// connection without losing application data. +/// +/// May be cloned to obtain another handle to the same connection. +/// +/// [`Connection::close()`]: Connection::close +#[derive(Debug, Clone)] +pub struct Connection(ConnectionRef); + +impl Connection { + /// Initiate a new outgoing unidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. + pub fn open_uni(&self) -> OpenUni<'_> { + OpenUni { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Uni as usize].notified(), + } + } + + /// Initiate a new outgoing bidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing + /// anything to [`SendStream`] will never succeed. + /// + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + pub fn open_bi(&self) -> OpenBi<'_> { + OpenBi { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Bi as usize].notified(), + } + } + + /// Accept the next incoming uni-directional stream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_uni(&self) -> AcceptUni<'_> { + AcceptUni { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Uni as usize].notified(), + } + } + + /// Accept the next incoming bidirectional stream + /// + /// **Important Note**: The `Connection` that calls [`open_bi()`] must write to its [`SendStream`] + /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then + /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. + /// + /// [`accept_bi()`]: crate::Connection::accept_bi + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_bi(&self) -> AcceptBi<'_> { + AcceptBi { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Bi as usize].notified(), + } + } + + /// Receive an application datagram + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no datagrams were consumed. + pub fn read_datagram(&self) -> ReadDatagram<'_> { + ReadDatagram { + conn: &self.0, + notify: self.0.shared.datagram_received.notified(), + } + } + + /// Wait for the connection to be closed for any reason + /// + /// Despite the return type's name, closed connections are often not an error condition at the + /// application layer. Cases that might be routine include [`ConnectionError::LocallyClosed`] + /// and [`ConnectionError::ApplicationClosed`]. + pub async fn closed(&self) -> ConnectionError { + { + let conn = self.0.state.lock("closed"); + if let Some(error) = conn.error.as_ref() { + return error.clone(); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.closed.notified() + } + .await; + self.0 + .state + .lock("closed") + .error + .as_ref() + .expect("closed without an error") + .clone() + } + + /// Whether the connection is closed, and why. + /// + /// The close_reason is always set to `Some(ConnectionError)` when a socket is + /// closed; whether it was closed manually by calling [`Connection::close()`] or due to + /// an internal error (such as an idle timeout or the peer closing the + /// connection). + /// + /// Note: when the connection is closed, `connection.close_reason().is_some()` will always be true. + pub fn close_reason(&self) -> Option { + self.0.state.lock("close_reason").error.clone() + } + + /// Close the connection immediately. + /// + /// Pending operations will fail immediately with [`ConnectionError::LocallyClosed`]. No + /// more data is sent to the peer and the peer may drop buffered data upon receiving + /// the CONNECTION_CLOSE frame. + /// + /// `error_code` and `reason` are not interpreted, and are provided directly to the peer. + /// + /// `reason` will be truncated to fit in a single packet with overhead; to improve odds that it + /// is preserved in full, it should be kept under 1KiB. + /// + /// # Gracefully closing a connection + /// + /// Only the peer last receiving application data can be certain that all data is + /// delivered. The only reliable action it can then take is to close the connection, + /// potentially with a custom error code. The delivery of the final CONNECTION_CLOSE + /// frame is very likely if both endpoints stay online long enough, and + /// [`Endpoint::wait_idle()`] can be used to provide sufficient time. Otherwise, the + /// remote peer will time out the connection, provided that the idle timeout is not + /// disabled. + /// + /// The sending side can not guarantee all stream data is delivered to the remote + /// application. It only knows the data is delivered to the QUIC stack of the remote + /// endpoint. Once the local side sends a CONNECTION_CLOSE frame in response to calling + /// [`close()`] the remote endpoint may drop any data it received but is as yet + /// undelivered to the application, including data that was acknowledged as received to + /// the local endpoint. + /// + /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`close()`]: Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let conn = &mut *self.0.state.lock("close"); + conn.close(error_code, Bytes::copy_from_slice(reason), &self.0.shared); + } + + /// Wait for the handshake to be confirmed. + /// + /// As a server, who must be authenticated by clients, + /// this happens when the handshake completes + /// upon receiving a TLS Finished message from the client. + /// In return, the server send a HANDSHAKE_DONE frame. + /// + /// As a client, this happens when receiving a HANDSHAKE_DONE frame. + /// At this point, the server has either accepted our authentication, + /// or, if client authentication is not required, accepted our lack of authentication. + pub async fn handshake_confirmed(&self) -> Result<(), ConnectionError> { + { + let conn = self.0.state.lock("handshake_confirmed"); + if let Some(error) = conn.error.as_ref() { + return Err(error.clone()); + } + if conn.handshake_confirmed { + return Ok(()); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.handshake_confirmed.notified() + } + .await; + if let Some(error) = self.0.state.lock("handshake_confirmed").error.as_ref() { + Err(error.clone()) + } else { + Ok(()) + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Application datagrams are a low-level primitive. They may be lost or delivered out of order, + /// and `data` must both fit inside a single QUIC packet and be smaller than the maximum + /// dictated by the peer. + /// + /// Previously queued datagrams which are still unsent may be discarded to make space for this + /// datagram, in order of oldest to newest. + pub fn send_datagram(&self, data: Bytes) -> Result<(), SendDatagramError> { + let conn = &mut *self.0.state.lock("send_datagram"); + if let Some(x) = &conn.error { + return Err(SendDatagramError::ConnectionLost(x.clone())); + } + use proto::SendDatagramError::*; + match conn.inner.datagrams().send(data, true) { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(e) => Err(match e { + Blocked(..) => unreachable!(), + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + }), + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Unlike [`send_datagram()`], this method will wait for buffer space during congestion + /// conditions, which effectively prioritizes old datagrams over new datagrams. + /// + /// See [`send_datagram()`] for details. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn send_datagram_wait(&self, data: Bytes) -> SendDatagram<'_> { + SendDatagram { + conn: &self.0, + data: Some(data), + notify: self.0.shared.datagrams_unblocked.notified(), + } + } + + /// Compute the maximum size of datagrams that may be passed to [`send_datagram()`]. + /// + /// Returns `None` if datagrams are unsupported by the peer or disabled locally. + /// + /// This may change over the lifetime of a connection according to variation in the path MTU + /// estimate. The peer can also enforce an arbitrarily small fixed limit, but if the peer's + /// limit is large this is guaranteed to be a little over a kilobyte at minimum. + /// + /// Not necessarily the maximum size of received datagrams. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn max_datagram_size(&self) -> Option { + self.0 + .state + .lock("max_datagram_size") + .inner + .datagrams() + .max_size() + } + + /// Bytes available in the outgoing datagram buffer + /// + /// When greater than zero, calling [`send_datagram()`](Self::send_datagram) with a datagram of + /// at most this size is guaranteed not to cause older datagrams to be dropped. + pub fn datagram_send_buffer_space(&self) -> usize { + self.0 + .state + .lock("datagram_send_buffer_space") + .inner + .datagrams() + .send_buffer_space() + } + + /// The side of the connection (client or server) + pub fn side(&self) -> Side { + self.0.state.lock("side").inner.side() + } + + /// The peer's UDP address + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn remote_address(&self) -> SocketAddr { + self.0.state.lock("remote_address").inner.remote_address() + } + + /// Watch the peer's UDP address. + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn watch_remote_address(&self) -> RemoteAddressWatcher { + let receiver = self + .0 + .state + .lock("watch_remote_address") + .remote_address + .subscribe(); + + RemoteAddressWatcher { receiver } + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + pub fn local_ip(&self) -> Option { + self.0.state.lock("local_ip").inner.local_ip() + } + + /// Current best estimate of this connection's latency (round-trip-time) + pub fn rtt(&self) -> Duration { + self.0.state.lock("rtt").inner.rtt() + } + + /// Minimum RTT seen on this path, ignoring ack delay + pub fn min_rtt(&self) -> Duration { + self.0.state.lock("min_rtt").inner.min_rtt() + } + + /// Returns connection statistics + pub fn stats(&self) -> ConnectionStats { + self.0.state.lock("stats").inner.stats() + } + + /// Current state of the congestion control algorithm, for debugging purposes + pub fn congestion_state(&self) -> Box { + self.0 + .state + .lock("congestion_state") + .inner + .congestion_state() + .clone_box() + } + + /// Succeeds when an incoming connection is proven not to be a replay attack. + /// + /// Only interesting for `Connection`s obtained from [`Connecting::into_0rtt`]. On 1-RTT + /// connections, always completes immediately. Contrast + /// [`handshake_confirmed`](Self::handshake_confirmed), which waits longer on clients e.g. to + /// confirm client authentication. + /// + /// For incoming connections, reads from [`RecvStream`]s are guaranteed not to arise from replay + /// attacks after this succeeds, even for streams accepted or read during 0-RTT. For outgoing + /// connections, streams opened after this succeeds will never be discarded by the server due to + /// 0-RTT rejection. + pub async fn authenticated(&self) -> Result<(), ConnectionError> { + let notified = { + let conn = self.0.state.lock("connected"); + if let Some(e) = &conn.error { + return Err(e.clone()); + } + if conn.connected { + return Ok(()); + } + self.0.shared.connected.notified() + }; + notified.await; + let conn = self.0.state.lock("connected"); + conn.error.clone().map_or(Ok(()), Err) + } + + /// Parameters negotiated during the handshake + /// + /// Guaranteed to return `Some` on fully established connections or after + /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on + /// the returned value. + /// + /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + pub fn handshake_data(&self) -> Option> { + self.0 + .state + .lock("handshake_data") + .inner + .crypto_session() + .handshake_data() + } + + /// Cryptographic identity of the peer + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. + /// + /// For custom asynchronous validation (e.g. checking a certificate against an external + /// database), call this after awaiting [`Connecting`] and use [`close()`][Self::close] to + /// reject untrusted peers. + pub fn peer_identity(&self) -> Option> { + self.0 + .state + .lock("peer_identity") + .inner + .crypto_session() + .peer_identity() + } + + /// A stable identifier for this connection + /// + /// Peer addresses and connection IDs can change, but this value will remain + /// fixed for the lifetime of the connection. + pub fn stable_id(&self) -> usize { + self.0.stable_id() + } + + /// Update traffic keys spontaneously + /// + /// This primarily exists for testing purposes. + pub fn force_key_update(&self) { + self.0 + .state + .lock("force_key_update") + .inner + .force_key_update() + } + + /// Derive keying material from this connection's TLS session secrets. + /// + /// When both peers call this method with the same `label` and `context` + /// arguments and `output` buffers of equal length, they will get the + /// same sequence of bytes in `output`. These bytes are cryptographically + /// strong and pseudorandom, and are suitable for use as keying material. + /// + /// See [RFC5705](https://tools.ietf.org/html/rfc5705) for more information. + pub fn export_keying_material( + &self, + output: &mut [u8], + label: &[u8], + context: &[u8], + ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + self.0 + .state + .lock("export_keying_material") + .inner + .crypto_session() + .export_keying_material(output, label, context) + } + + /// Modify the number of remotely initiated unidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_uni_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_uni_streams"); + conn.inner.set_max_concurrent_streams(Dir::Uni, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } + + /// See [`proto::TransportConfig::send_window()`] + pub fn set_send_window(&self, send_window: u64) { + let mut conn = self.0.state.lock("set_send_window"); + conn.inner.set_send_window(send_window); + conn.wake(); + } + + /// See [`proto::TransportConfig::receive_window()`] + pub fn set_receive_window(&self, receive_window: VarInt) { + let mut conn = self.0.state.lock("set_receive_window"); + conn.inner.set_receive_window(receive_window); + conn.wake(); + } + + /// Modify the number of remotely initiated bidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_bi_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_bi_streams"); + conn.inner.set_max_concurrent_streams(Dir::Bi, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } +} + +pin_project! { + /// Future produced by [`Connection::open_uni`] + pub struct OpenUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenUni<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(SendStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::open_bi`] + pub struct OpenBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Bi))?; + + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_open<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_open"); + if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } else if let Some(id) = state.inner.streams().open(dir) { + let is_0rtt = state.inner.side().is_client() && state.inner.is_handshaking(); + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => { + notify.set(conn.shared.stream_budget_available[dir as usize].notified()) + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::accept_uni`] + pub struct AcceptUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptUni<'_> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(RecvStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::accept_bi`] + pub struct AcceptBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Bi))?; + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_accept<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_accept"); + // Check for incoming streams before checking `state.error` so that already-received streams, + // which are necessarily finite, can be drained from a closed connection. + if let Some(id) = state.inner.streams().accept(dir) { + let is_0rtt = state.inner.is_handshaking(); + state.wake(); // To send additional stream ID credit + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => notify.set(conn.shared.stream_incoming[dir as usize].notified()), + } + } +} + +pin_project! { + /// Future produced by [`Connection::read_datagram`] + pub struct ReadDatagram<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for ReadDatagram<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("ReadDatagram::poll"); + // Check for buffered datagrams before checking `state.error` so that already-received + // datagrams, which are necessarily finite, can be drained from a closed connection. + if let Some(x) = state.inner.datagrams().recv() { + return Poll::Ready(Ok(x)); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagram_received.notified()), + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::send_datagram_wait`] + pub struct SendDatagram<'a> { + conn: &'a ConnectionRef, + data: Option, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for SendDatagram<'_> { + type Output = Result<(), SendDatagramError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("SendDatagram::poll"); + if let Some(e) = &state.error { + return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); + } + use proto::SendDatagramError::*; + match state + .inner + .datagrams() + .send(this.data.take().unwrap(), false) + { + Ok(()) => { + state.wake(); + Poll::Ready(Ok(())) + } + Err(e) => Poll::Ready(Err(match e { + Blocked(data) => { + this.data.replace(data); + loop { + match this.notify.as_mut().poll(ctx) { + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagrams_unblocked.notified()), + } + } + } + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + })), + } + } +} + +/// Watcher produced by [`Connection::watch_remote_address`] +pub struct RemoteAddressWatcher { + receiver: watch::Receiver, +} + +impl RemoteAddressWatcher { + /// Wait for an address change, mark it as seen and return the address. + /// + /// If multiple migrations occur between calls to `wait` or polls of the resulting future, + /// the last observed address will be returned, skipping intermediate updates. + pub async fn wait(&mut self) -> Option { + if self.receiver.changed().await.is_err() { + return None; + } + + Some(*self.receiver.borrow()) + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionRef(Arc); + +impl ConnectionRef { + fn stable_id(&self) -> usize { + &*self.0 as *const _ as usize + } +} + +impl Clone for ConnectionRef { + fn clone(&self) -> Self { + self.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for ConnectionRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let conn = &mut *self.state.lock("drop"); + + if !conn.inner.is_closed() { + // If the driver is alive, it's just it and us, so we'd better shut it down. If it's + // not, we can't do any harm. If there were any streams being opened, then either + // the connection will be closed for an unrelated reason or a fresh reference will + // be constructed for the newly opened stream. + conn.implicit_close(&self.shared); + } + } +} + +impl std::ops::Deref for ConnectionRef { + type Target = ConnectionInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +#[derive(Debug, Default)] +pub(crate) struct Shared { + handshake_confirmed: Notify, + /// Notified when new streams may be locally initiated due to an increase in stream ID flow + /// control budget + stream_budget_available: [Notify; 2], + /// Notified when the peer has initiated a new stream + stream_incoming: [Notify; 2], + datagram_received: Notify, + datagrams_unblocked: Notify, + closed: Notify, + connected: Arc, + /// Number of live handles that can used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +pub(crate) struct State { + pub(crate) inner: proto::Connection, + driver: Option, + handle: ConnectionHandle, + on_handshake_data: Option>, + connected: bool, + handshake_confirmed: bool, + timer: Option>>, + timer_deadline: Option, + conn_events: mpsc::UnboundedReceiver, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + pub(crate) blocked_writers: FxHashMap, + pub(crate) blocked_readers: FxHashMap, + pub(crate) stopped: FxHashMap>, + /// Always set to Some before the connection becomes drained + pub(crate) error: Option, + sender: Pin>, + runtime: Arc, + send_buffer: Vec, + /// We buffer a transmit when the underlying I/O would block + buffered_transmit: Option, + remote_address: watch::Sender, +} + +impl State { + #[allow(clippy::too_many_arguments)] + fn new( + inner: proto::Connection, + handle: ConnectionHandle, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + on_handshake_data: oneshot::Sender<()>, + sender: Pin>, + runtime: Arc, + ) -> Self { + let remote_address = inner.remote_address(); + + Self { + inner, + driver: None, + handle, + on_handshake_data: Some(on_handshake_data), + connected: false, + handshake_confirmed: false, + timer: None, + timer_deadline: None, + conn_events, + endpoint_events, + blocked_writers: FxHashMap::default(), + blocked_readers: FxHashMap::default(), + stopped: FxHashMap::default(), + error: None, + sender, + runtime, + send_buffer: Vec::new(), + buffered_transmit: None, + remote_address: watch::Sender::new(remote_address), + } + } + + fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { + let now = self.runtime.now(); + let mut transmits = 0; + + let max_datagrams = self + .sender + .max_transmit_segments() + .min(MAX_TRANSMIT_SEGMENTS); + + loop { + // Retry the last transmit, or get a new one. + let t = match self.buffered_transmit.take() { + Some(t) => t, + None => { + self.send_buffer.clear(); + self.send_buffer.reserve(self.inner.current_mtu() as usize); + match self + .inner + .poll_transmit(now, max_datagrams, &mut self.send_buffer) + { + Some(t) => { + transmits += match t.segment_size { + None => 1, + Some(s) => t.size.div_ceil(s), // round up + }; + t + } + None => break, + } + } + }; + + let len = t.size; + match self + .sender + .as_mut() + .poll_send(&udp_transmit(&t, &self.send_buffer[..len]), cx) + { + Poll::Pending => { + self.buffered_transmit = Some(t); + return Ok(false); + } + Poll::Ready(Err(e)) => return Err(e), + Poll::Ready(Ok(())) => {} + } + + if transmits >= MAX_TRANSMIT_DATAGRAMS { + // TODO: What isn't ideal here yet is that if we don't poll all + // datagrams that could be sent we don't go into the `app_limited` + // state and CWND continues to grow until we get here the next time. + // See https://github.com/quinn-rs/quinn/issues/1126 + return Ok(true); + } + } + + Ok(false) + } + + fn forward_endpoint_events(&mut self) { + while let Some(event) = self.inner.poll_endpoint_events() { + // If the endpoint driver is gone, noop. + let _ = self.endpoint_events.send((self.handle, event)); + } + } + + /// If this returns `Err`, the endpoint is dead, so the driver should exit immediately. + fn process_conn_events( + &mut self, + shared: &Shared, + cx: &mut Context<'_>, + ) -> Result<(), ConnectionError> { + loop { + match self.conn_events.poll_recv(cx) { + Poll::Ready(Some(ConnectionEvent::Rebind(sender))) => { + self.sender = sender; + self.inner.local_address_changed(); + } + Poll::Ready(Some(ConnectionEvent::Proto(event))) => { + self.inner.handle_event(event); + } + Poll::Ready(Some(ConnectionEvent::Close { reason, error_code })) => { + self.close(error_code, reason, shared); + } + Poll::Ready(None) => { + return Err(ConnectionError::TransportError(TransportError::new( + TransportErrorCode::INTERNAL_ERROR, + "endpoint driver future was dropped".to_string(), + ))); + } + Poll::Pending => { + return Ok(()); + } + } + } + } + + fn forward_app_events(&mut self, shared: &Shared) { + while let Some(event) = self.inner.poll() { + use proto::Event::*; + match event { + HandshakeDataReady => { + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + } + Connected => { + self.connected = true; + shared.connected.notify_waiters(); + if self.inner.side().is_client() && !self.inner.accepted_0rtt() { + // Wake up rejected 0-RTT streams so they can fail immediately with + // `ZeroRttRejected` errors. + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + wake_all_notify(&mut self.stopped); + } + } + HandshakeConfirmed => { + self.handshake_confirmed = true; + shared.handshake_confirmed.notify_waiters(); + } + ConnectionLost { reason } => { + self.terminate(reason, shared); + } + Stream(StreamEvent::Writable { id }) => wake_stream(id, &mut self.blocked_writers), + Stream(StreamEvent::Opened { dir: Dir::Uni }) => { + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + } + Stream(StreamEvent::Opened { dir: Dir::Bi }) => { + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + } + DatagramReceived => { + shared.datagram_received.notify_waiters(); + } + DatagramsUnblocked => { + shared.datagrams_unblocked.notify_waiters(); + } + Stream(StreamEvent::Readable { id }) => wake_stream(id, &mut self.blocked_readers), + Stream(StreamEvent::Available { dir }) => { + // Might mean any number of streams are ready, so we wake up everyone + shared.stream_budget_available[dir as usize].notify_waiters(); + } + Stream(StreamEvent::Finished { id }) => wake_stream_notify(id, &mut self.stopped), + Stream(StreamEvent::Stopped { id, .. }) => { + wake_stream_notify(id, &mut self.stopped); + wake_stream(id, &mut self.blocked_writers); + } + PathUpdated => { + // Using `send_replace` since `send` does not update the value + // when there are no subscribed receivers. + let _ = self + .remote_address + .send_replace(self.inner.remote_address()); + } + _ => {} + } + } + } + + fn drive_timer(&mut self, cx: &mut Context<'_>) -> bool { + let Some(deadline) = self.inner.poll_timeout() else { + self.timer_deadline = None; + return false; + }; + + // Use the clock rather than the async timer to detect expiry: Sleep::poll + // respects Tokio's cooperative budget and can return Pending for elapsed + // deadlines. + let now = self.runtime.now(); + if now >= deadline { + self.inner.handle_timeout(now); + self.timer_deadline = None; + return true; + } + + match &mut self.timer { + // Avoid resetting the timer when the deadline is unchanged. + Some(delay) if self.timer_deadline != Some(deadline) => { + delay.as_mut().reset(deadline); + } + None => { + self.timer = Some(self.runtime.new_timer(deadline)); + } + _ => {} + } + self.timer_deadline = Some(deadline); + + let delay = self + .timer + .as_mut() + .expect("timer must exist in this state") + .as_mut(); + if delay.poll(cx).is_pending() { + return false; + } + + // The deadline elapsed in the window between the clock check and poll. + self.inner.handle_timeout(self.runtime.now()); + self.timer_deadline = None; + true + } + + /// Wake up a blocked `Driver` task to process I/O + pub(crate) fn wake(&mut self) { + if let Some(x) = self.driver.take() { + x.wake(); + } + } + + /// Used to wake up all blocked futures when the connection becomes closed for any reason + fn terminate(&mut self, reason: ConnectionError, shared: &Shared) { + self.error = Some(reason.clone()); + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + shared.stream_budget_available[Dir::Uni as usize].notify_waiters(); + shared.stream_budget_available[Dir::Bi as usize].notify_waiters(); + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + shared.datagram_received.notify_waiters(); + shared.datagrams_unblocked.notify_waiters(); + shared.handshake_confirmed.notify_waiters(); + wake_all_notify(&mut self.stopped); + shared.closed.notify_waiters(); + shared.connected.notify_waiters(); + } + + fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { + self.inner.close(self.runtime.now(), error_code, reason); + self.terminate(ConnectionError::LocallyClosed, shared); + self.wake(); + } + + /// Close for a reason other than the application's explicit request + pub(crate) fn implicit_close(&mut self, shared: &Shared) { + self.close(0u32.into(), Bytes::new(), shared); + } + + pub(crate) fn check_0rtt(&self) -> Result<(), ()> { + if self.inner.is_handshaking() + || self.inner.accepted_0rtt() + || self.inner.side().is_server() + { + Ok(()) + } else { + Err(()) + } + } +} + +impl Drop for State { + fn drop(&mut self) { + if !self.inner.is_drained() { + // Ensure the endpoint can tidy up + let _ = self + .endpoint_events + .send((self.handle, EndpointEvent::drained())); + } + } +} + +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("State").field("inner", &self.inner).finish() + } +} + +fn wake_stream(stream_id: StreamId, wakers: &mut FxHashMap) { + if let Some(waker) = wakers.remove(&stream_id) { + waker.wake(); + } +} + +fn wake_all(wakers: &mut FxHashMap) { + wakers.drain().for_each(|(_, waker)| waker.wake()) +} + +fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { + if let Some(notify) = wakers.remove(&stream_id) { + notify.notify_waiters() + } +} + +fn wake_all_notify(wakers: &mut FxHashMap>) { + wakers + .drain() + .for_each(|(_, notify)| notify.notify_waiters()) +} + +/// Errors that can arise when sending a datagram +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub enum SendDatagramError { + /// The peer does not support receiving datagram frames + #[error("datagrams not supported by peer")] + UnsupportedByPeer, + /// Datagram support is disabled locally + #[error("datagram support disabled")] + Disabled, + /// The datagram is larger than the connection can currently accommodate + /// + /// Indicates that the path MTU minus overhead or the limit advertised by the peer has been + /// exceeded. + #[error("datagram too large")] + TooLarge, + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), +} + +/// The maximum amount of datagrams which will be produced in a single `drive_transmit` call +/// +/// This limits the amount of CPU resources consumed by datagram generation, +/// and allows other tasks (like receiving ACKs) to run in between. +const MAX_TRANSMIT_DATAGRAMS: usize = 20; + +/// The maximum amount of datagrams that are sent in a single transmit +/// +/// This can be lower than the maximum platform capabilities, to avoid excessive +/// memory allocations when calling `poll_transmit()`. Benchmarks have shown +/// that numbers around 10 are a good compromise. +const MAX_TRANSMIT_SEGMENTS: usize = 10; diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs new file mode 100644 index 00000000000..2012e415ff6 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -0,0 +1,1020 @@ +use std::{ + collections::VecDeque, + fmt, + future::Future, + io::{self, IoSliceMut}, + mem, + net::{SocketAddr, SocketAddrV6}, + pin::Pin, + str, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use crate::runtime::default_runtime; +use crate::{ + Instant, + runtime::{AsyncUdpSocket, Runtime, UdpSender}, + udp_transmit, +}; +use bytes::{Bytes, BytesMut}; +use pin_project_lite::pin_project; +use proto::{ + self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, + EndpointEvent, ServerConfig, +}; +use rustc_hash::FxHashMap; +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use socket2::{Domain, Protocol, Socket, Type}; +use tokio::sync::{Notify, futures::Notified, mpsc}; +use tracing::{Instrument, Span}; +use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; + +use crate::{ + ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, + connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, +}; + +/// A QUIC endpoint. +/// +/// An endpoint corresponds to a single UDP socket, may host many connections, and may act as both +/// client and server for different connections. +/// +/// May be cloned to obtain another handle to the same endpoint. +#[derive(Debug, Clone)] +pub struct Endpoint { + pub(crate) inner: EndpointRef, + runtime: Arc, +} + +impl Endpoint { + /// Helper to construct an endpoint for use with outgoing connections only + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 addresses. As such, calling `Endpoint::client` with + /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other + /// address. For example: + /// + /// ``` + /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ``` + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn client(addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + None, + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Returns relevant stats from this Endpoint + pub fn stats(&self) -> EndpointStats { + self.inner.state.lock().unwrap().stats + } + + /// Helper to construct an endpoint for use with both incoming and outgoing connections + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 clients. As such, calling `Endpoint::server` with + /// the address `[::]:0` is a reasonable default to maximize the ability to accept connections + /// from any address. + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never + /// dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + Some(config), + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Construct an endpoint with arbitrary configuration and socket + #[cfg(not(wasm_browser))] + pub fn new( + config: EndpointConfig, + server_config: Option, + socket: std::net::UdpSocket, + runtime: Arc, + ) -> io::Result { + let socket = runtime.wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket, runtime) + } + + /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket + /// + /// Useful when `socket` has additional state (e.g. sidechannels) attached for which shared + /// ownership is needed. + pub fn new_with_abstract_socket( + config: EndpointConfig, + server_config: Option, + socket: Box, + runtime: Arc, + ) -> io::Result { + let addr = socket.local_addr()?; + let allow_mtud = !socket.may_fragment(); + let rc = EndpointRef::new( + socket, + proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + addr.is_ipv6(), + runtime.clone(), + ); + let driver = EndpointDriver(rc.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {}", e); + } + } + .instrument(Span::current()), + )); + Ok(Self { inner: rc, runtime }) + } + + /// Get the next incoming connection attempt from a client + /// + /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] + /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// filter connection attempts or force address validation, or converted into an intermediate + /// `Connecting` future which can be used to e.g. send 0.5-RTT data. + pub fn accept(&self) -> Accept<'_> { + Accept { + endpoint: self, + notify: self.inner.shared.incoming.notified(), + } + } + + /// Set the client configuration used by `connect` + pub fn set_default_client_config(&self, config: ClientConfig) { + self.inner.0.state.lock().unwrap().default_client_config = Some(config); + } + + /// Connect to a remote endpoint + /// + /// `server_name` must be covered by the certificate presented by the server. This prevents a + /// connection from being intercepted by an attacker with a valid certificate for some other + /// server. + /// + /// May fail immediately due to configuration errors, or in the future if the connection could + /// not be established. + pub fn connect(&self, addr: SocketAddr, server_name: &str) -> Result { + let Some(config) = self + .inner + .0 + .state + .lock() + .unwrap() + .default_client_config + .clone() + else { + return Err(ConnectError::NoDefaultClientConfig); + }; + + self.connect_with(config, addr, server_name) + } + + /// Connect to a remote endpoint using a custom configuration. + /// + /// See [`connect()`] for details. + /// + /// [`connect()`]: Endpoint::connect + pub fn connect_with( + &self, + config: ClientConfig, + addr: SocketAddr, + server_name: &str, + ) -> Result { + let mut endpoint = self.inner.state.lock().unwrap(); + if endpoint.driver_lost || endpoint.recv_state.connections.close.is_some() { + return Err(ConnectError::EndpointStopping); + } + if addr.is_ipv6() && !endpoint.ipv6 { + return Err(ConnectError::InvalidRemoteAddress(addr)); + } + let addr = if endpoint.ipv6 { + SocketAddr::V6(ensure_ipv6(addr)) + } else { + addr + }; + + let (ch, conn) = endpoint + .inner + .connect(self.runtime.now(), config, addr, server_name)?; + + let sender = endpoint.socket.create_sender(); + endpoint.stats.outgoing_handshakes += 1; + Ok(endpoint + .recv_state + .connections + .insert(ch, conn, sender, false, self.runtime.clone())) + } + + /// Switch to a new UDP socket + /// + /// See [`Endpoint::rebind_abstract()`] for details. + #[cfg(not(wasm_browser))] + pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { + self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + } + + /// Switch to a new UDP socket + /// + /// Allows the endpoint's address to be updated live, affecting all active connections. Incoming + /// connections and connections to servers unreachable from the new address will be lost. + /// + /// On error, the old UDP socket is retained. + pub fn rebind_abstract(&self, socket: Box) -> io::Result<()> { + let addr = socket.local_addr()?; + let mut inner = self.inner.state.lock().unwrap(); + inner.prev_socket = Some(mem::replace(&mut inner.socket, socket)); + inner.sender = inner.socket.create_sender(); + inner.ipv6 = addr.is_ipv6(); + + // Update connection socket references + for sender in inner.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Rebind(inner.socket.create_sender())); + } + if let Some(driver) = inner.driver.take() { + // Ensure the driver can register for wake-ups from the new socket + driver.wake(); + } + + Ok(()) + } + + /// Replace the server configuration, affecting new incoming connections only + /// + /// Useful for e.g. refreshing TLS certificates without disrupting existing connections. + pub fn set_server_config(&self, server_config: Option) { + self.inner + .state + .lock() + .unwrap() + .inner + .set_server_config(server_config.map(Arc::new)) + } + + /// Get the local `SocketAddr` the underlying socket is bound to + pub fn local_addr(&self) -> io::Result { + self.inner.state.lock().unwrap().socket.local_addr() + } + + /// Get the number of connections that are currently open + pub fn open_connections(&self) -> usize { + self.inner.state.lock().unwrap().inner.open_connections() + } + + /// Close all of this endpoint's connections immediately and cease accepting new connections. + /// + /// See [`Connection::close()`] for details. + /// + /// [`Connection::close()`]: crate::Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let reason = Bytes::copy_from_slice(reason); + let mut endpoint = self.inner.state.lock().unwrap(); + endpoint.recv_state.connections.close = Some((error_code, reason.clone())); + for sender in endpoint.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Close { + error_code, + reason: reason.clone(), + }); + } + self.inner.shared.incoming.notify_waiters(); + } + + /// Wait for all connections on the endpoint to be cleanly shut down + /// + /// Waiting for this condition before exiting ensures that a good-faith effort is made to notify + /// peers of recent connection closes, whereas exiting immediately could force them to wait out + /// the idle timeout period. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + pub async fn wait_idle(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.is_idle() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.idle.notified() + } + .await; + } + } +} + +/// Statistics on [Endpoint] activity +#[non_exhaustive] +#[derive(Debug, Default, Copy, Clone)] +pub struct EndpointStats { + /// Cumulative number of Quic handshakes accepted by this [Endpoint] + pub accepted_handshakes: u64, + /// Cumulative number of Quic handshakes sent from this [Endpoint] + pub outgoing_handshakes: u64, + /// Cumulative number of Quic handshakes refused on this [Endpoint] + pub refused_handshakes: u64, + /// Cumulative number of Quic handshakes ignored on this [Endpoint] + pub ignored_handshakes: u64, +} + +/// A future that drives IO on an endpoint +/// +/// This task functions as the switch point between the UDP socket object and the +/// `Endpoint` responsible for routing datagrams to their owning `Connection`. +/// In order to do so, it also facilitates the exchange of different types of events +/// flowing between the `Endpoint` and the tasks managing `Connection`s. As such, +/// running this task is necessary to keep the endpoint's connections running. +/// +/// `EndpointDriver` futures terminate when all clones of the `Endpoint` have been dropped, or when +/// an I/O error occurs. +#[must_use = "endpoint drivers must be spawned for I/O to occur"] +#[derive(Debug)] +pub(crate) struct EndpointDriver(pub(crate) EndpointRef); + +impl Future for EndpointDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut endpoint = self.0.state.lock().unwrap(); + match endpoint.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => endpoint.driver = Some(cx.waker().clone()), + } + + let now = endpoint.runtime.now(); + let mut keep_going = false; + keep_going |= endpoint.drive_recv(cx, now)?; + keep_going |= endpoint.handle_events(cx, &self.0.shared); + + if !endpoint.recv_state.incoming.is_empty() { + self.0.shared.incoming.notify_waiters(); + } + + if self.0.shared.ref_count.load(Ordering::Relaxed) == 0 && endpoint.is_idle() { + Poll::Ready(Ok(())) + } else { + drop(endpoint); + // If there is more work to do schedule the endpoint task again. + // `wake_by_ref()` is called outside the lock to minimize + // lock contention on a multithreaded runtime. + if keep_going { + cx.waker().wake_by_ref(); + } + Poll::Pending + } + } +} + +impl Drop for EndpointDriver { + fn drop(&mut self) { + let mut endpoint = self.0.state.lock().unwrap(); + endpoint.driver_lost = true; + self.0.shared.incoming.notify_waiters(); + // Drop all outgoing channels, signaling the termination of the endpoint to the associated + // connections. + endpoint.recv_state.connections.senders.clear(); + // Clearing the senders is what makes the endpoint idle, but a waiter already parked on + // `idle` only re-reads that after being notified, and with the driver gone nothing else + // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. + if endpoint.is_idle() { + self.0.shared.idle.notify_waiters(); + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +impl EndpointInner { + pub(crate) fn accept( + &self, + incoming: proto::Incoming, + server_config: Option>, + ) -> Result { + let mut response_buffer = Vec::new(); + + // Phase 1: reserve endpoint state for the connection under the lock. + let accepting = { + let mut state = self.state.lock().unwrap(); + let now = state.runtime.now(); + match state + .inner + .start_accept(incoming, now, &mut response_buffer, server_config) + { + Ok(accepting) => { + state.pending_accepts += 1; + accepting + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + return Err(error.cause); + } + } + }; + + // Phase 2: TLS session setup, connection construction, and first-packet handling, + // without holding the lock. + let accepted = accepting.accept(); + + // Phase 3: register the connection, or release the reservation, under the lock. + let mut state = self.state.lock().unwrap(); + state.pending_accepts -= 1; + let result = match state.inner.finish_accept(accepted, &mut response_buffer) { + Ok((handle, conn)) => { + state.stats.accepted_handshakes += 1; + let sender = state.socket.create_sender(); + let runtime = state.runtime.clone(); + let driver_lost = state.driver_lost; + Ok(state + .recv_state + .connections + .insert(handle, conn, sender, driver_lost, runtime)) + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + Err(error.cause) + } + }; + // Failed accepts and accepts completed after driver loss cannot rely on a Drained event + // being processed to wake idle waiters. + if state.is_idle() { + self.shared.idle.notify_waiters(); + } + result + } + + pub(crate) fn refuse(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.refused_handshakes += 1; + let mut response_buffer = Vec::new(); + let transmit = state.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut state.sender); + } + + pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + let mut state = self.state.lock().unwrap(); + let mut response_buffer = Vec::new(); + let transmit = state.inner.retry(incoming, &mut response_buffer)?; + respond(transmit, &response_buffer, &mut state.sender); + Ok(()) + } + + pub(crate) fn ignore(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.ignored_handshakes += 1; + state.inner.ignore(incoming); + } +} + +#[derive(Debug)] +pub(crate) struct State { + socket: Box, + sender: Pin>, + /// During an active migration, abandoned_socket receives traffic + /// until the first packet arrives on the new socket. + prev_socket: Option>, + inner: proto::Endpoint, + recv_state: RecvState, + driver: Option, + ipv6: bool, + events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, + driver_lost: bool, + runtime: Arc, + stats: EndpointStats, + default_client_config: Option, + /// Connections in the process of being accepted + pending_accepts: usize, +} + +#[derive(Debug)] +pub(crate) struct Shared { + incoming: Notify, + idle: Notify, + /// Number of live handles that can be used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +impl State { + fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { + let get_time = || self.runtime.now(); + self.recv_state.recv_limiter.start_cycle(get_time); + if let Some(socket) = &mut self.prev_socket { + // We don't care about the `PollProgress` from old sockets. + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut **socket, + &mut self.sender, + &*self.runtime, + now, + ); + if poll_res.is_err() { + self.prev_socket = None; + } + }; + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut *self.socket, + &mut self.sender, + &*self.runtime, + now, + ); + self.recv_state.recv_limiter.finish_cycle(get_time); + let poll_res = poll_res?; + if poll_res.received_connection_packet { + // Traffic has arrived on self.socket, therefore there is no need for the abandoned + // one anymore. TODO: Account for multiple outgoing connections. + self.prev_socket = None; + } + Ok(poll_res.keep_going) + } + + fn handle_events(&mut self, cx: &mut Context<'_>, shared: &Shared) -> bool { + for _ in 0..IO_LOOP_BOUND { + let (ch, event) = match self.events.poll_recv(cx) { + Poll::Ready(Some(x)) => x, + Poll::Ready(None) => unreachable!("EndpointInner owns one sender"), + Poll::Pending => { + return false; + } + }; + + if event.is_drained() { + self.recv_state.connections.senders.remove(&ch); + if self.is_idle() { + shared.idle.notify_waiters(); + } + } + let Some(event) = self.inner.handle_event(ch, event) else { + continue; + }; + // Ignoring errors from dropped connections that haven't yet been cleaned up + let _ = self + .recv_state + .connections + .senders + .get_mut(&ch) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + + true + } + + fn is_idle(&self) -> bool { + self.recv_state.connections.is_empty() && self.pending_accepts == 0 + } +} + +impl Drop for State { + fn drop(&mut self) { + for incoming in self.recv_state.incoming.drain(..) { + self.inner.ignore(incoming); + } + } +} + +fn respond( + transmit: proto::Transmit, + response_buffer: &[u8], + sender: &mut Pin>, +) { + // Send if there's kernel buffer space; otherwise, drop it + // + // As an endpoint-generated packet, we know this is an + // immediate, stateless response to an unconnected peer, + // one of: + // + // - A version negotiation response due to an unknown version + // - A `CLOSE` due to a malformed or unwanted connection attempt + // - A stateless reset due to an unrecognized connection + // - A `Retry` packet due to a connection attempt when + // `use_retry` is set + // + // In each case, a well-behaved peer can be trusted to retry a + // few times, which is guaranteed to produce the same response + // from us. Repeated failures might at worst cause a peer's new + // connection attempt to time out, which is acceptable if we're + // under such heavy load that there's never room for this code + // to transmit. This is morally equivalent to the packet getting + // lost due to congestion further along the link, which + // similarly relies on peer retries for recovery. + + // Copied from rust 1.85's std::task::Waker::noop() implementation for backwards compatibility + const NOOP: RawWaker = { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| NOOP, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, + ); + RawWaker::new(std::ptr::null(), &VTABLE) + }; + // SAFETY: Copied from rust stdlib, the NOOP waker is thread-safe and doesn't violate the RawWakerVTable contract, + // it doesn't access the data pointer at all. + let waker = unsafe { Waker::from_raw(NOOP) }; + let mut cx = Context::from_waker(&waker); + _ = sender.as_mut().poll_send( + &udp_transmit(&transmit, &response_buffer[..transmit.size]), + &mut cx, + ); +} + +#[inline] +fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { + match ecn { + udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, + udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, + udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + } +} + +#[derive(Debug)] +struct ConnectionSet { + /// Senders for communicating with the endpoint's connections + senders: FxHashMap>, + /// Stored to give out clones to new ConnectionInners + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + /// Set if the endpoint has been manually closed + close: Option<(VarInt, Bytes)>, +} + +impl ConnectionSet { + fn insert( + &mut self, + handle: ConnectionHandle, + conn: proto::Connection, + sender: Pin>, + driver_lost: bool, + runtime: Arc, + ) -> Connecting { + let (send, recv) = mpsc::unbounded_channel(); + if let Some((error_code, reason)) = &self.close { + send.send(ConnectionEvent::Close { + error_code: *error_code, + reason: reason.clone(), + }) + .unwrap(); + } + match driver_lost { + // Close the event channel before spawning the connection driver, so it cannot + // transmit a handshake after the endpoint driver has stopped. + true => drop(send), + false => { + self.senders.insert(handle, send); + } + } + Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + } + + fn is_empty(&self) -> bool { + self.senders.is_empty() + } +} + +fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { + match x { + SocketAddr::V6(x) => x, + SocketAddr::V4(x) => SocketAddrV6::new(x.ip().to_ipv6_mapped(), x.port(), 0, 0), + } +} + +pin_project! { + /// Future produced by [`Endpoint::accept`] + pub struct Accept<'a> { + endpoint: &'a Endpoint, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for Accept<'_> { + type Output = Option; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut endpoint = this.endpoint.inner.state.lock().unwrap(); + if endpoint.driver_lost { + return Poll::Ready(None); + } + if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // Release the mutex lock on endpoint so cloning it doesn't deadlock + drop(endpoint); + let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); + return Poll::Ready(Some(incoming)); + } + if endpoint.recv_state.connections.close.is_some() { + return Poll::Ready(None); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.endpoint.inner.shared.incoming.notified()), + } + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointRef(Arc); + +impl EndpointRef { + pub(crate) fn new( + socket: Box, + inner: proto::Endpoint, + ipv6: bool, + runtime: Arc, + ) -> Self { + let (sender, events) = mpsc::unbounded_channel(); + let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); + let sender = socket.create_sender(); + Self(Arc::new(EndpointInner { + shared: Shared { + incoming: Notify::new(), + idle: Notify::new(), + ref_count: AtomicUsize::new(0), + }, + state: Mutex::new(State { + socket, + sender, + prev_socket: None, + inner, + ipv6, + events, + driver: None, + driver_lost: false, + recv_state, + runtime, + stats: EndpointStats::default(), + default_client_config: None, + pending_accepts: 0, + }), + })) + } +} + +impl Clone for EndpointRef { + fn clone(&self) -> Self { + self.0.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for EndpointRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let endpoint = &mut *self.0.state.lock().unwrap(); + // If the driver is about to be on its own, ensure it can shut down if the last + // connection is gone. + if let Some(task) = endpoint.driver.take() { + task.wake(); + } + } +} + +impl std::ops::Deref for EndpointRef { + type Target = EndpointInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// State directly involved in handling incoming packets +struct RecvState { + incoming: VecDeque, + connections: ConnectionSet, + recv_buf: Box<[u8]>, + /// Reused buffer that received datagrams are copied into and split into owned slices. + datagrams: BytesMut, + recv_limiter: WorkLimiter, +} + +impl RecvState { + fn new( + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + max_receive_segments: usize, + endpoint: &proto::Endpoint, + ) -> Self { + let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) + as usize + * max_receive_segments; + let recv_buf = vec![0; datagram_capacity * BATCH_SIZE]; + Self { + connections: ConnectionSet { + senders: FxHashMap::default(), + sender, + close: None, + }, + incoming: VecDeque::new(), + recv_buf: recv_buf.into(), + datagrams: BytesMut::with_capacity(datagram_capacity), + recv_limiter: WorkLimiter::new(RECV_TIME_BOUND), + } + } + + fn poll_socket( + &mut self, + cx: &mut Context<'_>, + endpoint: &mut proto::Endpoint, + socket: &mut dyn AsyncUdpSocket, + sender: &mut Pin>, + runtime: &dyn Runtime, + now: Instant, + ) -> Result { + let mut received_connection_packet = false; + let mut metas = [RecvMeta::default(); BATCH_SIZE]; + let mut iovs: [IoSliceMut<'_>; BATCH_SIZE] = { + let mut bufs = self + .recv_buf + .chunks_mut(self.recv_buf.len() / BATCH_SIZE) + .map(IoSliceMut::new); + + // expect() safe as self.recv_buf is chunked into BATCH_SIZE items + // and iovs will be of size BATCH_SIZE, thus from_fn is called + // exactly BATCH_SIZE times. + std::array::from_fn(|_| bufs.next().expect("BATCH_SIZE elements")) + }; + loop { + match socket.poll_recv(cx, &mut iovs, &mut metas) { + Poll::Ready(Ok(msgs)) => { + self.recv_limiter.record_work(msgs); + // Copy the received batch into `datagrams` and split each datagram off as + // an owned slice. We can't read directly into `datagrams` and drop + // `recv_buf`: once a slice has been handed to a connection the allocation + // is shared, so the next `reserve` must allocate fresh backing memory. + // Reading in place would therefore mean reserving the worst-case iovec + // envelope before every poll and reallocating it whenever a prior datagram + // is still in flight. `recv_buf` stays an allocate-once scratch buffer, and + // we reserve only the bytes actually received. + let batch_len = metas.iter().take(msgs).map(|meta| meta.len).sum(); + self.datagrams.reserve(batch_len); + for (meta, buf) in metas.iter().zip(iovs.iter()).take(msgs) { + self.datagrams.extend_from_slice(&buf[..meta.len]); + } + for meta in metas.iter().take(msgs) { + let mut data = self.datagrams.split_to(meta.len); + while !data.is_empty() { + let buf = data.split_to(meta.stride.min(data.len())); + let mut response_buffer = Vec::new(); + match endpoint.handle( + now, + meta.addr, + meta.dst_ip, + meta.ecn.map(proto_ecn), + buf, + &mut response_buffer, + ) { + Some(DatagramEvent::NewConnection(incoming)) => { + if self.connections.close.is_none() { + self.incoming.push_back(incoming); + } else { + let transmit = + endpoint.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, sender); + } + } + Some(DatagramEvent::ConnectionEvent(handle, event)) => { + // Ignoring errors from dropped connections that haven't yet been cleaned up + received_connection_packet = true; + let _ = self + .connections + .senders + .get_mut(&handle) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + Some(DatagramEvent::Response(transmit)) => { + respond(transmit, &response_buffer, sender); + } + None => {} + } + } + } + } + Poll::Pending => { + return Ok(PollProgress { + received_connection_packet, + keep_going: false, + }); + } + // Ignore ECONNRESET as it's undefined in QUIC and may be injected by an + // attacker + Poll::Ready(Err(e)) if e.kind() == io::ErrorKind::ConnectionReset => { + continue; + } + // Ignore EMSGSIZE as we're currently not handling ICMPv4 Fragmentation Needed + // and ICMPv6 Packet Too Big (PTB) messages since they cannot be authenticated, + // and Datagram Packetization Layer Path MTU Discovery (DPLPMTUD) works without + // it anyways. + Poll::Ready(Err(e)) if is_msg_size_err(&e) => { + continue; + } + Poll::Ready(Err(e)) => { + return Err(e); + } + } + if !self.recv_limiter.allow_work(|| runtime.now()) { + return Ok(PollProgress { + received_connection_packet, + keep_going: true, + }); + } + } + } +} + +impl fmt::Debug for RecvState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RecvState") + .field("incoming", &self.incoming) + .field("connections", &self.connections) + // recv_buf too large + .field("recv_limiter", &self.recv_limiter) + .finish_non_exhaustive() + } +} + +#[derive(Default)] +struct PollProgress { + /// Whether a datagram was routed to an existing connection + received_connection_packet: bool, + /// Whether datagram handling was interrupted early by the work limiter for fairness + keep_going: bool, +} diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs new file mode 100644 index 00000000000..b5d60a08ecc --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -0,0 +1,152 @@ +use std::{ + future::{Future, IntoFuture}, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use proto::{ConnectionError, ConnectionId, ServerConfig}; +use thiserror::Error; + +use crate::{ + connection::{Connecting, Connection}, + endpoint::EndpointRef, +}; + +/// An incoming connection for which the server has not yet begun its part of the handshake +#[derive(Debug)] +pub struct Incoming(Option); + +impl Incoming { + pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + Self(Some(State { inner, endpoint })) + } + + /// Attempt to accept this incoming connection (an error may still occur) + pub fn accept(mut self) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, None) + } + + /// Accept this incoming connection using a custom configuration + /// + /// See [`accept()`][Incoming::accept] for more details. + pub fn accept_with( + mut self, + server_config: Arc, + ) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, Some(server_config)) + } + + /// Reject this incoming connection attempt + pub fn refuse(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.refuse(state.inner); + } + + /// Respond with a retry packet, requiring the client to retry with address validation + /// + /// Errors if `may_retry()` is false. + pub fn retry(mut self) -> Result<(), RetryError> { + let state = self.0.take().unwrap(); + state.endpoint.retry(state.inner).map_err(|e| { + RetryError(Box::new(Self(Some(State { + inner: e.into_incoming(), + endpoint: state.endpoint, + })))) + }) + } + + /// Ignore this incoming connection attempt, not sending any packet in response + pub fn ignore(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.ignore(state.inner); + } + + /// The local IP address which was used when the peer established the connection + pub fn local_ip(&self) -> Option { + self.0.as_ref().unwrap().inner.local_ip() + } + + /// The peer's UDP address + pub fn remote_address(&self) -> SocketAddr { + self.0.as_ref().unwrap().inner.remote_address() + } + + /// Whether the socket address that is initiating this connection has been validated + /// + /// This means that the sender of the initial packet has proved that they can receive traffic + /// sent to `self.remote_address()`. + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn remote_address_validated(&self) -> bool { + self.0.as_ref().unwrap().inner.remote_address_validated() + } + + /// Whether it is legal to respond with a retry packet + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn may_retry(&self) -> bool { + self.0.as_ref().unwrap().inner.may_retry() + } + + /// The original destination CID when initiating the connection + pub fn orig_dst_cid(&self) -> ConnectionId { + self.0.as_ref().unwrap().inner.orig_dst_cid() + } +} + +impl Drop for Incoming { + fn drop(&mut self) { + // Implicit reject, similar to Connection's implicit close + if let Some(state) = self.0.take() { + state.endpoint.refuse(state.inner); + } + } +} + +#[derive(Debug)] +struct State { + inner: proto::Incoming, + endpoint: EndpointRef, +} + +/// Error for attempting to retry an [`Incoming`] which already bears a token from a previous retry +#[derive(Debug, Error)] +#[error("retry() with validated Incoming")] +pub struct RetryError(Box); + +impl RetryError { + /// Get the [`Incoming`] + pub fn into_incoming(self) -> Incoming { + *self.0 + } +} + +/// Basic adapter to let [`Incoming`] be `await`-ed like a [`Connecting`] +#[derive(Debug)] +pub struct IncomingFuture(Result); + +impl Future for IncomingFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match &mut self.0 { + Ok(connecting) => Pin::new(connecting).poll(cx), + Err(e) => Poll::Ready(Err(e.clone())), + } + } +} + +impl IntoFuture for Incoming { + type Output = Result; + type IntoFuture = IncomingFuture; + + fn into_future(self) -> Self::IntoFuture { + IncomingFuture(self.accept()) + } +} diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs new file mode 100644 index 00000000000..649801fff81 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -0,0 +1,134 @@ +//! QUIC transport protocol implementation +//! +//! [QUIC](https://en.wikipedia.org/wiki/QUIC) is a modern transport protocol addressing +//! shortcomings of TCP, such as head-of-line blocking, poor security, slow handshakes, and +//! inefficient congestion control. This crate provides a portable userspace implementation. It +//! builds on top of quinn-proto, which implements protocol logic independent of any particular +//! runtime. +//! +//! The entry point of this crate is the [`Endpoint`]. +//! +//! # About QUIC +//! +//! A QUIC connection is an association between two endpoints. The endpoint which initiates the +//! connection is termed the client, and the endpoint which accepts it is termed the server. A +//! single endpoint may function as both client and server for different connections, for example +//! in a peer-to-peer application. To communicate application data, each endpoint may open streams +//! up to a limit dictated by its peer. Typically, that limit is increased as old streams are +//! finished. +//! +//! Streams may be unidirectional or bidirectional, and are cheap to create and disposable. For +//! example, a traditionally datagram-oriented application could use a new stream for every +//! message it wants to send, no longer needing to worry about MTUs. Bidirectional streams behave +//! much like a traditional TCP connection, and are useful for sending messages that have an +//! immediate response, such as an HTTP request. Stream data is delivered reliably, and there is no +//! ordering enforced between data on different streams. +//! +//! By avoiding head-of-line blocking and providing unified congestion control across all streams +//! of a connection, QUIC is able to provide higher throughput and lower latency than one or +//! multiple TCP connections between the same two hosts, while providing more useful behavior than +//! raw UDP sockets. +//! +//! Quinn also exposes unreliable datagrams, which are a low-level primitive preferred when +//! automatic fragmentation and retransmission of certain data is not desired. +//! +//! QUIC uses encryption and identity verification built directly on TLS 1.3. Just as with a TLS +//! server, it is useful for a QUIC server to be identified by a certificate signed by a trusted +//! authority. If this is infeasible--for example, if servers are short-lived or not associated +//! with a domain name--then as with TLS, self-signed certificates can be used to provide +//! encryption alone. +#![warn(missing_docs)] + +use std::pin::Pin; + +mod connection; +mod endpoint; +mod incoming; +mod mutex; +mod recv_stream; +mod runtime; +mod send_stream; +mod work_limiter; + +#[cfg(not(wasm_browser))] +pub(crate) use std::time::{Duration, Instant}; +#[cfg(wasm_browser)] +pub(crate) use web_time::{Duration, Instant}; + +#[cfg(feature = "bloom")] +pub use proto::BloomTokenLog; +pub use proto::{ + AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, + ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, + ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, + InvalidCid, MtuDiscoveryConfig, NoneTokenLog, NoneTokenStore, PathStats, ServerConfig, Side, + StdSystemTime, StreamId, TimeSource, TokenLog, TokenMemoryCache, TokenReuseError, TokenStore, + Transmit, TransportConfig, TransportErrorCode, UdpStats, ValidationTokenConfig, VarInt, + VarIntBoundsExceeded, Written, congestion, crypto, +}; +#[cfg(feature = "qlog")] +pub use proto::{QlogConfig, QlogStream}; +#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +pub use rustls; +pub use udp; + +pub use crate::connection::{ + AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, + RemoteAddressWatcher, SendDatagram, SendDatagramError, +}; +pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; +#[cfg(feature = "runtime-smol")] +pub use crate::runtime::SmolRuntime; +#[cfg(feature = "runtime-tokio")] +pub use crate::runtime::TokioRuntime; +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +pub use crate::runtime::default_runtime; +pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; +pub use crate::send_stream::{SendStream, StoppedError, WriteError}; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +enum ConnectionEvent { + Close { + error_code: VarInt, + reason: bytes::Bytes, + }, + Proto(proto::ConnectionEvent), + Rebind(Pin>), +} + +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { + udp::Transmit { + destination: t.destination, + ecn: t.ecn.map(udp_ecn), + contents: buffer, + segment_size: t.segment_size, + src_ip: t.src_ip, + } +} + +fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { + match ecn { + EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + } +} + +/// Maximum number of datagrams processed in send/recv calls to make before moving on to other processing +/// +/// This helps ensure we don't starve anything when the CPU is slower than the link. +/// Value is selected by picking a low number which didn't degrade throughput in benchmarks. +const IO_LOOP_BOUND: usize = 160; + +/// The maximum amount of time that should be spent in `recvmsg()` calls per endpoint iteration +/// +/// 50us are chosen so that an endpoint iteration with a 50us sendmsg limit blocks +/// the runtime for a maximum of about 100us. +/// Going much lower does not yield any noticeable difference, since a single `recvmmsg` +/// batch of size 32 was observed to take 30us on some systems. +const RECV_TIME_BOUND: Duration = Duration::from_micros(50); diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs new file mode 100644 index 00000000000..91962cb072f --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -0,0 +1,163 @@ +use std::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +#[cfg(feature = "lock_tracking")] +mod tracking { + use super::*; + use crate::{Duration, Instant}; + use std::collections::VecDeque; + use tracing::warn; + + #[derive(Debug)] + struct Inner { + last_lock_owner: VecDeque<(&'static str, Duration)>, + value: T, + } + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + pub(crate) struct Mutex { + inner: std::sync::Mutex>, + } + + impl Debug for Mutex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.inner, f) + } + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(Inner { + last_lock_owner: VecDeque::new(), + value, + }), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { + // We don't bother dispatching through Runtime::now because they're pure performance + // diagnostics. + let now = Instant::now(); + let guard = self.inner.lock().unwrap(); + + let lock_time = Instant::now(); + let elapsed = lock_time.duration_since(now); + + if elapsed > Duration::from_millis(1) { + warn!( + "Locking the connection for {} took {:?}. Last owners: {:?}", + purpose, elapsed, guard.last_lock_owner + ); + } + + MutexGuard { + guard, + start_time: lock_time, + purpose, + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, Inner>, + start_time: Instant, + purpose: &'static str, + } + + impl Drop for MutexGuard<'_, T> { + fn drop(&mut self) { + if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { + self.guard.last_lock_owner.pop_back(); + } + + let duration = self.start_time.elapsed(); + + if duration > Duration::from_millis(1) { + warn!( + "Utilizing the connection for {} took {:?}", + self.purpose, duration + ); + } + + self.guard + .last_lock_owner + .push_front((self.purpose, duration)); + } + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard.value + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard.value + } + } + + const MAX_LOCK_OWNERS: usize = 20; +} + +#[cfg(feature = "lock_tracking")] +pub(crate) use tracking::Mutex; + +#[cfg(not(feature = "lock_tracking"))] +mod non_tracking { + use super::*; + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + #[derive(Debug)] + pub(crate) struct Mutex { + inner: std::sync::Mutex, + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(value), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, _purpose: &'static str) -> MutexGuard<'_, T> { + MutexGuard { + guard: self.inner.lock().unwrap(), + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, T>, + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.guard.deref() + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.deref_mut() + } + } +} + +#[cfg(not(feature = "lock_tracking"))] +pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs new file mode 100644 index 00000000000..d3548a1a331 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -0,0 +1,823 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use bytes::Bytes; +use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use thiserror::Error; +use tokio::io::ReadBuf; + +use crate::{VarInt, connection::ConnectionRef}; + +/// A stream that can only be used to receive data +/// +/// `stop(0)` is implicitly called on drop unless: +/// - A variant of [`ReadError`] has been yielded by a read call +/// - [`stop()`] was called explicitly +/// +/// # Cancellation +/// +/// A `read` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready cannot lead to loss of stream data. This is true of methods which succeed immediately when +/// any progress is made, and is not true of methods which might need to perform multiple reads +/// internally before succeeding. Each `read` method documents whether it is cancel-safe. +/// +/// # Common issues +/// +/// ## Data never received on a locally-opened stream +/// +/// Peers are not notified of streams until they or a later-numbered stream are used to send +/// data. If a bidirectional stream is locally opened but never used to send, then the peer may +/// never see it. Application protocols should always arrange for the endpoint which will first +/// transmit on a stream to be the endpoint responsible for opening it. +/// +/// ## Data never received on a remotely-opened stream +/// +/// Verify that the stream you are receiving is the same one that the server is sending on, e.g. by +/// logging the [`id`] of each. Streams are always accepted in the same order as they are created, +/// i.e. ascending order by [`StreamId`]. For example, even if a sender first transmits on +/// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver +/// will be bidirectional stream 0. +/// +/// [`ReadError`]: crate::ReadError +/// [`stop()`]: RecvStream::stop +/// [`SendStream::finish`]: crate::SendStream::finish +/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`id`]: RecvStream::id +/// [`Connection::accept_bi`]: crate::Connection::accept_bi +#[derive(Debug)] +pub struct RecvStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, + all_data_read: bool, + reset: Option, +} + +impl RecvStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + all_data_read: false, + reset: None, + } + } + + /// Read data contiguously from the stream. + /// + /// Yields the number of bytes read into `buf` on success, or `None` if the stream was finished. + /// + /// This operation is cancel-safe. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + Read { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Read an exact number of bytes contiguously from the stream. + /// + /// See [`read()`] for details. This operation is *not* cancel-safe. + /// + /// [`read()`]: RecvStream::read + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + ReadExact { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Attempts to read from the stream into the provided buffer + /// + /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this + /// returns zero bytes read (and `buf` has a non-zero length), that indicates that the remote + /// side has [`finish`]ed the stream and the local side has already read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read( + &mut self, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(self.poll_read_buf(cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } + + /// Attempts to read from the stream into the provided buffer, which may be uninitialized + /// + /// On success, returns `Poll::Ready(Ok(()))` and places data into the unfilled portion of + /// `buf`. If this does not write any bytes to `buf` (and `buf.remaining()` is non-zero), that + /// indicates that the remote side has [`finish`]ed the stream and the local side has already + /// read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read_buf( + &mut self, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = false; + loop { + if buf.remaining() == 0 { + // We know `read` is `true` because `buf.remaining()` was not 0 before + return ReadStatus::Readable(()); + } + + match chunks.next(buf.remaining()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read = true; + } + res => return (if read { Some(()) } else { None }, res.err()).into(), + } + } + }) + .map(|res| res.map(|_| ())) + } + + /// Read the next segment of data + /// + /// Yields `None` if the stream was finished. Otherwise, yields a segment of data and its + /// offset in the stream. If `ordered` is `true`, the chunk's offset will be immediately after + /// the last data yielded by `read()` or `read_chunk()`. If `ordered` is `false`, segments may + /// be received in any order, and the `Chunk`'s `offset` field can be used to determine + /// ordering in the caller. Unordered reads are less prone to head-of-line blocking within a + /// stream, but require the application to manage reassembling the original data. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries do not correspond + /// to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunk( + &mut self, + max_length: usize, + ordered: bool, + ) -> Result, ReadError> { + ReadChunk { + stream: self, + max_length, + ordered, + } + .await + } + + /// Attempts to read a chunk from the stream. + /// + /// On success, returns `Poll::Ready(Ok(Some(chunk)))`. If `Poll::Ready(Ok(None))` + /// is returned, it implies that EOF has been reached. + /// + /// If no data is available for reading, the method returns `Poll::Pending` + /// and arranges for the current task (via cx.waker()) to receive a notification + /// when the stream becomes readable or is closed. + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max_length: usize, + ordered: bool, + ) -> Poll, ReadError>> { + self.poll_read_generic(cx, ordered, |chunks| match chunks.next(max_length) { + Ok(Some(chunk)) => ReadStatus::Readable(chunk), + res => (None, res.err()).into(), + }) + } + + /// Read the next segments of data + /// + /// Fills `bufs` with the segments of data beginning immediately after the + /// last data yielded by `read` or `read_chunk`, or `None` if the stream was + /// finished. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries + /// do not correspond to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunks(&mut self, bufs: &mut [Bytes]) -> Result, ReadError> { + ReadChunks { stream: self, bufs }.await + } + + /// Foundation of [`Self::read_chunks`] + fn poll_read_chunks( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [Bytes], + ) -> Poll, ReadError>> { + if bufs.is_empty() { + return Poll::Ready(Ok(Some(0))); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = 0; + loop { + if read >= bufs.len() { + // We know `read > 0` because `bufs` cannot be empty here + return ReadStatus::Readable(read); + } + + match chunks.next(usize::MAX) { + Ok(Some(chunk)) => { + bufs[read] = chunk.bytes; + read += 1; + } + res => return (if read == 0 { None } else { Some(read) }, res.err()).into(), + } + } + }) + } + + /// Convenience method to read all remaining data into a buffer + /// + /// Fails with [`ReadToEndError::TooLong`] on reading more than `size_limit` bytes, discarding + /// all data read. Uses unordered reads to be more efficient than using `AsyncRead` would + /// allow. `size_limit` should be set to limit worst-case memory use. + /// + /// If unordered reads have already been made, the resulting buffer may have gaps containing + /// arbitrary data. + /// + /// This operation is *not* cancel-safe. + /// + /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + ReadToEnd { + stream: self, + buffer: ReadToEndBuffer::new(size_limit), + } + .await + } + + /// Stop accepting data + /// + /// Discards unread data and notifies the peer to stop transmitting. Once stopped, further + /// attempts to operate on a stream will yield `ClosedStream` errors. + pub fn stop(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("RecvStream::stop"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.recv_stream(self.stream).stop(error_code)?; + conn.wake(); + self.all_data_read = true; + // Clean up shared state that might be left over from a cancalled read + // operation, so `drop` doesn't have to + conn.blocked_readers.remove(&self.stream); + Ok(()) + } + + /// Check if this stream predates completion of the handshake on an incoming connection. + /// + /// True only if the stream was accepted before the handshake completed, which is only possible + /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// client chose to send 0-RTT data. + /// + /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application + /// data may be a replay attack. To guard against this, applications should not execute + /// non-idempotent operations until + /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + pub fn is_0rtt(&self) -> bool { + self.is_0rtt + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Completes when the stream has been reset by the peer or otherwise closed + /// + /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` + /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was + /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// which it is no longer meaningful for the stream to be reset. + /// + /// This operation is cancel-safe. + pub async fn received_reset(&mut self) -> Result, ResetError> { + poll_fn(|cx| { + let mut conn = self.conn.state.lock("RecvStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Poll::Ready(Err(ResetError::ZeroRttRejected)); + } + + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + + match conn.inner.recv_stream(self.stream).received_reset() { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(error_code)) => { + // Stream state has just now been freed, so the connection may need to issue new + // stream ID flow control credit + conn.wake(); + Poll::Ready(Ok(Some(error_code))) + } + Ok(None) => { + if let Some(e) = &conn.error { + return Poll::Ready(Err(e.clone().into())); + } + // Resets always notify readers, since a reset is an immediate read error. We + // could introduce a dedicated channel to reduce the risk of spurious wakeups, + // but that increased complexity is probably not justified, as an application + // that is expecting a reset is not likely to receive large amounts of data. + conn.blocked_readers.insert(self.stream, cx.waker().clone()); + Poll::Pending + } + } + }) + .await + } + + /// Handle common logic related to reading out of a receive stream + /// + /// This takes an `FnMut` closure that takes care of the actual reading process, matching + /// the detailed read semantics for the calling function with a particular return type. + /// The closure can read from the passed `&mut Chunks` and has to return the status after + /// reading: the amount of data read, and the status after the final read call. + fn poll_read_generic( + &mut self, + cx: &mut Context<'_>, + ordered: bool, + mut read_fn: T, + ) -> Poll, ReadError>> + where + T: FnMut(&mut Chunks<'_>) -> ReadStatus, + { + use proto::ReadError::*; + use std::collections::hash_map::Entry; + + if self.all_data_read { + return Poll::Ready(Ok(None)); + } + + let mut conn = self.conn.state.lock("RecvStream::poll_read"); + if self.is_0rtt { + conn.check_0rtt().map_err(|()| ReadError::ZeroRttRejected)?; + } + + // If we stored an error during a previous call, return it now. This can happen if a + // `read_fn` both wants to return data and also returns an error in its final stream status. + let status = match self.reset { + Some(code) => ReadStatus::Failed(None, Reset(code)), + None => { + let mut recv = conn.inner.recv_stream(self.stream); + let mut chunks = recv.read(ordered)?; + let status = read_fn(&mut chunks); + if chunks.finalize().should_transmit() { + conn.wake(); + } + status + } + }; + + match status { + ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), + ReadStatus::Finished(read) => { + self.all_data_read = true; + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Ok(read)) + } + ReadStatus::Failed(read, Blocked) => match read { + Some(val) => Poll::Ready(Ok(Some(val))), + None => { + if let Some(x) = &conn.error { + return Poll::Ready(Err(ReadError::ConnectionLost(x.clone()))); + } + match conn.blocked_readers.entry(self.stream) { + Entry::Occupied(mut entry) => { + entry.get_mut().clone_from(cx.waker()); + } + Entry::Vacant(entry) => { + entry.insert(cx.waker().clone()); + } + } + Poll::Pending + } + }, + ReadStatus::Failed(read, Reset(error_code)) => match read { + None => { + self.all_data_read = true; + self.reset = Some(error_code); + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Err(ReadError::Reset(error_code))) + } + done => { + self.reset = Some(error_code); + Poll::Ready(Ok(done)) + } + }, + } + } +} + +enum ReadStatus { + Readable(T), + Finished(Option), + Failed(Option, proto::ReadError), +} + +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { + match status { + (read, None) => Self::Finished(read), + (read, Some(e)) => Self::Failed(read, e), + } + } +} + +/// Future produced by [`RecvStream::read_to_end()`]. +/// +/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +struct ReadToEnd<'a> { + stream: &'a mut RecvStream, + buffer: ReadToEndBuffer, +} + +impl Future for ReadToEnd<'_> { + type Output = Result, ReadToEndError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + loop { + match ready!(self.stream.poll_read_chunk(cx, usize::MAX, false))? { + Some(chunk) => self.buffer.push(chunk)?, + None => return Poll::Ready(Ok(self.buffer.finish())), + } + } + } +} + +struct ReadToEndBuffer { + read: Vec<(Bytes, u64)>, + start: u64, + end: u64, + size_limit: usize, +} + +impl ReadToEndBuffer { + fn new(size_limit: usize) -> Self { + Self { + read: Vec::new(), + start: u64::MAX, + end: 0, + size_limit, + } + } + + fn push(&mut self, chunk: Chunk) -> Result<(), ReadToEndError> { + self.start = self.start.min(chunk.offset); + self.end = self.end.max(chunk.bytes.len() as u64 + chunk.offset); + if (self.end - self.start) > self.size_limit as u64 { + return Err(ReadToEndError::TooLong); + } + self.read.push((chunk.bytes, chunk.offset)); + Ok(()) + } + + fn finish(&mut self) -> Vec { + if self.end == 0 { + return Vec::new(); + } + let mut buffer = vec![0; (self.end - self.start) as usize]; + for (data, offset) in self.read.drain(..) { + let offset = (offset - self.start) as usize; + buffer[offset..offset + data.len()].copy_from_slice(&data); + } + buffer + } +} + +/// Errors from [`RecvStream::read_to_end`] +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadToEndError { + /// An error occurred during reading + #[error("read error: {0}")] + Read(#[from] ReadError), + /// The stream is larger than the user-supplied limit + #[error("stream too long")] + TooLong, +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + ready!(Self::poll_read_buf(self.get_mut(), cx, buf))?; + Poll::Ready(Ok(())) + } +} + +impl Drop for RecvStream { + fn drop(&mut self) { + if self.all_data_read { + debug_assert!( + !self + .conn + .state + .lock("RecvStream:drop") + .blocked_readers + .contains_key(&self.stream), + "Stream {} should not have a blocked reader when all data read is true", + self.stream + ); + return; + } + let mut conn = self.conn.state.lock("RecvStream::drop"); + + // clean up any previously registered wakers + conn.blocked_readers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + + // Ignore ClosedStream errors + let _ = conn.inner.recv_stream(self.stream).stop(0u32.into()); + conn.wake(); + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadError { + /// The peer abandoned transmitting data on this stream + /// + /// Carries an application-defined error code. + #[error("stream reset by peer: error {0}")] + Reset(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been stopped, finished, or reset + #[error("closed stream")] + ClosedStream, + /// Attempted an ordered read following an unordered read + /// + /// Performing an unordered read allows discontinuities to arise in the receive buffer of a + /// stream which cannot be recovered, making further ordered reads impossible. + #[error("ordered read after unordered read")] + IllegalOrderedRead, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for ReadError { + fn from(e: ReadableError) -> Self { + match e { + ReadableError::ClosedStream => Self::ClosedStream, + ReadableError::IllegalOrderedRead => Self::IllegalOrderedRead, + } + } +} + +impl From for ReadError { + fn from(e: ResetError) -> Self { + match e { + ResetError::ConnectionLost(e) => Self::ConnectionLost(e), + ResetError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: ReadError) -> Self { + use ReadError::*; + let kind = match x { + Reset { .. } | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + IllegalOrderedRead => io::ErrorKind::InvalidInput, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while waiting for a stream to be reset +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ResetError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: ResetError) -> Self { + use ResetError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Future produced by [`RecvStream::read()`]. +/// +/// [`RecvStream::read()`]: crate::RecvStream::read +struct Read<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for Read<'_> { + type Output = Result, ReadError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + match this.buf.filled().len() { + 0 if this.buf.capacity() != 0 => Poll::Ready(Ok(None)), + n => Poll::Ready(Ok(Some(n))), + } + } +} + +/// Future produced by [`RecvStream::read_exact()`]. +/// +/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +struct ReadExact<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for ReadExact<'_> { + type Output = Result<(), ReadExactError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let mut remaining = this.buf.remaining(); + while remaining > 0 { + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + let new = this.buf.remaining(); + if new == remaining { + return Poll::Ready(Err(ReadExactError::FinishedEarly(this.buf.filled().len()))); + } + remaining = new; + } + Poll::Ready(Ok(())) + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadExactError { + /// The stream finished before all bytes were read + #[error("stream finished early ({0} bytes read)")] + FinishedEarly(usize), + /// A read error occurred + #[error(transparent)] + ReadError(#[from] ReadError), +} + +/// Future produced by [`RecvStream::read_chunk()`]. +/// +/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +struct ReadChunk<'a> { + stream: &'a mut RecvStream, + max_length: usize, + ordered: bool, +} + +impl Future for ReadChunk<'_> { + type Output = Result, ReadError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (max_length, ordered) = (self.max_length, self.ordered); + self.stream.poll_read_chunk(cx, max_length, ordered) + } +} + +/// Future produced by [`RecvStream::read_chunks()`]. +/// +/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +struct ReadChunks<'a> { + stream: &'a mut RecvStream, + bufs: &'a mut [Bytes], +} + +impl Future for ReadChunks<'_> { + type Output = Result, ReadError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + this.stream.poll_read_chunks(cx, this.bufs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_to_end_reordered_limit() { + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 4, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 0, + bytes: Bytes::from_static(b"abcd") + }), + Err(ReadToEndError::TooLong) + ); + assert_eq!(buffer.read.len(), 1); + } + + #[test] + fn read_to_end_reordered_exact_limit_after_prefix() { + let mut buffer = ReadToEndBuffer::new(8); + // A previously consumed prefix must not count against the remaining-data limit. + buffer + .push(Chunk { + offset: 104, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 100, + bytes: Bytes::from_static(b"abcd"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"abcdefgh"); + } + + #[test] + fn read_to_end_gaps_and_empty() { + assert!(ReadToEndBuffer::new(0).finish().is_empty()); + let mut buffer = ReadToEndBuffer::new(5); + buffer + .push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"a\0\0\0e"); + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e") + }), + Err(ReadToEndError::TooLong) + ); + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs new file mode 100644 index 00000000000..9898a0e879c --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -0,0 +1,330 @@ +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +use std::sync::Arc; +use std::{ + fmt::{self, Debug}, + future::Future, + io::{self, IoSliceMut}, + net::SocketAddr, + pin::Pin, + task::{self, Context, Poll}, + time::Duration, +}; + +use udp::{RecvMeta, Transmit}; + +use crate::Instant; + +/// Abstracts I/O and timer operations for runtime independence +pub trait Runtime: Send + Sync + Debug + 'static { + /// Construct a timer that will expire at `i` + fn new_timer(&self, i: Instant) -> Pin>; + /// Drive `future` to completion in the background + #[track_caller] + fn spawn(&self, future: Pin + Send>>); + /// Convert `t` into the socket type used by this runtime + #[cfg(not(wasm_browser))] + fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; + /// Look up the current time + /// + /// Allows simulating the flow of time for testing. + fn now(&self) -> Instant { + Instant::now() + } +} + +/// Abstract implementation of an async timer for runtime independence +pub trait AsyncTimer: Send + Debug + 'static { + /// Update the timer to expire at `i` + fn reset(self: Pin<&mut Self>, i: Instant); + /// Check whether the timer has expired, and register to be woken if not + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; +} + +/// Abstract implementation of a UDP socket for runtime independence +pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { + /// Create a [`UdpSender`] that can register a single task for write-readiness notifications + /// and send a transmit, if ready. + /// + /// A `poll_send` method on a single object can usually store only one [`Waker`] at a time, + /// i.e. allow at most one caller to wait for an event. This method allows any number of + /// interested tasks to construct their own [`UdpSender`] object. They can all then wait for the + /// same event and be notified concurrently, because each [`UdpSender`] can store a separate + /// [`Waker`]. + /// + /// [`Waker`]: std::task::Waker + fn create_sender(&self) -> Pin>; + + /// Receive UDP datagrams, or register to be woken if receiving may succeed in the future + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll>; + + /// Look up the local IP address and port used by this socket + fn local_addr(&self) -> io::Result; + + /// Maximum number of datagrams that might be described by a single [`RecvMeta`] + fn max_receive_segments(&self) -> usize { + 1 + } + + /// Whether datagrams might get fragmented into multiple parts + /// + /// Sockets should prevent this for best performance. See e.g. the `IPV6_DONTFRAG` socket + /// option. + fn may_fragment(&self) -> bool { + true + } +} + +/// An object for asynchronously writing to an associated [`AsyncUdpSocket`]. +/// +/// Any number of [`UdpSender`]s may exist for a single [`AsyncUdpSocket`]. Each [`UdpSender`] is +/// responsible for notifying at most one task for send readiness. +pub trait UdpSender: Send + Sync + Debug + 'static { + /// Send a UDP datagram, or register to be woken if sending may succeed in the future. + /// + /// Usually implementations of this will poll the socket for writability before trying to + /// write to them, and retry both if writing fails. + /// + /// Quinn will create multiple [`UdpSender`]s, one for each task it's using it from. Thus it's + /// important to poll the underlying socket in a way that doesn't overwrite wakers. + /// + /// A single [`UdpSender`] will be re-used, even if `poll_send` returns `Poll::Ready` once, + /// unlike [`Future::poll`], so calling it again after readiness should not panic. + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll>; + + /// Maximum number of datagrams that a [`Transmit`] may encode. + fn max_transmit_segments(&self) -> usize { + 1 + } +} + +pin_project_lite::pin_project! { + /// A helper for constructing [`UdpSender`]s from an underlying `Socket` type. + /// + /// This struct implements [`UdpSender`] if `MakeWritableFn` produces a `WritableFut`. + /// + /// Also serves as a trick, since `WritableFut` doesn't need to be a named future, + /// it can be an anonymous async block, as long as `MakeWritableFn` produces that + /// anonymous async block type. + /// + /// The `UdpSenderHelper` generic type parameters don't need to named, as it will be + /// used in its dyn-compatible form as a `Pin>`. + struct UdpSenderHelper { + socket: Socket, + last_send_error: Option, + make_writable_fut_fn: MakeWritableFutFn, + #[pin] + writable_fut: Option, + } +} + +impl Debug + for UdpSenderHelper +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("UdpSender") + } +} + +impl + UdpSenderHelper +{ + /// Create helper that implements [`UdpSender`] from a socket. + /// + /// Additionally you need to provide what is essentially an async function + /// that resolves once the socket is write-ready. + /// + /// See also the bounds on this struct's [`UdpSender`] implementation. + #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] + fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { + Self { + socket: inner, + last_send_error: None, + make_writable_fut_fn: make_fut, + writable_fut: None, + } + } +} + +impl UdpSender + for UdpSenderHelper +where + Socket: UdpSenderHelperSocket, + MakeWritableFutFn: Fn(&Socket) -> WritableFut + Send + Sync + 'static, + WritableFut: Future> + Send + Sync + 'static, +{ + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let mut this = self.project(); + loop { + if this.writable_fut.is_none() { + this.writable_fut + .set(Some((this.make_writable_fut_fn)(this.socket))); + } + // We're forced to `unwrap` here because `Fut` may be `!Unpin`, which means we can't safely + // obtain an `&mut WritableFut` after storing it in `self.writable_fut` when `self` is already behind `Pin`, + // and if we didn't store it then we wouldn't be able to keep it alive between + // `poll_send` calls. + let result = task::ready!(this.writable_fut.as_mut().as_pin_mut().unwrap().poll(cx)); + + // Polling an arbitrary `Future` after it becomes ready is a logic error, so arrange for + // a new `Future` to be created on the next call. + this.writable_fut.set(None); + + // If .writable() fails, propagate the error + result?; + + match this.socket.try_send(transmit) { + // We thought the socket was writable, but it wasn't, then retry so that either another + // `writable().await` call determines that the socket is indeed not writable and + // registers us for a wakeup, or the send succeeds if this really was just a + // transient failure. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => { + log_sendmsg_error(this.last_send_error, &e, transmit); + return Poll::Ready(Ok(())); + } + Ok(()) => return Poll::Ready(Ok(())), + } + } + } + + fn max_transmit_segments(&self) -> usize { + self.socket.max_transmit_segments() + } +} + +/// Limits I/O error logging to one message per minute. +const IO_ERROR_LOG_INTERVAL: Duration = Duration::from_secs(60); + +fn log_sendmsg_error( + last_send_error: &mut Option, + error: &io::Error, + transmit: &Transmit<'_>, +) { + #[cfg(unix)] + // Unix `EMSGSIZE` is expected for MTU probes. + if udp::is_msg_size_err(error) { + return; + } + + let now = Instant::now(); + if last_send_error + .is_some_and(|last| now.saturating_duration_since(last) <= IO_ERROR_LOG_INTERVAL) + { + return; + } + *last_send_error = Some(now); + + tracing::warn!( + "sendmsg error: {:?}, Transmit: {{ destination: {:?}, src_ip: {:?}, ecn: {:?}, len: {:?}, segment_size: {:?} }}", + error, + transmit.destination, + transmit.src_ip, + transmit.ecn, + transmit.contents.len(), + transmit.segment_size + ); +} + +/// Parts of the [`UdpSender`] trait that aren't asynchronous or require storing wakers. +/// +/// This trait is used by [`UdpSenderHelper`] to help construct [`UdpSender`]s. +trait UdpSenderHelperSocket: Send + Sync + 'static { + /// Try to send a transmit, if the socket happens to be write-ready. + /// + /// If not write-ready, this is allowed to return [`std::io::ErrorKind::WouldBlock`]. + /// + /// The [`UdpSenderHelper`] will use this to implement [`UdpSender::poll_send`]. + fn try_send(&self, transmit: &Transmit<'_>) -> io::Result<()>; + + /// See [`UdpSender::max_transmit_segments`]. + fn max_transmit_segments(&self) -> usize; +} + +/// Automatically select an appropriate runtime from those enabled at compile time +/// +/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, +/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is +/// returned. Otherwise, `None` is returned. +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +#[allow(clippy::needless_return)] // Be sure we return the right thing +pub fn default_runtime() -> Option> { + #[cfg(feature = "runtime-tokio")] + { + if ::tokio::runtime::Handle::try_current().is_ok() { + return Some(Arc::new(TokioRuntime)); + } + } + + #[cfg(feature = "runtime-smol")] + { + return Some(Arc::new(SmolRuntime)); + } + + #[cfg(not(feature = "runtime-smol"))] + None +} + +#[cfg(feature = "runtime-tokio")] +mod tokio; +#[cfg(feature = "runtime-tokio")] +pub use tokio::TokioRuntime; + +#[cfg(feature = "runtime-smol")] +mod smol; +#[cfg(feature = "runtime-smol")] +pub use smol::*; + +#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +mod tests { + use std::{future, task::Waker}; + + use super::*; + + #[test] + fn non_would_block_send_errors_are_logged_and_ignored() { + let mut sender = Box::pin(UdpSenderHelper::new(TestSocket, writable)); + let transmit = Transmit { + destination: SocketAddr::from(([127, 0, 0, 1], 4433)), + ecn: None, + contents: &[], + segment_size: None, + src_ip: None, + }; + let mut cx = Context::from_waker(Waker::noop()); + + let result = sender.as_mut().poll_send(&transmit, &mut cx); + + assert!(matches!(result, Poll::Ready(Ok(())))); + assert!(sender.last_send_error.is_some()); + } + + #[derive(Debug)] + struct TestSocket; + + impl UdpSenderHelperSocket for TestSocket { + fn try_send(&self, _transmit: &Transmit<'_>) -> io::Result<()> { + Err(io::ErrorKind::PermissionDenied.into()) + } + + fn max_transmit_segments(&self) -> usize { + 1 + } + } + + fn writable(_: &TestSocket) -> future::Ready> { + future::ready(Ok(())) + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs new file mode 100644 index 00000000000..b5ae3d0cf2d --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/smol.rs @@ -0,0 +1,105 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, + time::Instant, +}; +use std::{io, sync::Arc, task::ready}; + +use async_io::Async; +use async_io::Timer; + +use super::AsyncTimer; +use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for smol +#[derive(Debug)] +pub struct SmolRuntime; + +impl Runtime for SmolRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(Timer::at(t)) + } + + fn spawn(&self, future: Pin + Send>>) { + ::smol::spawn(future).detach(); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket::new(sock)?)) + } +} + +impl AsyncTimer for Timer { + fn reset(mut self: Pin<&mut Self>, t: Instant) { + self.set_at(t) + } + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx).map(|_| ()) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc>, + inner: Arc, +} + +impl UdpSocket { + fn new(sock: std::net::UdpSocket) -> io::Result { + Ok(Self { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(Async::new_nonblocking(sock)?), + }) + } +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.inner.try_send((&self.io).into(), transmit)?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_readable(cx))?; + match self.inner.recv((&self.io).into(), bufs, meta) { + Ok(res) => return Poll::Ready(Ok(res)), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.as_ref().as_ref().local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs new file mode 100644 index 00000000000..ac5dfd6f205 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -0,0 +1,111 @@ +use std::{ + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll, ready}, + time::Instant, +}; + +use tokio::{ + io::Interest, + time::{Sleep, sleep_until}, +}; + +use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for Tokio +#[derive(Debug)] +pub struct TokioRuntime; + +impl Runtime for TokioRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(sleep_until(t.into())) + } + + fn spawn(&self, future: Pin + Send>>) { + tokio::spawn(future); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) + } + + fn now(&self) -> Instant { + tokio::time::Instant::now().into_std() + } +} + +impl AsyncTimer for Sleep { + fn reset(self: Pin<&mut Self>, t: Instant) { + Self::reset(self, t.into()) + } + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc, + inner: Arc, +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.io.try_io(Interest::WRITABLE, || { + self.inner.try_send((&self.io).into(), transmit) + })?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_recv_ready(cx))?; + match self.io.try_io(Interest::READABLE, || { + self.inner.recv((&self.io).into(), bufs, meta) + }) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`. Looping on any other + // error would spin: readiness stays asserted, so `poll_recv_ready` + // completes at once and the recv fails again, without ever yielding. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs new file mode 100644 index 00000000000..570db9ac817 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -0,0 +1,454 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::{Pin, pin}, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use thiserror::Error; + +use crate::{ + VarInt, + connection::{ConnectionRef, State}, +}; + +/// A stream that can only be used to send data +/// +/// If dropped, streams that haven't been explicitly [`reset()`] will be implicitly [`finish()`]ed, +/// continuing to (re)transmit previously written data until it has been fully acknowledged or the +/// connection is closed. +/// +/// # Cancellation +/// +/// A `write` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready will always result in no data being written to the stream. This is true of methods which +/// succeed immediately when any progress is made, and is not true of methods which might need to +/// perform multiple writes internally before succeeding. Each `write` method documents whether it is +/// cancel-safe. +/// +/// [`reset()`]: SendStream::reset +/// [`finish()`]: SendStream::finish +#[derive(Debug)] +pub struct SendStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, +} + +impl SendStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + } + } + + /// Write a buffer into this stream, returning how many bytes were written + /// + /// Unless this method errors, it waits until some amount of `buf` can be written into this + /// stream, and then writes as much as it can without waiting again. Due to congestion and flow + /// control, this may be shorter than `buf.len()`. On success this yields the length of the + /// prefix that was written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write(&mut self, buf: &[u8]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write(buf))).await + } + + /// Write a buffer into this stream in its entirety + /// + /// This method repeatedly calls [`write`](Self::write) until all bytes are written, or an + /// error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some prefix of `buf` + /// may have been written when previously polled. + pub async fn write_all(&mut self, mut buf: &[u8]) -> Result<(), WriteError> { + while !buf.is_empty() { + let written = self.write(buf).await?; + buf = &buf[written..]; + } + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream, returning how much was written + /// + /// Bytes to try to write are provided to this method as an array of cheaply cloneable chunks. + /// Unless this method errors, it waits until some amount of those bytes can be written into + /// this stream, and then writes as much as it can without waiting again. Due to congestion and + /// flow control, this may be less than the total number of bytes. + /// + /// On success, this method both mutates `bufs` and yields an informative [`Written`] struct + /// indicating how much was written: + /// + /// - [`Bytes`] chunks that were fully written are mutated to be [empty](Bytes::is_empty). + /// - If a [`Bytes`] chunk was partially written, it is [split to](Bytes::split_to) contain + /// only the suffix of bytes that were not written. + /// - The yielded [`Written`] struct indicates how many chunks were fully written as well as + /// how many bytes were written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write_chunks(&mut self, bufs: &mut [Bytes]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write_chunks(bufs))).await + } + + /// Write a single [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an single cheaply cloneable chunk. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.write_all_chunks(&mut [buf]).await?; + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an array of cheaply cloneable chunks. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. This method mutates `bufs` by mutating all chunks to be + /// [empty](Bytes::is_empty). + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_all_chunks(&mut self, mut bufs: &mut [Bytes]) -> Result<(), WriteError> { + while !bufs.is_empty() { + let written = self.write_chunks(bufs).await?; + bufs = &mut bufs[written.chunks..]; + } + Ok(()) + } + + fn execute_poll( + &mut self, + cx: &mut Context<'_>, + write_fn: F, + ) -> Poll> + where + F: FnOnce(&mut proto::SendStream<'_>) -> Result, + { + use proto::WriteError::*; + let mut conn = self.conn.state.lock("SendStream::poll_write"); + if self.is_0rtt { + conn.check_0rtt() + .map_err(|()| WriteError::ZeroRttRejected)?; + } + if let Some(x) = &conn.error { + return Poll::Ready(Err(WriteError::ConnectionLost(x.clone()))); + } + + let result = match write_fn(&mut conn.inner.send_stream(self.stream)) { + Ok(result) => result, + Err(Blocked) => { + conn.blocked_writers.insert(self.stream, cx.waker().clone()); + return Poll::Pending; + } + Err(Stopped(error_code)) => { + return Poll::Ready(Err(WriteError::Stopped(error_code))); + } + Err(ClosedStream) => { + return Poll::Ready(Err(WriteError::ClosedStream)); + } + }; + + conn.wake(); + Poll::Ready(Ok(result)) + } + + /// Notify the peer that no more data will ever be written to this stream + /// + /// It is an error to write to a [`SendStream`] after `finish()`ing it. [`reset()`](Self::reset) + /// may still be called after `finish` to abandon transmission of any stream data that might + /// still be buffered. + /// + /// To wait for the peer to receive all buffered stream data, see [`stopped()`](Self::stopped). + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("finish"); + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(FinishError::ClosedStream) => Err(ClosedStream::default()), + // Harmless. If the application needs to know about stopped streams at this point, it + // should call `stopped`. + Err(FinishError::Stopped(_)) => Ok(()), + } + } + + /// Close the send stream immediately. + /// + /// No new data can be written after calling this method. Locally buffered data is dropped, and + /// previously transmitted data will no longer be retransmitted if lost. If an attempt has + /// already been made to finish the stream, the peer may still receive all written data. + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn reset(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.send_stream(self.stream).reset(error_code)?; + conn.wake(); + Ok(()) + } + + /// Set the priority of the send stream + /// + /// Every send stream has an initial priority of 0. Locally buffered data from streams with + /// higher priority will be transmitted before data from streams with lower priority. Changing + /// the priority of a stream with pending data may only take effect after that data has been + /// transmitted. Using many different priority levels per connection may have a negative + /// impact on performance. + pub fn set_priority(&self, priority: i32) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::set_priority"); + conn.inner.send_stream(self.stream).set_priority(priority)?; + Ok(()) + } + + /// Get the priority of the send stream + pub fn priority(&self) -> Result { + let mut conn = self.conn.state.lock("SendStream::priority"); + conn.inner.send_stream(self.stream).priority() + } + + /// Completes when the peer stops the stream or reads the stream to completion + /// + /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the + /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt + /// of all stream data (although not necessarily the processing of it), after which the peer + /// closing the stream is no longer meaningful. + /// + /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving + /// data. As such, relying on `stopped` to know when the peer has read a stream to completion + /// may introduce more latency than using an application-level response of some sort. + /// + /// Clients may wish to await this after finishing a unidirectional 0-RTT stream to reliably + /// determine whether the stream was rejected. + pub fn stopped( + &self, + ) -> impl Future, StoppedError>> + Send + Sync + 'static { + let conn = self.conn.clone(); + let stream = self.stream; + let is_0rtt = self.is_0rtt; + async move { + loop { + // The `Notify::notified` future needs to be created while the lock is being held, + // otherwise a wakeup could be missed if triggered inbetween releasing the lock + // and creating the future. + // The lock may only be held in a block without `await`s, otherwise the future + // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore + // we need to declare `notify` outside of the block, and initialize it inside. + let notify; + { + let mut conn = conn.state.lock("SendStream::stopped"); + if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { + return output; + } + + notify = conn.stopped.entry(stream).or_default().clone(); + notify.notified() + } + .await + } + } + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Attempt to write bytes from buf into the stream. + /// + /// On success, returns Poll::Ready(Ok(num_bytes_written)). + /// + /// If the stream is not ready for writing, the method returns Poll::Pending and arranges + /// for the current task (via cx.waker().wake_by_ref()) to receive a notification when the + /// stream becomes writable or is closed. + pub fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + pin!(self.get_mut().write(buf)).as_mut().poll(cx) + } +} + +/// Check if a send stream is stopped. +/// +/// Returns `Some` if the stream is stopped or the connection is closed. +/// Returns `None` if the stream is not stopped. +fn send_stream_stopped( + conn: &mut State, + stream: StreamId, + is_0rtt: bool, +) -> Option, StoppedError>> { + if is_0rtt && conn.check_0rtt().is_err() { + return Some(Err(StoppedError::ZeroRttRejected)); + } + match conn.inner.send_stream(stream).stopped() { + Err(ClosedStream { .. }) => Some(Ok(None)), + Ok(Some(error_code)) => Some(Ok(Some(error_code))), + Ok(None) => conn.error.clone().map(|error| Err(error.into())), + } +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl Drop for SendStream { + fn drop(&mut self) { + let mut conn = self.conn.state.lock("SendStream::drop"); + + // clean up any previously registered wakers + conn.blocked_writers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => conn.wake(), + Err(FinishError::Stopped(reason)) => { + if conn.inner.send_stream(self.stream).reset(reason).is_ok() { + conn.wake(); + } + } + // Already finished or reset, which is fine. + Err(FinishError::ClosedStream) => {} + } + } +} + +/// Errors that arise from writing to a stream +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum WriteError { + /// The peer is no longer accepting data on this stream + /// + /// Carries an application-defined error code. + #[error("sending stopped by peer: error {0}")] + Stopped(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been finished or reset + #[error("closed stream")] + ClosedStream, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for WriteError { + #[inline] + fn from(_: ClosedStream) -> Self { + Self::ClosedStream + } +} + +impl From for WriteError { + fn from(x: StoppedError) -> Self { + match x { + StoppedError::ConnectionLost(e) => Self::ConnectionLost(e), + StoppedError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: WriteError) -> Self { + use WriteError::*; + let kind = match x { + Stopped(_) | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while monitoring for a send stream stop from the peer +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum StoppedError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: StoppedError) -> Self { + use StoppedError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs new file mode 100755 index 00000000000..e0c6f3ef9e0 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -0,0 +1,1871 @@ +#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] + +#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +use rustls::crypto::aws_lc_rs::default_provider; +#[cfg(feature = "rustls-ring")] +use rustls::crypto::ring::default_provider; + +use std::{ + convert::TryInto, + future::Future, + io::{self, IoSliceMut}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}, + pin::{Pin, pin}, + str, + sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; +use crate::{Duration, Instant}; +use bytes::Bytes; +use proto::{ + ConnectionId, RandomConnectionIdGenerator, + crypto::rustls::{QuicClientConfig, QuicServerConfig}, + crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, + transport_parameters::TransportParameters, +}; +use rand::{Rng, SeedableRng, rngs::StdRng}; +use rustls::{ + RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}, +}; +use tokio::time::{sleep, timeout}; +use tokio::{ + join, + runtime::{Builder, Runtime}, +}; +use tracing::instrument::Instrument as _; +use tracing::{error_span, info}; +use tracing_subscriber::EnvFilter; +use udp::RecvMeta; + +use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; + +#[test] +fn handshake_timeout() { + let _guard = subscribe(); + let runtime = rt_threaded(); + let client = { + let _guard = runtime.enter(); + Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap() + }; + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + const IDLE_TIMEOUT: Duration = Duration::from_millis(500); + let mut transport_config = TransportConfig::default(); + transport_config + .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .initial_rtt(Duration::from_millis(10)); + client_config.transport_config(Arc::new(transport_config)); + + let start = Instant::now(); + runtime.block_on(async move { + match client + .connect_with( + client_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1), + "localhost", + ) + .unwrap() + .await + { + Err(crate::ConnectionError::TimedOut) => {} + Err(e) => panic!("unexpected error: {e:?}"), + Ok(_) => panic!("unexpected success"), + } + }); + let dt = start.elapsed(); + assert!(dt > IDLE_TIMEOUT && dt < 2 * IDLE_TIMEOUT); +} + +#[tokio::test] +async fn close_endpoint() { + let _guard = subscribe(); + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let endpoint = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + endpoint + .set_default_client_config(ClientConfig::with_root_certificates(Arc::new(roots)).unwrap()); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + + tokio::spawn(async move { + let _ = conn.await; + }); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + endpoint.close(0u32.into(), &[]); + match conn.await { + Err(crate::ConnectionError::LocallyClosed) => (), + Err(e) => panic!("unexpected error: {e}"), + Ok(_) => { + panic!("unexpected success"); + } + } +} + +#[test] +fn local_addr() { + let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).unwrap(); + let addr = socket.local_addr().unwrap(); + let runtime = rt_basic(); + let ep = { + let _guard = runtime.enter(); + Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + }; + assert_eq!( + addr, + ep.local_addr() + .expect("Could not obtain our local endpoint") + ); +} + +#[test] +fn read_after_close() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + const MSG: &[u8] = b"goodbye!"; + let endpoint2 = endpoint.clone(); + runtime.spawn(async move { + let new_conn = endpoint2 + .accept() + .await + .expect("endpoint") + .await + .expect("connection"); + let mut s = new_conn.open_uni().await.unwrap(); + s.write_all(MSG).await.unwrap(); + s.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = s.stopped().await; + }); + runtime.block_on(async move { + let new_conn = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect"); + sleep(Duration::from_millis(100)).await; + let mut stream = new_conn.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG); + }); +} + +#[test] +fn export_keying_material() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + runtime.block_on(async move { + let outgoing_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect") + } + }); + let incoming_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .accept() + .await + .expect("endpoint") + .await + .expect("connection") + } + }); + let outgoing_conn = outgoing_conn_fut.await.unwrap(); + let incoming_conn = incoming_conn_fut.await.unwrap(); + let mut i_buf = [0u8; 64]; + incoming_conn + .export_keying_material(&mut i_buf, b"asdf", b"qwer") + .unwrap(); + let mut o_buf = [0u8; 64]; + outgoing_conn + .export_keying_material(&mut o_buf, b"asdf", b"qwer") + .unwrap(); + assert_eq!(&i_buf[..], &o_buf[..]); + }); +} + +#[tokio::test] +async fn ip_blocking() { + let _guard = subscribe(); + let endpoint_factory = EndpointFactory::new(); + let client_1 = endpoint_factory.endpoint(); + let client_1_addr = client_1.local_addr().unwrap(); + let client_2 = endpoint_factory.endpoint(); + let server = endpoint_factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let server_task = tokio::spawn(async move { + loop { + let accepting = server.accept().await.unwrap(); + if accepting.remote_address() == client_1_addr { + accepting.refuse(); + } else if accepting.remote_address_validated() { + accepting.await.expect("connection"); + } else { + accepting.retry().unwrap(); + } + } + }); + tokio::join!( + async move { + let e = client_1 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect_err("server should have blocked this"); + assert!( + matches!(e, crate::ConnectionError::ConnectionClosed(_)), + "wrong error" + ); + }, + async move { + client_2 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + } + ); + server_task.abort(); +} + +/// Construct an endpoint suitable for connecting to itself +fn endpoint() -> Endpoint { + EndpointFactory::new().endpoint() +} + +fn endpoint_with_config(transport_config: TransportConfig) -> Endpoint { + EndpointFactory::new().endpoint_with_config(transport_config) +} + +/// Constructs endpoints suitable for connecting to themselves and each other +struct EndpointFactory { + cert: rcgen::CertifiedKey, + endpoint_config: EndpointConfig, +} + +impl EndpointFactory { + fn new() -> Self { + Self { + cert: rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(), + endpoint_config: EndpointConfig::default(), + } + } + + fn endpoint(&self) -> Endpoint { + self.endpoint_with_config(TransportConfig::default()) + } + + fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { + let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); + let transport_config = Arc::new(transport_config); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + server_config.transport_config(transport_config.clone()); + + let mut roots = RootCertStore::empty(); + roots.add(self.cert.cert.der().clone()).unwrap(); + let endpoint = Endpoint::new( + self.endpoint_config.clone(), + Some(server_config), + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + Arc::new(TokioRuntime), + ) + .unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(transport_config); + endpoint.set_default_client_config(client_config); + + endpoint + } +} + +#[derive(Default)] +struct HandshakeBlocker { + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct HandshakeBlockerState { + started: bool, + released: bool, +} + +impl HandshakeBlocker { + fn block(&self) { + let mut state = self.state.lock().unwrap(); + state.started = true; + self.changed.notify_all(); + while !state.released { + state = self.changed.wait(state).unwrap(); + } + } + + fn wait_until_started(&self) { + let state = self.state.lock().unwrap(); + let (state, _) = self + .changed + .wait_timeout_while(state, Duration::from_secs(5), |state| !state.started) + .unwrap(); + assert!(state.started, "timed out waiting for handshake to start"); + } + + fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.released = true; + self.changed.notify_all(); + } +} + +struct HandshakeReleaseGuard(Arc); + +impl Drop for HandshakeReleaseGuard { + fn drop(&mut self) { + self.0.release(); + } +} + +struct BlockingServerConfig { + inner: Arc, + blocker: Arc, +} + +impl ProtoServerConfig for BlockingServerConfig { + fn initial_keys( + &self, + version: u32, + dst_cid: ConnectionId, + ) -> Result { + self.inner.initial_keys(version, dst_cid) + } + + fn retry_tag(&self, version: u32, orig_dst_cid: ConnectionId, packet: &[u8]) -> [u8; 16] { + self.inner.retry_tag(version, orig_dst_cid, packet) + } + + fn start_session( + self: Arc, + version: u32, + params: &TransportParameters, + ) -> Box { + self.blocker.block(); + self.inner.clone().start_session(version, params) + } +} + +/// Wraps a socket so that a test can make its next receive fail, which ends the endpoint driver +#[derive(Debug)] +struct FailingSocket { + inner: Box, + failure: Arc, +} + +#[derive(Debug, Default)] +struct SocketFailure { + armed: AtomicBool, + recv_waker: Mutex>, +} + +impl SocketFailure { + /// Make the next receive fail, waking the endpoint driver so that it observes the failure + fn trigger(&self) { + self.armed.store(true, Ordering::SeqCst); + if let Some(waker) = self.recv_waker.lock().unwrap().as_ref() { + waker.wake_by_ref(); + } + } +} + +impl AsyncUdpSocket for FailingSocket { + fn create_sender(&self) -> Pin> { + self.inner.create_sender() + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + if self.failure.armed.load(Ordering::SeqCst) { + return Poll::Ready(Err(io::Error::other("injected socket failure"))); + } + *self.failure.recv_waker.lock().unwrap() = Some(cx.waker().clone()); + self.inner.poll_recv(cx, bufs, meta) + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } +} + +/// Poll a connection driver before its registration returns, once the endpoint has failed. +#[derive(Debug)] +struct DriverLossRuntime { + failure: Arc, + stopped_on_first_poll: AtomicBool, +} + +impl crate::runtime::Runtime for DriverLossRuntime { + fn new_timer(&self, deadline: Instant) -> Pin> { + TokioRuntime.new_timer(deadline) + } + + fn spawn(&self, mut future: Pin + Send>>) { + if self.failure.armed.load(Ordering::SeqCst) { + let mut cx = Context::from_waker(Waker::noop()); + if future.as_mut().poll(&mut cx).is_ready() { + self.stopped_on_first_poll.store(true, Ordering::SeqCst); + return; + } + } + TokioRuntime.spawn(future); + } + + fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { + TokioRuntime.wrap_udp_socket(socket) + } + + fn now(&self) -> Instant { + TokioRuntime.now() + } +} + +fn localhost_socket() -> Box { + TokioRuntime + .wrap_udp_socket( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + ) + .unwrap() +} + +fn blocking_server_pair( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> (Endpoint, Endpoint, Arc) { + let factory = EndpointFactory::new(); + let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); + let mut server_crypto = rustls::ServerConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![factory.cert.cert.der().clone()], key) + .unwrap(); + server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; + let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + QuicServerConfig::try_from(server_crypto).unwrap(), + )); + let blocker = Arc::new(HandshakeBlocker::default()); + server_config.crypto = Arc::new(BlockingServerConfig { + inner: server_config.crypto.clone(), + blocker: blocker.clone(), + }); + + let server = Endpoint::new_with_abstract_socket( + factory.endpoint_config.clone(), + Some(server_config), + server_socket, + server_runtime, + ) + .unwrap(); + + let mut roots = RootCertStore::empty(); + roots.add(factory.cert.cert.der().clone()).unwrap(); + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.alpn_protocols = vec![client_alpn.to_vec()]; + client.set_default_client_config(ClientConfig::new(Arc::new( + QuicClientConfig::try_from(client_crypto).unwrap(), + ))); + + (client, server, blocker) +} + +async fn wait_for_blocked_handshake(blocker: Arc) { + tokio::task::spawn_blocking(move || blocker.wait_until_started()) + .await + .unwrap(); +} + +async fn accept_with_blocked_start_session( + server: Endpoint, +) -> Result { + let incoming = server.accept().await.unwrap(); + // The test crypto provider intentionally blocks in `start_session`, so run + // accept on the blocking pool rather than parking a Tokio worker thread. + let connecting = tokio::task::spawn_blocking(move || incoming.accept()) + .await + .unwrap()?; + connecting.await +} + +struct BlockedHandshake { + client: Endpoint, + server: Endpoint, + blocker: Arc, + release_guard: HandshakeReleaseGuard, + accept: tokio::task::JoinHandle>, + connect: tokio::task::JoinHandle>, +} + +fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { + blocked_handshake_with_socket(server_alpn, client_alpn, localhost_socket()) +} + +fn blocked_handshake_with_socket( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, +) -> BlockedHandshake { + blocked_handshake_with_runtime( + server_alpn, + client_alpn, + server_socket, + Arc::new(TokioRuntime), + ) +} + +fn blocked_handshake_with_runtime( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> BlockedHandshake { + let (client, server, blocker) = + blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let server_addr = server.local_addr().unwrap(); + + let accept = tokio::spawn({ + let server = server.clone(); + async move { accept_with_blocked_start_session(server).await } + }); + let connect = tokio::spawn({ + let client = client.clone(); + async move { client.connect(server_addr, "localhost").unwrap().await } + }); + + BlockedHandshake { + client, + server, + blocker: blocker.clone(), + release_guard: HandshakeReleaseGuard(blocker), + accept, + connect, + } +} + +async fn await_connection( + task: tokio::task::JoinHandle>, +) -> Result { + timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() +} + +async fn assert_wait_idle_pending(endpoint: &Endpoint) { + assert!( + timeout(Duration::from_millis(50), endpoint.wait_idle()) + .await + .is_err() + ); +} + +async fn wait_idle(endpoint: &Endpoint) { + timeout(Duration::from_secs(5), endpoint.wait_idle()) + .await + .unwrap(); +} + +/// A parked `wait_idle` must not hang when the endpoint driver is dropped. +/// +/// `EndpointDriver::drop` clears the connection senders, which is what makes +/// the endpoint idle, but it wakes only `incoming`. A caller already parked on +/// `idle` is therefore never woken, and with the driver gone nothing else can +/// wake it. `EndpointInner::accept` already guards this same hazard for accepts +/// that complete after driver loss. +#[test] +fn wait_idle_returns_when_the_driver_is_dropped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + + // The endpoint under test owns its runtime, so shutting that runtime down + // drops its driver. + let endpoint_runtime = rt_threaded(); + let endpoint_handle = endpoint_runtime.handle().clone(); + let endpoint = { + let _guard = endpoint_runtime.enter(); + factory.endpoint() + }; + let endpoint_addr = endpoint.local_addr().unwrap(); + + // A peer on a second runtime, so a live connection exists to wait for. + let peer_runtime = rt_threaded(); + let peer = { + let _guard = peer_runtime.enter(); + factory.endpoint() + }; + let peer_handle = peer_runtime.handle().clone(); + // Accept concurrently: the client's handshake cannot complete until the + // server drives its half, and the accepted handle must stay alive or the + // connection would close and the endpoint become idle before the assertion. + let accepted = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(async { + let incoming = endpoint.accept().await.expect("incoming"); + incoming.await.expect("accept") + }) + }) + }; + let connection = peer_handle.block_on(async { + peer.connect(endpoint_addr, "localhost") + .expect("connect") + .await + .expect("connection") + }); + let _server_connection = accepted.join().expect("accept thread"); + + // Park `wait_idle`; it blocks while that connection is live. + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let waiter = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(endpoint.wait_idle()); + let _ = done_tx.send(()); + }) + }; + + // Otherwise a pass below would be vacuous. + assert!( + done_rx.recv_timeout(Duration::from_millis(200)).is_err(), + "wait_idle returned before the driver was dropped" + ); + + // Dropping the runtime drops the endpoint driver. + endpoint_runtime.shutdown_background(); + + let woke = done_rx.recv_timeout(Duration::from_secs(10)).is_ok(); + drop(connection); + drop(peer); + drop(peer_runtime); + assert!( + woke, + "wait_idle never returned after the endpoint driver was dropped" + ); + let _ = waiter.join(); +} + +async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { + timeout(Duration::from_secs(5), conn.closed()) + .await + .unwrap() +} + +#[tokio::test] +async fn zero_rtt() { + let _guard = subscribe(); + let endpoint = endpoint(); + + const MSG0: &[u8] = b"zero"; + const MSG1: &[u8] = b"one"; + let endpoint2 = endpoint.clone(); + let accept_connection = move |expect_0rtt| { + let endpoint2 = endpoint2.clone(); + async move { + let incoming = endpoint2.accept().await.unwrap().accept().unwrap(); + let connection = incoming.into_0rtt().unwrap_or_else(|_| unreachable!()); + let c = connection.clone(); + tokio::spawn(async move { + while let Ok(mut x) = c.accept_uni().await { + assert_eq!(x.is_0rtt(), expect_0rtt); + let msg = x.read_to_end(usize::MAX).await.unwrap(); + assert_eq!(msg, MSG0); + } + }); + info!("sending 0.5-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG0).await.expect("write"); + s.finish().unwrap(); + connection.authenticated().await.expect("connected"); + info!("sending 1-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG1).await.expect("write"); + // The peer might close the connection before ACKing + let _ = s.finish(); + } + }; + + tokio::spawn(accept_connection(false)); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .expect_err("0-RTT succeeded without keys") + .await + .expect("connect"); + + { + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + // Read a 1-RTT message to ensure the handshake completes fully, allowing the server's + // NewSessionTicket frame to be received. + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG1); + drop(connection); + } + + info!("initial connection complete"); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .unwrap_or_else(|_| panic!("missing 0-RTT keys")); + + // Send something before the handshake can make progress, thereby forcing 0-RTT + let mut stream_0rtt = connection.open_uni().await.expect("0-RTT open uni"); + info!("sending 0-RTT"); + stream_0rtt.write_all(MSG0).await.expect("0-RTT write"); + stream_0rtt.finish().unwrap(); + + tokio::spawn(accept_connection(true)); + + // Receive 0.5-RTT + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + connection.authenticated().await.expect("connected"); + + // Ensure 0-RTT was accepted + stream_0rtt.stopped().await.expect("0-RTT stopped"); + + // Allow the connection to close + drop((stream_0rtt, stream, connection)); + + endpoint.wait_idle().await; +} + +#[test] +#[cfg_attr( + any(target_os = "solaris", target_os = "illumos"), + ignore = "Fails on Solaris and Illumos" +)] +fn echo_v6() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Sometimes hangs in poll() on Solaris")] +fn echo_v4() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn echo_dualstack() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_receive_window() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(100 * 1024 * 1024), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_stream_receive_window() { + // Note that there is no point in running this with too many streams, + // since the window is only active within a stream. + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 2, + stream_size: 250 * 1024 + 11, + receive_window: Some(100 * 1024 * 1024), + stream_receive_window: Some(37), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_both_windows() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(37), + }); +} + +fn run_echo(args: EchoArgs) { + let _guard = subscribe(); + let runtime = rt_basic(); + let handle = { + // Use small receive windows + let mut transport_config = TransportConfig::default(); + if let Some(receive_window) = args.receive_window { + transport_config.receive_window(receive_window.try_into().unwrap()); + } + if let Some(stream_receive_window) = args.stream_receive_window { + transport_config.stream_receive_window(stream_receive_window.try_into().unwrap()); + } + transport_config.max_concurrent_bidi_streams(1_u8.into()); + transport_config.max_concurrent_uni_streams(1_u8.into()); + let transport_config = Arc::new(transport_config); + + // We don't use the `endpoint` helper here because we want two different endpoints with + // different addresses. + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + + server_config.transport = transport_config.clone(); + let server_sock = UdpSocket::bind(args.server_addr).unwrap(); + let server_addr = server_sock.local_addr().unwrap(); + let server = { + let _guard = runtime.enter(); + let _guard = error_span!("server").entered(); + Endpoint::new( + Default::default(), + Some(server_config), + server_sock, + Arc::new(TokioRuntime), + ) + .unwrap() + }; + + let mut roots = RootCertStore::empty(); + roots.add(cert).unwrap(); + let mut client_crypto = + rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.key_log = Arc::new(rustls::KeyLogFile::new()); + + let client = { + let _guard = runtime.enter(); + let _guard = error_span!("client").entered(); + Endpoint::client(args.client_addr).unwrap() + }; + let mut client_config = + ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())); + client_config.transport_config(transport_config); + client.set_default_client_config(client_config); + + let handle = runtime.spawn(async move { + let incoming = server.accept().await.unwrap(); + + // Note for anyone modifying the platform support in this test: + // If `local_ip` gets available on additional platforms - which + // requires modifying this test - please update the list of supported + // platforms in the doc comment of `quinn_udp::RecvMeta::dst_ip`. + if cfg!(target_os = "linux") + || cfg!(target_os = "android") + || cfg!(target_os = "freebsd") + || cfg!(target_os = "openbsd") + || cfg!(target_os = "netbsd") + || cfg!(target_os = "macos") + || cfg!(target_os = "windows") + { + let local_ip = incoming.local_ip().expect("Local IP must be available"); + assert!(local_ip.is_loopback()); + } else { + assert_eq!(None, incoming.local_ip()); + } + + let new_conn = incoming.await.unwrap(); + tokio::spawn(async move { + while let Ok(stream) = new_conn.accept_bi().await { + tokio::spawn(echo(stream)); + } + }); + server.wait_idle().await; + }); + + info!("connecting from {} to {}", args.client_addr, server_addr); + runtime.block_on( + async move { + let new_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + + /// This is just an arbitrary number to generate deterministic test data + const SEED: u64 = 0x12345678; + + for i in 0..args.nr_streams { + println!("Opening stream {i}"); + let (mut send, mut recv) = new_conn.open_bi().await.expect("stream open"); + let msg = gen_data(args.stream_size, SEED); + + let send_task = async { + send.write_all(&msg).await.expect("write"); + send.finish().unwrap(); + }; + let recv_task = async { recv.read_to_end(usize::MAX).await.expect("read") }; + + let (_, data) = tokio::join!(send_task, recv_task); + + assert_eq!(data[..], msg[..], "Data mismatch"); + } + new_conn.close(0u32.into(), b"done"); + client.wait_idle().await; + } + .instrument(error_span!("client")), + ); + handle + }; + runtime.block_on(handle).unwrap(); +} + +struct EchoArgs { + client_addr: SocketAddr, + server_addr: SocketAddr, + nr_streams: usize, + stream_size: usize, + receive_window: Option, + stream_receive_window: Option, +} + +async fn echo((mut send, mut recv): (SendStream, RecvStream)) { + loop { + // These are 32 buffers, for reading approximately 32kB at once + #[rustfmt::skip] + let mut bufs = [ + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + ]; + + match recv.read_chunks(&mut bufs).await.expect("read chunks") { + Some(n) => { + send.write_all_chunks(&mut bufs[..n]) + .await + .expect("write chunks"); + } + None => break, + } + } + + let _ = send.finish(); +} + +fn gen_data(size: usize, seed: u64) -> Vec { + let mut rng: StdRng = SeedableRng::seed_from_u64(seed); + let mut buf = vec![0; size]; + rng.fill_bytes(&mut buf); + buf +} + +fn subscribe() -> tracing::subscriber::DefaultGuard { + let sub = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(|| TestWriter) + .finish(); + tracing::subscriber::set_default(sub) +} + +struct TestWriter; + +impl io::Write for TestWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + print!( + "{}", + str::from_utf8(buf).expect("tried to log invalid UTF-8") + ); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + io::stdout().flush() + } +} + +fn rt_basic() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +fn rt_threaded() -> Runtime { + Builder::new_multi_thread().enable_all().build().unwrap() +} + +#[tokio::test] +async fn retry_after_rebind() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + server + .rebind(UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap()) + .unwrap(); + + timeout(Duration::from_secs(5), async { + let connecting = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + let incoming = server.accept().await.unwrap(); + assert!(!incoming.remote_address_validated()); + incoming.retry().unwrap(); + + let incoming = server.accept().await.unwrap(); + assert!(incoming.remote_address_validated()); + let (accepted, connected) = join!(incoming, connecting); + let _server_connection = accepted.unwrap(); + let _client_connection = connected.unwrap(); + server.close(0u32.into(), b"done"); + client.close(0u32.into(), b"done"); + }) + .await + .expect("retried handshake must complete using the rebound socket"); +} + +#[tokio::test] +async fn rebind_recv() { + let _guard = subscribe(); + + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + + let mut roots = RootCertStore::empty(); + roots.add(cert.clone()).unwrap(); + + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(Arc::new({ + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + cfg + })); + client.set_default_client_config(client_config); + + let server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + let server = { + let _guard = tracing::error_span!("server").entered(); + Endpoint::server( + server_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + ) + .unwrap() + }; + let server_addr = server.local_addr().unwrap(); + + const MSG: &[u8; 5] = b"hello"; + + let write_send = Arc::new(tokio::sync::Notify::new()); + let write_recv = write_send.clone(); + let connected_send = Arc::new(tokio::sync::Notify::new()); + let connected_recv = connected_send.clone(); + let server = tokio::spawn(async move { + let connection = server.accept().await.unwrap().await.unwrap(); + info!("got conn"); + connected_send.notify_one(); + write_recv.notified().await; + let mut stream = connection.open_uni().await.unwrap(); + stream.write_all(MSG).await.unwrap(); + stream.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = stream.stopped().await; + }); + + let connection = { + let _guard = tracing::error_span!("client").entered(); + client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap() + }; + info!("connected"); + connected_recv.notified().await; + client + .rebind(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) + .unwrap(); + info!("rebound"); + write_send.notify_one(); + let mut stream = connection.accept_uni().await.unwrap(); + assert_eq!(stream.read_to_end(MSG.len()).await.unwrap(), MSG); + server.await.unwrap(); +} + +/// Verify endpoint behavior while a connection is between `start_accept` and `finish_accept`: +/// `open_connections` must be 0, and `wait_idle` must not complete while `pending_accepts > 0`. +/// After the accept completes and the connection is closed, `wait_idle` must resolve. +#[tokio::test] +async fn split_accept_wait_idle_and_open_connections() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + let client_conn = await_connection(connect).await.unwrap(); + let server_conn = await_connection(accept).await.unwrap(); + assert_eq!(server.open_connections(), 1); + + client_conn.close(0u32.into(), b"done"); + let _ = wait_closed(&client_conn).await; + let _ = wait_closed(&server_conn).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// A failed accept never becomes a connection, so no Drained event will wake idle waiters. +#[tokio::test] +async fn split_accept_failure_wakes_idle_waiter() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"server", b"client"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + let error = await_connection(accept).await.unwrap_err(); + assert!(matches!( + error, + crate::ConnectionError::TransportError(error) + if error.code == proto::TransportErrorCode::crypto(0x78) + )); + // Repolling wait_idle alone would hide a missing notification now that the endpoint is + // idle. Verify that the waiter registered before the failure was actually woken. + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + assert_eq!(server.open_connections(), 0); + + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +/// Verify that calling `Endpoint::close` while a connection is in the `Accepting` state +/// (between `start_accept` and `finish_accept`) produces `LocallyClosed` on the server side +/// and `ConnectionClosed` on the client side, and that `wait_idle` resolves afterward. +#[tokio::test] +async fn split_accept_close_during_pending_accept() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + match await_connection(accept).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + } + Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + } + + match await_connection(connect).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + Err(err) => { + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + } + + wait_idle(&server).await; + wait_idle(&client).await; + assert_eq!(server.open_connections(), 0); +} + +/// The endpoint driver may exit while an accept runs without the lock. The connection registered +/// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly +/// and `wait_idle` does not wait on a connection nobody drives. +#[tokio::test] +async fn split_accept_driver_lost_during_handshake() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_socket(b"test", b"test", socket); + + wait_for_blocked_handshake(blocker.clone()).await; + // Fail the server socket, which ends the endpoint driver while the handshake is blocked. + // `accept` yields `None` once the driver is gone. + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + // Register an idle waiter while the accept is still pending. Nothing but the accept's + // completion can wake it now that the driver is gone. + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + // Without the fix this waits for the handshake timeout, well past `await_connection`'s limit. + assert!(await_connection(accept).await.is_err()); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn split_accept_driver_lost_before_connection_spawn() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let runtime = Arc::new(DriverLossRuntime { + failure: failure.clone(), + stopped_on_first_poll: AtomicBool::new(false), + }); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); + + wait_for_blocked_handshake(blocker.clone()).await; + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + blocker.release(); + // The runtime polls the connection driver before registration returns. It must observe the + // closed event channel immediately, before transmitting any handshake packets. + assert!(await_connection(accept).await.is_err()); + assert!( + runtime.stopped_on_first_poll.load(Ordering::SeqCst), + "connection driver ran before its event channel was closed" + ); + wait_idle(&server).await; + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn remote_address_monitoring() { + let _guard = subscribe(); + + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let client = factory.endpoint(); + + let trigger_migration = Arc::new(tokio::sync::Notify::new()); + let trigger_migration_clone = trigger_migration.clone(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let old_remote = conn.remote_address(); + let mut addr_watch = conn.watch_remote_address(); + + trigger_migration_clone.notify_one(); + + let addr_a = addr_watch.wait().await.unwrap(); + assert_eq!(addr_a.port(), 23_001); + assert_ne!(old_remote, addr_a); + + trigger_migration_clone.notify_one(); + + let addr_b = addr_watch.wait().await.unwrap(); + assert_eq!(addr_b.port(), 23_003); + assert_ne!(addr_a, addr_b); + }); + + let client_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + + let mut port = 23_000; + let mut migrate = async || { + port += 1; + client + .rebind( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)).unwrap(), + ) + .unwrap(); + + // send data from the new socket to trigger migration on the server + let mut stream = client_conn.open_uni().await.unwrap(); + stream.write_all(b"hello").await.unwrap(); + stream.finish().unwrap(); + }; + + trigger_migration.notified().await; + migrate().await; + trigger_migration.notified().await; + migrate().await; + migrate().await; + + server_task.await.unwrap(); + client_conn.close(0u32.into(), b"done"); +} + +#[tokio::test] +async fn stream_id_flow_control() { + let _guard = subscribe(); + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + let endpoint = endpoint_with_config(cfg); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + // If `open_uni` doesn't get unblocked when the previous stream is dropped, this will time out. + tokio::join!( + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + server.accept_uni().await.unwrap(); + server.accept_uni().await.unwrap(); + } + ); +} + +#[tokio::test] +async fn two_datagram_readers() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + let done = tokio::sync::Notify::new(); + let (a, b, ()) = tokio::join!( + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + server.send_datagram(b"one"[..].into()).unwrap(); + done.notified().await; + server.send_datagram_wait(b"two"[..].into()).await.unwrap(); + } + ); + assert!(*a == *b"one" || *b == *b"one"); + assert!(*a == *b"two" || *b == *b"two"); +} + +#[tokio::test] +async fn multiple_conns_with_zero_length_cids() { + let _guard = subscribe(); + let mut factory = EndpointFactory::new(); + factory + .endpoint_config + .cid_generator(Arc::new(|| Box::new(RandomConnectionIdGenerator::new(0)))); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client1 = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + let client2 = { + let _guard = error_span!("client2").entered(); + factory.endpoint() + }; + + let client1 = async move { + let conn = client1 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client1")); + let client2 = async move { + let conn = client2 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client2")); + let server = async move { + let client1 = server.accept().await.unwrap().await.unwrap(); + let client2 = server.accept().await.unwrap().await.unwrap(); + // Both connections are now concurrently live. + client1.close(42u32.into(), &[]); + client2.close(42u32.into(), &[]); + } + .instrument(error_span!("server")); + tokio::join!(client1, client2, server); +} + +#[tokio::test] +async fn stream_stopped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + + let client = async move { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + let stopped1 = stream.stopped(); + let stopped2 = stream.stopped(); + let stopped3 = stream.stopped(); + + stream.write_all(b"hi").await.unwrap(); + // spawn one of the futures into a task + let stopped1 = tokio::task::spawn(stopped1); + // verify that both futures resolved + let (stopped1, stopped2) = tokio::join!(stopped1, stopped2); + assert!(matches!(stopped1, Ok(Ok(Some(val))) if val == 42u32.into())); + assert!(matches!(stopped2, Ok(Some(val)) if val == 42u32.into())); + // drop the stream + drop(stream); + // verify that a future also resolves after dropping the stream + let stopped3 = stopped3.await; + assert_eq!(stopped3, Ok(Some(42u32.into()))); + }; + let client = timeout(Duration::from_millis(100), client).instrument(error_span!("client")); + let server = async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + let mut buf = [0u8; 2]; + stream.read_exact(&mut buf).await.unwrap(); + stream.stop(42u32.into()).unwrap(); + conn + } + .instrument(error_span!("server")); + let (client, conn) = tokio::join!(client, server); + client.expect("timeout"); + drop(conn); +} + +#[tokio::test] +async fn stream_stopped_2() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (conn, _server_conn) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let send_stream = conn.open_uni().await.unwrap(); + let stopped = timeout(Duration::from_millis(100), send_stream.stopped()) + .instrument(error_span!("stopped")); + tokio::pin!(stopped); + // poll the future once so that the waker is registered. + tokio::select! { + biased; + _x = &mut stopped => {}, + _x = std::future::ready(()) => {} + } + // drop the send stream + drop(send_stream); + // make sure the stopped future still resolves + let res = stopped.await; + assert_eq!(res, Ok(Ok(None))); +} + +#[tokio::test] +async fn stream_drop_removes_blocked_reader() { + let _guard = subscribe(); + + for drop_stream in [false, true] { + let endpoint_factory = EndpointFactory::new(); + let server = endpoint_factory.endpoint(); + let server_address = server.local_addr().unwrap(); + let client = endpoint_factory.endpoint(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + + // read "hello" + let mut buf = [0u8; 5]; + stream.read_exact(&mut buf).await.unwrap(); + + let (waker, wake_counter) = new_count_waker(); + let mut cx = Context::from_waker(&waker); + // do a blocking read which will add the stream in conn.blocked_readers + { + let mut buf = [0u8; 64]; + let read_fut = stream.read(&mut buf); + tokio::pin!(read_fut); + assert!(matches!(read_fut.as_mut().poll(&mut cx), Poll::Pending)); + } + + if !drop_stream { + assert_eq!(wake_counter.wakes(), 0); + // We have a blocked reader, closing the connection should wake it. We use this as + // a proxy to assert that the stream is in conn.blocked_readers. + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 1); + } else { + // dropping the stream should remove it from conn.blocked_readers, so we don't + // expect any wakeups + drop(stream); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + } + }); + + let conn = client + .connect(server_address, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + // need to send some data to actually start the stream + stream.write_all(b"hello").await.unwrap(); + + server_task.await.unwrap(); + } +} + +/// Test that dropping a `RecvStream` after cancelling a read and then +/// explicitly `stop`ing it doesn't panic. +#[tokio::test] +async fn recv_stream_cancel_stop_drop() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client").entered(); + factory.endpoint() + }; + let recv_dropped = tokio::sync::SetOnce::new(); + join!( + async { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut recv = conn.accept_uni().await.unwrap(); + // Create a future to read from the stream, poll it once, then immediately drop it + { + let fut = pin!(recv.read_to_end(usize::MAX)); + let mut cx = Context::from_waker(Waker::noop()); + assert!(fut.poll(&mut cx).is_pending()); + } + recv_dropped.set(()).unwrap(); + recv.stop(0u32.into()).unwrap(); + }, + async { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut send = conn.open_uni().await.unwrap(); + _ = send.write_all(b"hello").await; + // Don't drop (finish) the send stream until the read has been + // cancelled by the server, ensuring that read_to_end can't complete + // immediately. + recv_dropped.wait().await; + }, + ); +} + +#[tokio::test(start_paused = true)] +async fn dropped_endpoint_cleans_up() { + let _guard = subscribe(); + + let mut endpoint_factory = EndpointFactory::new(); + let cid_generator = Arc::new(|| -> Box { + Box::::default() + }); + endpoint_factory + .endpoint_config + .cid_generator(cid_generator.clone()); + let endpoint = endpoint_factory.endpoint(); + drop(endpoint_factory); + assert_eq!(Arc::strong_count(&cid_generator), 2); + drop(endpoint); + // Let the driver task run; paused runtimes are guaranteed to drain pending work on sleep. + sleep(Duration::from_millis(1)).await; + assert_eq!(Arc::strong_count(&cid_generator), 1); +} + +#[tokio::test] +async fn dropped_connection_cleans_up() { + let _guard = subscribe(); + let endpoint = endpoint(); + join!( + async { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .unwrap() + }, + async { endpoint.accept().await.unwrap().await.unwrap() } + ); + endpoint.wait_idle().await; +} + +#[derive(Default)] +struct WakeCounter { + wakes: AtomicUsize, +} + +impl WakeCounter { + fn wakes(&self) -> usize { + self.wakes.load(Ordering::SeqCst) + } +} + +fn new_count_waker() -> (Waker, Arc) { + // instance of WakeCounter + let counter = Arc::new(WakeCounter::default()); + + // convert + let waker = unsafe { Waker::from_raw(raw_waker(counter.clone())) }; + (waker, counter) +} + +fn raw_waker(counter: Arc) -> RawWaker { + // Store an Arc behind the raw pointer. + let ptr = Arc::into_raw(counter) as *const (); + RawWaker::new(ptr, &VTABLE) +} + +static VTABLE: RawWakerVTable = + RawWakerVTable::new(clone_waker, wake_waker, wake_by_ref_waker, drop_waker); + +unsafe fn clone_waker(data: *const ()) -> RawWaker { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + let cloned = arc.clone(); + std::mem::forget(arc); + raw_waker(cloned) +} + +unsafe fn wake_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + // arc drops here +} + +unsafe fn wake_by_ref_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + std::mem::forget(arc); +} + +unsafe fn drop_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + drop(unsafe { Arc::::from_raw(data as *const WakeCounter) }); +} diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs new file mode 100644 index 00000000000..c3c3d3551d1 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -0,0 +1,224 @@ +use crate::{Duration, Instant}; + +/// Limits the amount of time spent on a certain type of work in a cycle +/// +/// The limiter works dynamically: For a sampled subset of cycles it measures +/// the time that is approximately required for fulfilling 1 work item, and +/// calculates the amount of allowed work items per cycle. +/// The estimates are smoothed over all cycles where the exact duration is measured. +/// +/// In cycles where no measurement is performed the previously determined work limit +/// is used. +/// +/// For the limiter the exact definition of a work item does not matter. +/// It could for example track the amount of transmitted bytes per cycle, +/// or the amount of transmitted datagrams per cycle. +/// It will however work best if the required time to complete a work item is +/// constant. +#[derive(Debug)] +pub(crate) struct WorkLimiter { + /// Whether to measure the required work time, or to use the previous estimates + mode: Mode, + /// The current cycle number + cycle: u16, + /// The time the cycle started - only used in measurement mode + start_time: Option, + /// How many work items have been completed in the cycle + completed: usize, + /// The amount of work items which are allowed for a cycle + allowed: usize, + /// The desired cycle time + desired_cycle_time: Duration, + /// The estimated and smoothed time per work item in nanoseconds + smoothed_time_per_work_item_nanos: f64, +} + +impl WorkLimiter { + pub(crate) fn new(desired_cycle_time: Duration) -> Self { + Self { + mode: Mode::Measure, + cycle: 0, + start_time: None, + completed: 0, + allowed: 0, + desired_cycle_time, + smoothed_time_per_work_item_nanos: 0.0, + } + } + + /// Starts one work cycle + pub(crate) fn start_cycle(&mut self, now: impl Fn() -> Instant) { + self.completed = 0; + if let Mode::Measure = self.mode { + self.start_time = Some(now()); + } + } + + /// Returns whether more work can be performed inside the `desired_cycle_time` + /// + /// Requires that previous work was tracked using `record_work`. + pub(crate) fn allow_work(&mut self, now: impl Fn() -> Instant) -> bool { + match self.mode { + Mode::Measure => (now() - self.start_time.unwrap()) < self.desired_cycle_time, + Mode::HistoricData => self.completed < self.allowed, + } + } + + /// Records that `work` additional work items have been completed inside the cycle + /// + /// Must be called between `start_cycle` and `finish_cycle`. + pub(crate) fn record_work(&mut self, work: usize) { + self.completed += work; + } + + /// Finishes one work cycle + /// + /// For cycles where the exact duration is measured this will update the estimates + /// for the time per work item and the limit of allowed work items per cycle. + /// The estimate is updated using the same exponential averaging (smoothing) + /// mechanism which is used for determining QUIC path rtts: The last value is + /// weighted by 1/8, and the previous average by 7/8. + pub(crate) fn finish_cycle(&mut self, now: impl Fn() -> Instant) { + // If no work was done in the cycle drop the measurement, it won't be useful + if self.completed == 0 { + return; + } + + if let Mode::Measure = self.mode { + let elapsed = now() - self.start_time.unwrap(); + + let time_per_work_item_nanos = (elapsed.as_nanos()) as f64 / self.completed as f64; + + // Calculate the time per work item. We set this to at least 1ns to avoid + // dividing by 0 when calculating the allowed amount of work items. + self.smoothed_time_per_work_item_nanos = if self.allowed == 0 { + // Initial estimate + time_per_work_item_nanos + } else { + // Smoothed estimate + (7.0 * self.smoothed_time_per_work_item_nanos + time_per_work_item_nanos) / 8.0 + } + .max(1.0); + + // Allow at least 1 work item in order to make progress + self.allowed = (((self.desired_cycle_time.as_nanos()) as f64 + / self.smoothed_time_per_work_item_nanos) as usize) + .max(1); + self.start_time = None; + } + + self.cycle = self.cycle.wrapping_add(1); + self.mode = match self.cycle % SAMPLING_INTERVAL { + 0 => Mode::Measure, + _ => Mode::HistoricData, + }; + } +} + +/// We take a measurement sample once every `SAMPLING_INTERVAL` cycles +const SAMPLING_INTERVAL: u16 = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Measure, + HistoricData, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[test] + fn limit_work() { + const CYCLE_TIME: Duration = Duration::from_millis(500); + const BATCH_WORK_ITEMS: usize = 12; + const BATCH_TIME: Duration = Duration::from_millis(100); + + const EXPECTED_INITIAL_BATCHES: usize = + (CYCLE_TIME.as_nanos() / BATCH_TIME.as_nanos()) as usize; + const EXPECTED_ALLOWED_WORK_ITEMS: usize = EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS; + + let mut limiter = WorkLimiter::new(CYCLE_TIME); + reset_time(); + + // The initial cycle is measuring + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, EXPECTED_ALLOWED_WORK_ITEMS); + let initial_time_per_work_item = limiter.smoothed_time_per_work_item_nanos; + + // The next cycles are using historic data + const BATCH_SIZES: [usize; 4] = [1, 2, 3, 5]; + for &batch_size in &BATCH_SIZES { + limiter.start_cycle(get_time); + let mut allowed_work = 0; + while limiter.allow_work(get_time) { + limiter.record_work(batch_size); + allowed_work += batch_size; + } + limiter.finish_cycle(get_time); + + assert_eq!(allowed_work, EXPECTED_ALLOWED_WORK_ITEMS); + } + + // After `SAMPLING_INTERVAL`, we get into measurement mode again + for _ in 0..(SAMPLING_INTERVAL as usize - BATCH_SIZES.len() - 1) { + limiter.start_cycle(get_time); + limiter.record_work(1); + limiter.finish_cycle(get_time); + } + + // We now do more work per cycle, and expect the estimate of allowed + // work items to go up + const BATCH_WORK_ITEMS_2: usize = 96; + const TIME_PER_WORK_ITEMS_2_NANOS: f64 = + CYCLE_TIME.as_nanos() as f64 / (EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS_2) as f64; + + let expected_updated_time_per_work_item = + (initial_time_per_work_item * 7.0 + TIME_PER_WORK_ITEMS_2_NANOS) / 8.0; + let expected_updated_allowed_work_items = + (CYCLE_TIME.as_nanos() as f64 / expected_updated_time_per_work_item) as usize; + + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS_2); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, expected_updated_allowed_work_items); + } + + thread_local! { + /// Mocked time + pub static TIME: RefCell = RefCell::new(Instant::now()); + } + + fn reset_time() { + TIME.with(|t| { + *t.borrow_mut() = Instant::now(); + }) + } + + fn get_time() -> Instant { + TIME.with(|t| *t.borrow()) + } + + fn advance_time(duration: Duration) { + TIME.with(|t| { + *t.borrow_mut() += duration; + }) + } +} diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs new file mode 100644 index 00000000000..c3dcee477b7 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -0,0 +1,378 @@ +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use crate::proto::ConnectRequest; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use noq::crypto::rustls::QuicClientConfig; +use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; +use tokio::net::lookup_host; +use url::Host; + +use crate::crypto; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::ALPN; +use crate::{ClientError, Session}; + +/// Congestion control algorithm to use for the connection. +/// +/// Different algorithms make different tradeoffs between throughput and latency. +pub enum CongestionControl { + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Client] using sane defaults. +/// +/// This is optional; advanced users may use [Client::new] directly. +#[derive(Clone)] +pub struct ClientBuilder { + provider: crypto::Provider, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ClientBuilder { + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self + .builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates( + self, + certs: Vec, + ) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes( + self, + hashes: Vec>, + ) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); + + let mut transport = noq::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ClientBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Builder for dangerous TLS configuration options. +/// +/// This builder provides access to potentially insecure TLS configurations. +/// These options should only be used when you understand the security implications, +/// such as in local development or over a secure VPN connection. +pub struct DangerousClientBuilder { + inner: ClientBuilder, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl DangerousClientBuilder { + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } +} + +/// A client for connecting to a WebTransport server. +#[derive(Clone, Debug)] +pub struct Client { + endpoint: noq::Endpoint, + config: noq::ClientConfig, +} + +impl Client { + /// Manually create a client via a Noq endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect( + &self, + request: impl Into, + ) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self + .endpoint + .connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for Client { + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } +} + +#[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] +#[derive(Debug)] +struct ServerFingerprints { + provider: crypto::Provider, + fingerprints: Vec>, +} + +impl ServerCertVerifier for ServerFingerprints { + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider + .signature_verification_algorithms + .supported_schemes() + } +} + +#[derive(Debug)] +pub struct NoCertificateVerification(Arc); + +impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs new file mode 100644 index 00000000000..4307355d3f9 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -0,0 +1,158 @@ +use std::ops::Deref; + +use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum ConnectError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), + + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), + + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), +} + +/// An HTTP/3 CONNECT request/response for establishing a WebTransport session. +pub struct Connecting { + // The request that was sent by the client. + pub request: ConnectRequest, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connecting { + pub async fn accept(conn: &noq::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { + request, + send, + recv, + }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond( + mut self, + response: impl Into, + ) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol { + if !self.request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } +} + +impl Deref for Connecting { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.request + } +} + +pub struct Connected { + // The request that was sent by the client. + pub request: ConnectRequest, + + // The response sent by the server. + pub response: ConnectResponse, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connected { + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol { + if !request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. + // We don't use the noq::VarInt because that would mean a noq dependency in + // web-transport-proto + let stream_id = noq::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } +} diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs new file mode 100644 index 00000000000..1b8bb8ecf11 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -0,0 +1,67 @@ +//! Simple crypto provider utilities for rustls. +//! +//! This module provides helper functions for working with rustls crypto providers, +//! supporting both ring and aws-lc-rs backends. + +use std::sync::Arc; + +use rustls::crypto::hash::{self, HashAlgorithm}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::CertificateDer; + +/// A shared reference to a crypto provider. +pub type Provider = Arc; + +/// Returns the default crypto provider. +/// +/// This function checks for a process-wide default provider first, +/// then falls back to feature-enabled providers (aws-lc-rs or ring). +/// +/// # Panics +/// +/// Panics if no provider is available. Either call `CryptoProvider::set_default()` +/// or enable exactly one of the `ring` or `aws-lc-rs` features. +pub fn default_provider() -> Provider { + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } + + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!( + "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." + ); + } +} + +/// Computes the SHA-256 hash of a certificate using the provided crypto provider. +/// +/// # Panics +/// +/// Panics if the provider doesn't expose a SHA-256 hash algorithm. +pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } + + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); +} diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs new file mode 100644 index 00000000000..654260fc4ce --- /dev/null +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -0,0 +1,405 @@ +use std::sync::{Arc, OnceLock}; + +use thiserror::Error; + +use crate::{ConnectError, SettingsError}; + +/// An error returned when connecting to a WebTransport endpoint. +#[derive(Error, Debug, Clone)] +pub enum ClientError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error: {0}")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write: {0}")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read: {0}")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), + + #[error("quic error: {0}")] + NoqError(#[from] noq::ConnectError), + + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// WebTransport errors. +#[derive(Clone, Error, Debug)] +pub enum SessionError { + #[error("connection error: {0}")] + ConnectionError(noq::ConnectionError), + + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), + + #[error("send datagram error: {0}")] + SendDatagramError(#[from] noq::SendDatagramError), +} + +impl From for SessionError { + fn from(e: noq::ConnectionError) -> Self { + match &e { + noq::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } +} + +/// Why a session closed, shared by the session and its streams. The first close wins. +/// +/// It also knows whether the session is raw QUIC, which decides how its close and stream +/// codes are encoded. +#[derive(Debug)] +pub(crate) struct CloseReason { + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, +} + +impl CloseReason { + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { + if self.raw { + return code.into(); + } + noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { + Some(conn) + } + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + Err(_) => err, + } + } + _ => err, + } + } +} + +/// An error that can occur when reading/writing the WebTransport stream header. +#[derive(Clone, Error, Debug)] +pub enum WebTransportError { + #[error("closed: code={0} reason={1}")] + Closed(u32, String), + + #[error("unknown session")] + UnknownSession, + + #[error("read error: {0}")] + ReadError(#[from] noq::ReadExactError), + + #[error("write error: {0}")] + WriteError(#[from] noq::WriteError), +} + +/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +#[derive(Clone, Error, Debug)] +pub enum WriteError { + #[error("STOP_SENDING: {0}")] + Stopped(u32), + + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(noq::VarInt), + + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("stream closed")] + ClosedStream, +} + +impl From for WriteError { + fn from(e: noq::WriteError) -> Self { + match e { + noq::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + noq::WriteError::ClosedStream => WriteError::ClosedStream, + noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadError { + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("RESET_STREAM: {0}")] + Reset(u32), + + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(noq::VarInt), + + #[error("stream already closed")] + ClosedStream, +} + +impl From for ReadError { + fn from(value: noq::ReadError) -> Self { + match value { + noq::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + noq::ReadError::ClosedStream => ReadError::ClosedStream, + noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadExactError { + #[error("finished early")] + FinishedEarly(usize), + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadExactError { + fn from(e: noq::ReadExactError) -> Self { + match e { + noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } +} + +/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadToEndError { + #[error("too long")] + TooLong, + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadToEndError { + fn from(e: noq::ReadToEndError) -> Self { + match e { + noq::ReadToEndError::TooLong => ReadToEndError::TooLong, + noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } +} + +/// An error indicating the stream was already closed. +#[derive(Clone, Error, Debug)] +#[error("stream closed")] +pub struct ClosedStream; + +impl From for ClosedStream { + fn from(_: noq::ClosedStream) -> Self { + ClosedStream + } +} + +/// An error returned when receiving a new WebTransport session. +#[derive(Error, Debug, Clone)] +pub enum ServerError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), + + #[error("io error: {0}")] + IoError(Arc), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +// #[derive(Clone, Error, Debug)] +// pub enum SendDatagramError { +// #[error("Unsupported peer")] +// UnsupportedPeer, + +// #[error("Datagram support Disabled by peer")] +// DatagramSupportDisabled, + +// #[error("Datagram Too large")] +// TooLarge, + +// #[error("Session errorr: {0}")] +// SessionError(#[from] SessionError), +// } + +// impl From for SendDatagramError { +// fn from(value: noq::SendDatagramError) -> Self { +// match value { +// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// noq::SendDatagramError::ConnectionLost(e) => +// SendDatagramError::SessionError(e.into()), } +// } +// } + +impl web_transport_trait::Error for SessionError { + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } + + None + } +} + +impl web_transport_trait::Error for WriteError { + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } +} + +impl web_transport_trait::Error for ReadError { + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs new file mode 100644 index 00000000000..e5f37064927 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -0,0 +1,71 @@ +//! WebTransport is a protocol for client-server communication over QUIC. +//! It's [available in the browser](https://caniuse.com/webtransport) as an alternative to HTTP and WebSockets. +//! +//! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. +//! This library hides that detail and tries to expose only the QUIC API, delegating as much as +//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! +//! QUIC provides two primary APIs: +//! +//! # Streams +//! QUIC streams are ordered, reliable, flow-controlled, and optionally bidirectional. +//! Both endpoints can create and close streams (including an error code) with no overhead. +//! You can think of them as TCP connections, but shared over a single QUIC connection. +//! +//! # Datagrams +//! QUIC datagrams are unordered, unreliable, and not flow-controlled. +//! Both endpoints can send datagrams below the MTU size (~1.2kb minimum) and they might arrive out +//! of order or not at all. They are basically UDP packets, except they are encrypted and congestion +//! controlled. +//! +//! # Limitations +//! WebTransport is able to be pooled with HTTP/3 and multiple WebTransport sessions. +//! This crate avoids that complexity, doing the bare minimum to support a single WebTransport +//! session that owns the entire QUIC connection. If you want to support HTTP/3 on the same +//! host/port, you should use another crate (ex. `h3-webtransport`). If you want to support multiple +//! WebTransport sessions over the same QUIC connection... you should just dial a new QUIC +//! connection instead. + +// External +mod client; +mod error; +mod recv; +mod send; +mod server; +mod session; + +pub use client::*; +pub use error::*; +pub use recv::*; +pub use send::*; +pub use server::*; +pub use session::*; + +// Internal +mod connect; +mod settings; + +use connect::*; +use settings::*; + +// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +pub use connect::ConnectError; + +/// The HTTP/3 ALPN is required when negotiating a QUIC connection. +pub const ALPN: &str = "h3"; + +/// Export our simple crypto provider. +pub mod crypto; + +/// Re-export the underlying QUIC implementation. +pub use noq; + +/// Re-export the http crate because it's in the public API. +pub use http; + +/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of +/// [`SessionAccept`]'s poll methods holds its place in the accept queue. +pub use kio; + +/// Re-export the WebTransport protocol implementation. +pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs new file mode 100644 index 00000000000..4c3d1881722 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -0,0 +1,159 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::Bytes; + +use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; + +/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +#[derive(Debug)] +pub struct RecvStream { + inner: noq::RecvStream, + close: Arc, +} + +impl RecvStream { + pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { + Self { + inner: stream, + close, + } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::ReadError) -> ReadError { + match e { + noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + noq::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. + pub async fn read_many_chunks( + &mut self, + bufs: &mut [Bytes], + ) -> Result, ReadError> { + self.inner + .read_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner + .read_to_end(size_limit) + .await + .map_err(|e| match e { + noq::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`noq::RecvStream::received_reset`]. + /// + /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.inner.id() + } + + /// Returns the number of bytes read from this stream. + /// + /// This is the offset of the next byte to be read, i.e. the length of the contiguous + /// prefix of the stream consumed by the application. + pub fn bytes_read(&self) -> Result { + self.inner.bytes_read().map_err(|_| ClosedStream) + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl web_transport_trait::RecvStream for RecvStream { + type Error = ReadError; + + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } + + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } + + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } +} diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs new file mode 100644 index 00000000000..39af0d1fc7e --- /dev/null +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -0,0 +1,197 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::{Buf, Bytes}; + +use crate::{CloseReason, ClosedStream, SessionError, WriteError}; + +/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// +/// This wrapper is mainly needed for error codes, which is unfortunate. +/// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. +#[derive(Debug)] +pub struct SendStream { + stream: noq::SendStream, + close: Arc, +} + +impl SendStream { + pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::WriteError) -> WriteError { + match e { + noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`noq::SendStream::stopped`]. + /// + /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. + /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(noq::StoppedError::ConnectionLost(conn_err)) => { + Err(self.close.map(conn_err.into())) + } + Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream + .write_all(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write chunks of data to the stream, returning the number of bytes written. + /// + /// See [`noq::SendStream::write_many_chunks`]. + pub async fn write_many_chunks( + &mut self, + bufs: &mut &mut [Bytes], + ) -> Result { + self.stream + .write_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream + .write_chunk(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream + .write_all_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`noq::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.stream.id() + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + // We have to use this syntax because noq added its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + +impl web_transport_trait::SendStream for SendStream { + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + // NOTE: This used to require &mut in an older version of Noq. + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } +} diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs new file mode 100644 index 00000000000..8af3e81c983 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -0,0 +1,200 @@ +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use std::sync::Arc; + +use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::{crypto, CongestionControl}; +use crate::{ + proto::{ConnectRequest, ConnectResponse}, + Connecting, ServerError, Session, Settings, +}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Server] using sane defaults. +/// +/// This is optional; advanced users may use [Server::new] directly. +pub struct ServerBuilder { + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ServerBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ServerBuilder { + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard Noq setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important + + let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = noq::ServerConfig::with_crypto(Arc::new(config)); + + let server = + noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } +} + +/// A WebTransport server that accepts new sessions. +pub struct Server { + endpoint: noq::Endpoint, + accept: FuturesUnordered>>, +} + +impl core::ops::Deref for Server { + type Target = noq::Endpoint; + + fn deref(&self) -> &Self::Target { + &self.endpoint + } +} + +impl Server { + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. + pub fn new(endpoint: noq::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } +} + +/// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to +/// accept or reject the session based on the URL. +pub struct Request { + conn: noq::Connection, + settings: Settings, + connect: Connecting, +} + +impl Request { + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: noq::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond( + self, + response: impl Into, + ) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &noq::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } +} + +impl core::ops::Deref for Request { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.connect + } +} diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs new file mode 100644 index 00000000000..82a9149d847 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -0,0 +1,1331 @@ +use std::{ + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use bytes::{Bytes, BytesMut}; +use futures::stream::{FuturesUnordered, Stream, StreamExt}; +use kio::{Fan, Waiter}; + +use crate::{ + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, + WebTransportError, +}; + +/// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake +/// has not completed, or it is not valid UTF-8. +/// +/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// is owned rather than borrowed from the connection. +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +fn negotiated_alpn(conn: &noq::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data.downcast::().ok()?; + + String::from_utf8(data.protocol?).ok() +} + +/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// handshake data type to name here. +/// +/// This crate's own builders are gated the same way, so normally there is no connection +/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// brings its own endpoint can still establish one, and [`Session::protocol`] reports +/// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to +/// get the ALPN. +#[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] +fn negotiated_alpn(_conn: &noq::Connection) -> Option { + None +} + +/// An established WebTransport session, acting like a full QUIC connection. See +/// [`noq::Connection`]. +/// +/// It is important to remember that WebTransport is layered on top of QUIC: +/// 1. Each stream starts with a few bytes identifying the stream type and session ID. +/// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. +/// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. +/// +/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// These should be safe to use with WebTransport, but file a PR if you find one that isn't. +#[derive(Clone)] +pub struct Session { + conn: noq::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, +} + +impl Session { + pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = noq::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { + code, + reason, + })) => return Some((code, reason)), + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self + .conn + .accept_uni() + .await + .map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self + .conn + .read_datagram() + .await + .map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = + VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = + web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .unwrap_or(Duration::from_millis(100)); + let timeout = (rtt * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session + .close_with_capsule(send, capsule, code, timeout) + .await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: noq::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!( + ?e, + "connection ended before the close capsule was acknowledged" + ); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()) + .await + .is_err() + { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`noq::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`noq::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: noq::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + let path = self.conn.path_stats(noq::PathId::ZERO); + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .or_else(|| path.map(|p| p.rtt)); + SessionStats { + stats: self.conn.stats(), + path, + rtt, + } + } +} + +impl Deref for Session { + type Target = noq::Connection; + + fn deref(&self) -> &Self::Target { + &self.conn + } +} + +impl fmt::Debug for Session { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } +} + +impl PartialEq for Session { + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } +} + +impl Eq for Session {} + +// Poll the shared accept state, then wake the *other* accepters once the lock is +// released. +// +// `SessionAccept` does not wake them itself: a waker is free to resume its task +// inline, and the first thing a resumed accepter does is take this same lock. An +// arrival or a failure is exactly what the others are parked waiting to retry +// after, so `Ready` is the signal. +fn poll_accept_uni_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +fn poll_accept_bi_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +// Type aliases just so clippy doesn't complain about the complexity. +type AcceptUni = dyn Stream> + Send; +type AcceptBi = + dyn Stream> + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = + dyn Future, SessionError>> + Send; + +// Logic just for accepting streams, which is annoying because of the stream header. +pub struct SessionAccept { + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, +} + +impl SessionAccept { + pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::new(); + let uni_waiters = Fan::new(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, noq::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi( + &mut self, + waiter: &Waiter, + ) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: noq::SendStream, + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } +} + +// Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any +// failure as a truncation, which would hide a stream the peer reset before its header +// arrived. +async fn read_varint(recv: &mut noq::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; + + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; + + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) +} + +// Without reliable reset, a stream the peer resets early loses its header with it, which +// moq does routinely. Anything else is a misbehaving peer. +fn log_header_error(err: SessionError, direction: &'static str) { + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError( + noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } +} + +pub struct SessionStats { + stats: noq::ConnectionStats, + path: Option, + rtt: Option, +} + +impl web_transport_trait::Stats for SessionStats { + fn bytes_sent(&self) -> Option { + Some(self.stats.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.stats.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + self.path.map(|path| path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.stats.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.stats.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + self.path.map(|path| path.lost_packets) + } + + fn rtt(&self) -> Option { + self.rtt + } + + fn estimated_send_rate(&self) -> Option { + let path = self.path?; + let rtt = self.rtt?; + let rtt_secs = rtt.as_secs_f64(); + if path.cwnd > 0 && rtt_secs > 0.0 { + Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) + } else { + None + } + } +} + +impl web_transport_trait::Session for Session { + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } +} + +#[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] +mod tests { + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl noq::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &noq::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl noq::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [noq::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + noq::Connection, + noq::Connection, + [noq::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( + &signing_key, + ))); + + let server_config = + noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let runtime: Arc = Arc::new(noq::TokioRuntime); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: runtime.wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = noq::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + runtime, + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { + crate::Request::accept(server_conn) + .await + .unwrap() + .ok() + .await + }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert!( + matches!( + err, + SessionError::WebTransportError(WebTransportError::UnknownSession) + ), + "{err:?}" + ); + } +} diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs new file mode 100644 index 00000000000..08ea9a53a1f --- /dev/null +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -0,0 +1,70 @@ +use futures::try_join; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum SettingsError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), + + #[error("WebTransport is not supported")] + WebTransportUnsupported, + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), +} + +pub struct Settings { + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: noq::SendStream, + + #[allow(dead_code)] + recv: noq::RecvStream, +} + +impl Settings { + // Establish the H3 connection. + pub async fn connect(conn: &noq::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); + + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } + + async fn accept(conn: &noq::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; + + tracing::debug!(?settings, "received SETTINGS frame"); + + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } + + Ok(recv) + } + + async fn open(conn: &noq::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); + + tracing::debug!(?settings, "sending SETTINGS frame"); + + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; + + Ok(send) + } +} From e8195bdb5952909497cf01bab7ea514c4eaf519f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:35:17 -0700 Subject: [PATCH 03/51] feat(quic)!: run moq-tokio and moq-uring on moq-quic Drop every moq-noq dependency. moq-tokio's QUIC backend now drives moq-quic through quinn's async layer (`quic::driver`) with web-transport-moq folded in as `quic::web`; moq-uring's sans-IO driver moves from moq-noq-proto to moq-quic. Upstream noq stays only behind moq-tokio's `iroh` feature. - The `noq` cargo feature is `quic` everywhere it was forwarded. - `moq_tokio::noq::Error` is `moq_tokio::quic::Error`; `Error::Noq` is `Error::Quic`. - moq-quic takes `moq_clock::Instant` and raises a once-per-connection Draining endpoint event, which backs the driver's `wait_all_draining`. - qlog writes one trace per endpoint (per dial on moq-uring's connect side), since moq-quic holds one sink per transport config. - Session stats report the congestion controller's bandwidth estimate. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 114 +- Cargo.toml | 12 +- doc/bin/cli.md | 2 +- doc/bin/relay/config.md | 2 +- doc/setup/upgrade.md | 7 + rs/hang/Cargo.toml | 2 +- rs/justfile | 10 +- rs/moq-bench/Cargo.toml | 4 +- rs/moq-boy/Cargo.toml | 4 +- rs/moq-c/Cargo.toml | 2 +- rs/moq-cli/Cargo.toml | 8 +- rs/moq-gst/src/lib.rs | 2 +- rs/moq-hls/Cargo.toml | 2 +- rs/moq-quic/Cargo.toml | 1 + rs/moq-quic/README.md | 15 +- rs/moq-quic/src/config/transport.rs | 15 +- rs/moq-quic/src/connection/mod.rs | 15 + rs/moq-quic/src/connection/qlog.rs | 16 +- rs/moq-quic/src/endpoint.rs | 1 + rs/moq-quic/src/lib.rs | 6 +- rs/moq-quic/src/shared.rs | 11 + rs/moq-quic/src/tests/mod.rs | 31 + rs/moq-quic/src/tests/util.rs | 15 + rs/moq-relay/Cargo.toml | 7 +- rs/moq-relay/src/config.rs | 2 +- rs/moq-relay/src/relay.rs | 6 +- rs/moq-relay/src/shutdown.rs | 2 +- rs/moq-relay/tests/cluster_idle.rs | 2 +- rs/moq-relay/tests/drills.rs | 2 +- rs/moq-relay/tests/goaway_cluster.rs | 2 +- rs/moq-relay/tests/runtime_uring.rs | 23 +- rs/moq-sock/src/bind.rs | 6 +- rs/moq-tokio/Cargo.toml | 39 +- rs/moq-tokio/README.md | 2 +- rs/moq-tokio/src/client.rs | 122 +- rs/moq-tokio/src/connect.rs | 4 +- rs/moq-tokio/src/connection.rs | 10 +- rs/moq-tokio/src/error.rs | 26 +- rs/moq-tokio/src/iroh.rs | 6 +- rs/moq-tokio/src/lib.rs | 12 +- rs/moq-tokio/src/listen.rs | 14 +- rs/moq-tokio/src/log.rs | 2 +- rs/moq-tokio/src/quic.rs | 33 +- rs/moq-tokio/src/{noq.rs => quic/backend.rs} | 289 +- rs/moq-tokio/src/quic/driver/.rustfmt.toml | 2 + rs/moq-tokio/src/quic/driver/connection.rs | 99 +- rs/moq-tokio/src/quic/driver/endpoint.rs | 178 +- rs/moq-tokio/src/quic/driver/incoming.rs | 8 +- rs/moq-tokio/src/quic/driver/mod.rs | 54 +- rs/moq-tokio/src/quic/driver/mutex.rs | 111 - rs/moq-tokio/src/quic/driver/recv_stream.rs | 59 +- rs/moq-tokio/src/quic/driver/runtime/mod.rs | 70 +- rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 - rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 53 +- rs/moq-tokio/src/quic/driver/send_stream.rs | 31 +- rs/moq-tokio/src/quic/driver/tests.rs | 205 +- rs/moq-tokio/src/quic/driver/work_limiter.rs | 2 +- rs/moq-tokio/src/quic/web/client.rs | 577 ++-- rs/moq-tokio/src/quic/web/connect.rs | 238 +- rs/moq-tokio/src/quic/web/crypto.rs | 64 +- rs/moq-tokio/src/quic/web/error.rs | 547 ++-- rs/moq-tokio/src/quic/web/fan.rs | 140 + rs/moq-tokio/src/quic/web/mod.rs | 12 +- rs/moq-tokio/src/quic/web/recv.rs | 246 +- rs/moq-tokio/src/quic/web/send.rs | 323 +-- rs/moq-tokio/src/quic/web/server.rs | 302 +-- rs/moq-tokio/src/quic/web/session.rs | 2399 ++++++++--------- rs/moq-tokio/src/quic/web/settings.rs | 84 +- rs/moq-tokio/src/resolve.rs | 2 +- rs/moq-tokio/src/server.rs | 146 +- rs/moq-tokio/src/tcp.rs | 2 +- rs/moq-tokio/src/tls.rs | 34 +- rs/moq-tokio/src/transport.rs | 2 +- rs/moq-tokio/src/websocket.rs | 8 +- rs/moq-tokio/src/worker/group.rs | 8 +- rs/moq-tokio/src/worker/mod.rs | 4 +- rs/moq-tokio/tests/backend.rs | 100 +- rs/moq-tokio/tests/broadcast.rs | 2 +- rs/moq-tokio/tests/close_code.rs | 2 +- rs/moq-tokio/tests/stream_code.rs | 12 +- rs/moq-tokio/tests/worker.rs | 2 +- rs/moq-transcode/Cargo.toml | 2 +- rs/moq-uring/Cargo.toml | 23 +- rs/moq-uring/README.md | 24 +- .../benches/{echo_noq.rs => echo_quic.rs} | 10 +- rs/moq-uring/benches/session_lite.rs | 10 +- rs/moq-uring/src/lib.rs | 4 +- .../src/quic/{noq => driver}/connection.rs | 133 +- .../src/quic/{noq => driver}/endpoint.rs | 42 +- rs/moq-uring/src/quic/{noq => driver}/mod.rs | 134 +- .../src/quic/{noq => driver}/stream.rs | 38 +- rs/moq-uring/src/quic/endpoint.rs | 2 +- rs/moq-uring/src/quic/mod.rs | 17 +- rs/moq-uring/src/quic/qlog.rs | 37 +- rs/moq-uring/src/quic/web.rs | 2 +- rs/moq-uring/tests/echo.rs | 12 +- rs/moq-uring/tests/ecn.rs | 2 +- rs/moq-uring/tests/endpoint.rs | 10 +- rs/moq-uring/tests/identity.rs | 2 +- rs/moq-uring/tests/qlog.rs | 6 +- rs/moq-uring/tests/session.rs | 4 +- rs/moq-uring/tests/teardown.rs | 2 +- rs/moq-uring/tests/web.rs | 49 +- rs/moq-uring/tests/workers.rs | 2 +- sh/rs/select.test.py | 2 +- sh/rs/tokio-features.sh | 10 +- 106 files changed, 3656 insertions(+), 4064 deletions(-) rename rs/moq-tokio/src/{noq.rs => quic/backend.rs} (79%) create mode 100644 rs/moq-tokio/src/quic/driver/.rustfmt.toml delete mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/web/fan.rs rename rs/moq-uring/benches/{echo_noq.rs => echo_quic.rs} (93%) rename rs/moq-uring/src/quic/{noq => driver}/connection.rs (87%) rename rs/moq-uring/src/quic/{noq => driver}/endpoint.rs (92%) rename rs/moq-uring/src/quic/{noq => driver}/mod.rs (79%) rename rs/moq-uring/src/quic/{noq => driver}/stream.rs (90%) diff --git a/Cargo.lock b/Cargo.lock index 0447ee73e6c..a2a305b404d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4675,72 +4675,6 @@ dependencies = [ name = "moq-net-sim-macros" version = "0.0.0" -[[package]] -name = "moq-noq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20d9d1334f556a15e60f6d406020fc3a40409335ae8a2ad472f6bf074a01acfa" -dependencies = [ - "bytes", - "cfg_aliases", - "derive_more 2.1.1", - "moq-noq-proto", - "moq-noq-udp", - "pin-project-lite", - "rustc-hash 2.1.3", - "rustls", - "socket2 0.6.5", - "thiserror 2.0.21", - "tokio", - "tokio-stream", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-proto" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50ac444c6c1ed18d45536cd939be7f0fd664996e79c0519119b13d6ae9d3d5e2" -dependencies = [ - "aes-gcm", - "aws-lc-rs", - "bytes", - "derive_more 2.1.1", - "enum-assoc", - "fastbloom", - "getrandom 0.4.3", - "identity-hash", - "lru-slab", - "n0-qlog", - "rand 0.10.3", - "rand_pcg", - "ring", - "rustc-hash 2.1.3", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "slab", - "sorted-index-buffer", - "thiserror 2.0.21", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-udp" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e978cded333ca8a6d50b96a7a4388ccc8d98db51d89df7dbee598e9b849ee79" -dependencies = [ - "cfg_aliases", - "libc", - "socket2 0.6.5", - "tracing", - "windows-sys 0.61.2", -] - [[package]] name = "moq-nvenc" version = "0.1.2" @@ -4771,6 +4705,7 @@ dependencies = [ "getrandom 0.4.3", "hex-literal", "lru-slab", + "moq-clock", "qlog", "rand 0.10.3", "rand_pcg", @@ -4835,7 +4770,6 @@ dependencies = [ "tracing-subscriber", "url", "usage-rs", - "web-transport-moq", "web-transport-trait", ] @@ -4992,19 +4926,26 @@ dependencies = [ "futures", "hex", "hmac 0.13.0", + "http", "humantime", "jni 0.22.4", + "kio 0.6.2", "libc", "mdns-sd", + "moq-clock", "moq-net", + "moq-quic", "moq-sock", "noq-proto", "notify", + "pin-project-lite", "qmux", "rand 0.10.3", "rcgen", "reqwest", + "rustc-hash 2.1.3", "rustls", + "rustls-native-certs", "rustls-platform-verifier", "rustls-webpki", "serde", @@ -5027,7 +4968,6 @@ dependencies = [ "url", "usage-rs", "web-transport-iroh", - "web-transport-moq", "web-transport-proto", "web-transport-trait", "webpki-roots", @@ -5064,8 +5004,9 @@ dependencies = [ "io-uring", "kio 0.6.2", "libc", + "moq-clock", "moq-net", - "moq-noq-proto", + "moq-quic", "moq-sock", "moq-tokio", "nix", @@ -5081,7 +5022,6 @@ dependencies = [ "tokio", "tracing", "url", - "web-transport-moq", "web-transport-proto", "web-transport-trait", ] @@ -5292,19 +5232,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "n0-qlog" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1c2fc610d1facce51b0bf5d8245b309c91996a933defdf7742c92ac78a58cb" -dependencies = [ - "humantime", - "serde", - "serde_json", - "serde_with", - "strum", -] - [[package]] name = "n0-watcher" version = "1.0.0" @@ -10240,27 +10167,6 @@ dependencies = [ "web-transport-trait", ] -[[package]] -name = "web-transport-moq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc946e718371f88a79d849c352b6bdd95e0a36139e9b76818c1a101b45ed97a" -dependencies = [ - "bytes", - "futures", - "http", - "kio 0.5.9", - "moq-noq", - "rustls", - "rustls-native-certs", - "thiserror 2.0.21", - "tokio", - "tracing", - "url", - "web-transport-proto", - "web-transport-trait", -] - [[package]] name = "web-transport-proto" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 19a3faf7eed..80f784458fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,7 @@ mdns-sd = { version = "0.21", features = ["async"] } moq-archive = { version = "0.0.11", path = "rs/moq-archive" } moq-audio = { version = "0.1.10", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.2.1", path = "rs/moq-auth" } +moq-clock = { version = "0.11.0", path = "rs/moq-clock" } moq-clock-macros = { version = "0.1.0", path = "rs/moq-clock/macros" } moq-flate = { version = "0.2.0", path = "rs/moq-flate" } moq-hls = { version = "0.5.11", path = "rs/moq-hls", default-features = false } @@ -157,14 +158,13 @@ moq-loc = { version = "0.2.19", path = "rs/moq-loc" } moq-msf = { version = "0.5.1", path = "rs/moq-msf" } moq-mux = { version = "0.10.11", path = "rs/moq-mux" } moq-net = { version = "0.3.10", path = "rs/moq-net" } -# The MoQ fork of noq (moq-dev/noq). iroh keeps upstream noq, so a build with the -# iroh feature carries both stacks. -moq-noq-proto = { version = "2.0.5", default-features = false } # NVENC bindings, forked from ViliamVadocz/nvidia-video-codec-sdk to dlopen the # driver at runtime. Compiles on any platform (macOS included) but only actually # used by moq-video on Linux. moq-nvenc = { version = "0.1.2", path = "rs/moq-nvenc" } moq-pattern = { version = "0.1.2", path = "rs/moq-pattern" } +# default-features off so each consumer picks its crypto provider. +moq-quic = { version = "0.1.0", path = "rs/moq-quic", default-features = false } moq-relay = { version = "0.17.2", path = "rs/moq-relay", default-features = false } moq-rtc = { version = "0.3.12", path = "rs/moq-rtc" } moq-rtmp = { version = "0.3.11", path = "rs/moq-rtmp" } @@ -178,7 +178,7 @@ moq-tokio = { version = "0.19.23", path = "rs/moq-tokio", default-features = fal # VAAPI is opt-in everywhere; its decoder is hardware-validated, while its # encoder is not yet. moq-transcode = { version = "0.1.10", path = "rs/moq-transcode", default-features = false } -# default-features off (the noq backend) so the consumer picks which QUIC +# default-features off (the `quic` backend) so the consumer opts into the QUIC # stack the io_uring path compiles; cargo features are additive, so a default-on # backend could not be opted out of. moq-uring = { version = "0.0.12", path = "rs/moq-uring", default-features = false } @@ -240,10 +240,6 @@ usage = { package = "usage-rs", version = "6.11.1" } uuid = { version = "1.26", features = ["v7"] } web-async = { version = "0.1.5", features = ["tracing"] } web-transport-iroh = "0.8.3" -# default-features off so the QUIC crypto provider is chosen by moq-tokio's -# aws-lc-rs / ring features. -# The WebTransport adapter released with moq-noq, from the same repository. -web-transport-moq = { version = "2.0.5", default-features = false } web-transport-proto = "0.6" web-transport-trait = "0.5" web-transport-wasm = "0.8" diff --git a/doc/bin/cli.md b/doc/bin/cli.md index d127621494c..ac4f645cf24 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -168,7 +168,7 @@ Playback is behind the `play` feature, since it pulls in windowing and audio-device dependencies: ```bash -cargo install moq-cli --no-default-features --features "noq,websocket,play" +cargo install moq-cli --no-default-features --features "quic,websocket,play" ``` `moq play` skips disabled renditions. If none are enabled, it waits for a later catalog. Disabling the current rendition stops it and selects an enabled replacement. diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index f4a5099962e..9bd91a4a5e3 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -122,7 +122,7 @@ io_uring = false # Drive them with io_uring instead of tokio ``` Packets are steered by connection ID, so a client that migrates stays with its -worker. `workers` needs the `noq` feature. Every worker and the TLS stream +worker. `workers` needs the `quic` feature. Every worker and the TLS stream listener serve one shared identity, generated or file-backed, so a certificate rotation reaches them together and `/certificate.sha256` stays accurate. diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index dd5c605fdc7..e0e6502d998 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -23,6 +23,13 @@ These land with the next breaking release, not the 2026-09-23 train. with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` orders audio by enabled state, bitrate, sample rate, channel count, and name. Both helpers put disabled renditions last. +- **The `noq` cargo feature is `quic`.** Rename it wherever you enable it on + moq-tokio, moq-uring, moq-relay, moq-cli, or the other crates that forward + it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` + crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and + `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one + file per endpoint instead of one per connection; on io_uring workers, one per + endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC `idle_timeout` and `keep_alive`; backoff `initial`, `max`, and `timeout`; diff --git a/rs/hang/Cargo.toml b/rs/hang/Cargo.toml index dd2f9d495b7..24dfae2525f 100644 --- a/rs/hang/Cargo.toml +++ b/rs/hang/Cargo.toml @@ -33,7 +33,7 @@ features = ["from", "display", "debug"] [dev-dependencies] anyhow = { workspace = true } moq-mux = { path = "../moq-mux" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } tokio = { workspace = true, features = ["macros", "fs"] } tracing = { workspace = true } url = { workspace = true } diff --git a/rs/justfile b/rs/justfile index 8f1bdf33cc5..92fd2235648 100644 --- a/rs/justfile +++ b/rs/justfile @@ -46,8 +46,8 @@ check-test *args: lint *args: python3 sh/rs/stats-docs.py cargo fmt --all --check - # quinn-udp keeps quinn's formatting; cargo fmt skips it (see moq-sock's lib.rs). - rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs + # quinn-udp and quinn keep quinn's formatting; cargo fmt skips them (see moq-sock's lib.rs, moq-tokio's quic.rs). + rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps "$@" cargo shear cargo sort --workspace --check --no-format @@ -57,7 +57,7 @@ lint *args: fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets "$@" cargo fmt --all - rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs + rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs cargo shear --fix cargo sort --workspace --no-format @@ -312,11 +312,11 @@ bench-udp *args: cargo bench -p moq-uring --bench udp_tokio -- "$@" cargo bench -p moq-uring --bench udp_uring -- "$@" -# Measure a noq echo through the io_uring worker, ablating receive +# Measure a QUIC echo through the io_uring worker, ablating receive # batching, GRO, and GSO one at a time. Linux 6.12+ (skips loudly otherwise). [positional-arguments] bench-echo *args: - cargo bench -p moq-uring --bench echo_noq -- "$@" + cargo bench -p moq-uring --bench echo_quic -- "$@" # Measure a full moq-lite session (publisher to subscriber) through the # io_uring worker, same ablations and per-iteration unit as bench-echo. diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index 0de559a2e6a..80e9ed80952 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -35,9 +35,9 @@ required-features = ["uring"] doc = false [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] iroh = ["moq-tokio/iroh"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] uring = ["dep:moq-uring"] websocket = ["moq-tokio/websocket"] diff --git a/rs/moq-boy/Cargo.toml b/rs/moq-boy/Cargo.toml index cffec179026..ca15ca12d15 100644 --- a/rs/moq-boy/Cargo.toml +++ b/rs/moq-boy/Cargo.toml @@ -12,9 +12,9 @@ edition = "2024" rust-version.workspace = true [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] websocket = ["moq-tokio/websocket"] [dependencies] diff --git a/rs/moq-c/Cargo.toml b/rs/moq-c/Cargo.toml index c7564df3878..a7cd65b3aa7 100644 --- a/rs/moq-c/Cargo.toml +++ b/rs/moq-c/Cargo.toml @@ -26,7 +26,7 @@ moq-audio = { workspace = true, features = ["aac"] } moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } -moq-tokio = { workspace = true, features = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] } +moq-tokio = { workspace = true, features = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] } # Select the OpenH264 fallback plus NVIDIA and VAAPI hardware codecs explicitly # because workspace defaults are off. The hardware backends dlopen their driver; # `vaapi` also runs bindgen, so Linux builds need libclang. diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9e8d1066930..14401ce2245 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,13 +26,13 @@ path = "src/main.rs" doc = false [features] -default = ["cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] +default = ["cluster-lan", "quic", "websocket", "nvidia", "s3", "gcs", "azure"] # Experimental transport with a heavy dependency tree, enabled only on request. iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] # qlog trace capture (`--client-quic-qlog `). Off by default: it compiles the # qlog machinery into the enabled backends and is only wanted for debugging or # congestion-control testing. @@ -63,7 +63,7 @@ play = ["dep:moq-video", "moq-video/render", "dep:moq-audio", "moq-audio/playbac # the matching moq-video feature, so it only bites when something else pulls in # moq-video: `capture`, `transcode`, or `play`. For a build with no CUDA # dependencies, drop it: -# cargo build -p moq-cli --no-default-features --features "noq,websocket,capture" +# cargo build -p moq-cli --no-default-features --features "quic,websocket,capture" nvidia = ["moq-video?/nvidia", "moq-transcode?/nvidia"] # Compatibility aliases for the pre-consolidation names, kept so a manifest that # already asks for the old split still resolves. Undocumented on purpose: the @@ -112,7 +112,7 @@ moq-hls = { workspace = true, features = ["server"] } moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the -# CLI's own feature set (iroh, noq, cluster-lan) is what selects them. +# CLI's own feature set (iroh, quic, cluster-lan) is what selects them. moq-relay = { workspace = true } moq-rtc = { workspace = true } moq-rtmp = { workspace = true } diff --git a/rs/moq-gst/src/lib.rs b/rs/moq-gst/src/lib.rs index 57c518542ef..aaaa5f1f89a 100644 --- a/rs/moq-gst/src/lib.rs +++ b/rs/moq-gst/src/lib.rs @@ -22,7 +22,7 @@ pub fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { .with_default_directive(LevelFilter::INFO.into()) .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse().unwrap()) - .add_directive("noq=info".parse().unwrap()) + .add_directive("moq_tokio::quic::driver=info".parse().unwrap()) .add_directive("tracing::span=off".parse().unwrap()) .add_directive("tracing::span::active=off".parse().unwrap()); diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 60ab13990ee..4439f49b3c0 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -40,6 +40,6 @@ url = { workspace = true } async-trait = "0.1" futures = { workspace = true } moq-archive = { path = "../moq-archive" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-quic/Cargo.toml b/rs/moq-quic/Cargo.toml index a34ddfdadce..3580b37e95e 100644 --- a/rs/moq-quic/Cargo.toml +++ b/rs/moq-quic/Cargo.toml @@ -56,6 +56,7 @@ aws-lc-rs = { version = "1.9", default-features = false, optional = true } bytes = { workspace = true } fastbloom = { version = "0.17", default-features = false, optional = true } lru-slab = "0.1.2" +moq-clock = { workspace = true } qlog = { version = "0.18", optional = true } rand = { workspace = true } rand_pcg = "0.10" diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 789d1b1b446..d47b7629b0b 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -34,6 +34,17 @@ Changes to its `Cargo.toml` and `build.rs` go into moq-sock's by hand, and its b The module logs through `tracing` only, so quinn-udp's `log` feature and no-op logger are dropped. A `.rustfmt.toml` in each imported directory keeps quinn's formatting; `cargo fmt` cannot apply it to a submodule, so moq-sock skips `udp` and `just rs fix` formats it separately. +quinn itself, the async layer from the same commit, is the `quic::driver` module of [moq-tokio](../moq-tokio), formatted the same way: + +```sh +git -C ../quinn format-patch -1 --stdout -- quinn/src \ + | sed -E 's#([ab])/quinn/src/lib\.rs#\1/rs/moq-tokio/src/quic/driver/mod.rs#g; s#([ab])/quinn/src/#\1/rs/moq-tokio/src/quic/driver/#g; s/\bcrate::/crate::quic::driver::/g; s/\bproto::/moq_quic::/g; s/\budp::/moq_sock::udp::/g' \ + | git am -3 +``` + +The driver is tokio-only: the `Runtime` trait, smol, the `futures-io` impls, and the lock-tracking mutex are dropped, so hunks touching them are skipped. +moq-tokio's `quic::web` module is `web-transport-moq` from moq-dev/noq at [`e27c0ce0`](https://github.com/moq-dev/noq/commit/e27c0ce00741d81c8620b98e345d4f2c88e0768c), which has no upstream left to track. + ### Carried changes Changes on top of the upstream commit, besides the renames: @@ -51,7 +62,9 @@ Changes on top of the upstream commit, besides the renames: - Apple fast-path fallback preserves the segmentation of batches prepared before the fast path was disabled, including when a private symbol is unavailable. - Discarding Initial or Handshake keys resets the PTO backoff, as RFC 9002 A.11 does, so a client's Initial backoff no longer delays the retransmission of a lost Finished. Also [moq-dev/noq#32](https://github.com/moq-dev/noq/pull/32); not offered upstream yet. - `TransportConfig::handshake_idle_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. - quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the switch's import has to set both timeouts there. + quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. +- **Instants on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. +- **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/transport.rs b/rs/moq-quic/src/config/transport.rs index eb72d813851..48e26c5b661 100644 --- a/rs/moq-quic/src/config/transport.rs +++ b/rs/moq-quic/src/config/transport.rs @@ -1,16 +1,16 @@ use std::{fmt, sync::Arc}; #[cfg(feature = "qlog")] -use std::{io, sync::Mutex, time::Instant}; +use std::{io, sync::Mutex}; #[cfg(feature = "qlog")] use qlog::streamer::QlogStreamer; -#[cfg(feature = "qlog")] -use crate::QlogStream; use crate::{ Duration, INITIAL_MTU, MAX_UDP_PAYLOAD, VarInt, VarIntBoundsExceeded, congestion, connection::qlog::QlogSink, }; +#[cfg(feature = "qlog")] +use crate::{Instant, QlogStream}; /// Parameters governing the core QUIC state machine /// @@ -634,10 +634,12 @@ impl QlogConfig { vec![], ); + // The streamer's own epoch only times the events it stamps itself, which this + // crate never asks it to; `QlogStream` stamps them on MoQ's clock instead. let mut streamer = QlogStreamer::new( self.title, self.description, - self.start_time, + std::time::Instant::now(), trace, qlog::events::EventImportance::Core, // `Instant` should have sub-microsecond precision on most platforms @@ -646,7 +648,10 @@ impl QlogConfig { ); match streamer.start_log() { - Ok(()) => Some(QlogStream(Arc::new(Mutex::new(streamer)))), + Ok(()) => Some(QlogStream { + streamer: Arc::new(Mutex::new(streamer)), + start: self.start_time, + }), Err(e) => { warn!("could not initialize endpoint qlog streamer: {e}"); None diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 0ad0ddbbb41..e828b460551 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -171,6 +171,8 @@ pub struct Connection { retry_src_cid: Option, events: VecDeque, endpoint_events: VecDeque, + /// Whether `EndpointEventInner::Draining` was emitted, which happens once. + draining_reported: bool, /// Whether the spin bit is in use for this connection spin_enabled: bool, /// Outgoing spin bit state @@ -317,6 +319,7 @@ impl Connection { retry_src_cid: None, events: VecDeque::new(), endpoint_events: VecDeque::new(), + draining_reported: false, spin_enabled: config.allow_spin && rng.random_ratio(7, 8), spin: false, spaces: [initial_space, PacketSpace::new(now), PacketSpace::new(now)], @@ -1256,6 +1259,7 @@ impl Connection { match timer { Timer::Close => { self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } Timer::Idle => { @@ -2625,6 +2629,9 @@ impl Connection { self.set_close_timer(now); } } + if matches!(self.state, State::Draining | State::Drained) { + self.report_draining(); + } if !was_drained && self.state.is_drained() { self.endpoint_events.push_back(EndpointEventInner::Drained); // Close timer may have been started previously, e.g. if we sent a close and got a @@ -3968,9 +3975,17 @@ impl Connection { self.close_common(); self.error = Some(reason); self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } + /// Tell the endpoint the connection is no longer active, the first time it stops being so + fn report_draining(&mut self) { + if !mem::replace(&mut self.draining_reported, true) { + self.endpoint_events.push_back(EndpointEventInner::Draining); + } + } + /// Storage size required for the largest packet known to be supported by the current path /// /// Buffers passed to [`Connection::poll_transmit`] should be at least this large. diff --git a/rs/moq-quic/src/connection/qlog.rs b/rs/moq-quic/src/connection/qlog.rs index 6120e4e6f93..0294a695160 100644 --- a/rs/moq-quic/src/connection/qlog.rs +++ b/rs/moq-quic/src/connection/qlog.rs @@ -27,17 +27,23 @@ use crate::{ /// Shareable handle to a single qlog output stream #[cfg(feature = "qlog")] #[derive(Clone)] -pub struct QlogStream(pub(crate) Arc>); +pub struct QlogStream { + pub(crate) streamer: Arc>, + /// The epoch event times count from, on MoQ's clock rather than the std one the + /// streamer keeps. + pub(crate) start: Instant, +} #[cfg(feature = "qlog")] impl QlogStream { fn emit_event(&self, orig_rem_cid: ConnectionId, event: EventData, now: Instant) { - // Time will be overwritten by `add_event_with_instant` - let mut event = Event::with_time(0.0, event); + // Milliseconds since the epoch, at the nanosecond precision the streamer is built with. + let time = now.saturating_duration_since(self.start).as_nanos() as f64 / 1e6; + let mut event = Event::with_time(time, event); event.group_id = Some(Box::new(orig_rem_cid.to_string())); - let mut qlog_streamer = self.0.lock().unwrap(); - if let Err(e) = qlog_streamer.add_event_with_instant(event, now) { + let mut qlog_streamer = self.streamer.lock().unwrap(); + if let Err(e) = qlog_streamer.add_event(event) { warn!("could not emit qlog event: {e}"); } } diff --git a/rs/moq-quic/src/endpoint.rs b/rs/moq-quic/src/endpoint.rs index 616afffd824..19496b44789 100644 --- a/rs/moq-quic/src/endpoint.rs +++ b/rs/moq-quic/src/endpoint.rs @@ -126,6 +126,7 @@ impl Endpoint { } } } + Draining => {} Drained => { if let Some(conn) = self.connections.try_remove(ch.0) { self.index.remove(&conn); diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index 65ddf62da28..a5c39cb9576 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -105,10 +105,12 @@ pub use token_memory_cache::TokenMemoryCache; use arbitrary::Arbitrary; // Deal with time +// MoQ's clock, so a caller-driven clock (a test's or the simulator's) reaches the state machine. +pub(crate) use moq_clock::Instant; #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] -pub(crate) use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(all(target_family = "wasm", target_os = "unknown"))] -pub(crate) use web_time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use web_time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(fuzzing)] pub mod fuzzing { diff --git a/rs/moq-quic/src/shared.rs b/rs/moq-quic/src/shared.rs index f2d0ad5d2ac..047c6d7ce00 100644 --- a/rs/moq-quic/src/shared.rs +++ b/rs/moq-quic/src/shared.rs @@ -45,10 +45,21 @@ impl EndpointEvent { pub fn is_drained(&self) -> bool { self.0 == EndpointEventInner::Drained } + + /// Determine whether the connection has stopped being active + /// + /// Emitted once per connection, before [`Self::is_drained`]'s event: when the peer's close + /// arrives, or at the drain if it never does. Past it the peer has our close or never will, + /// so a shutdown can stop waiting without sitting out the closing period. + pub fn is_draining(&self) -> bool { + self.0 == EndpointEventInner::Draining + } } #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum EndpointEventInner { + /// The connection has stopped being active; see [`EndpointEvent::is_draining`] + Draining, /// The connection has been drained Drained, /// The reset token and/or address eligible for generating resets has been updated diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index fedc9af91d7..60a8e74199a 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -2004,6 +2004,37 @@ fn initial_retransmit() { ); } +/// Each side tells its endpoint once that the connection stopped being active, before it drains, +/// so a shutdown waiting on every connection can stop at the first of these +#[test] +fn draining_is_reported_once_before_drained() { + let _guard = subscribe(); + let mut pair = Pair::default(); + let (client_ch, server_ch) = pair.connect(); + let now = pair.time; + pair.client_conn_mut(client_ch) + .close(now, VarInt(0), Bytes::new()); + pair.drive(); + + let expected = [Lifecycle::Draining, Lifecycle::Drained]; + let client: Vec<_> = pair + .client + .lifecycle + .iter() + .filter(|(ch, _)| *ch == client_ch) + .map(|(_, l)| *l) + .collect(); + let server: Vec<_> = pair + .server + .lifecycle + .iter() + .filter(|(ch, _)| *ch == server_ch) + .map(|(_, l)| *l) + .collect(); + assert_eq!(client, expected); + assert_eq!(server, expected); +} + #[test] fn instant_close_1() { let _guard = subscribe(); diff --git a/rs/moq-quic/src/tests/util.rs b/rs/moq-quic/src/tests/util.rs index e64c1ccf09f..f98c7f5eca5 100644 --- a/rs/moq-quic/src/tests/util.rs +++ b/rs/moq-quic/src/tests/util.rs @@ -397,6 +397,14 @@ pub(super) struct TestEndpoint { pub(super) capture_inbound_packets: bool, pub(super) handle_incoming: Box IncomingConnectionBehavior>, pub(super) waiting_incoming: Vec, + /// Every draining and drained endpoint event, in the order connections raised them + pub(super) lifecycle: Vec<(ConnectionHandle, Lifecycle)>, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum Lifecycle { + Draining, + Drained, } #[derive(Debug, Copy, Clone)] @@ -441,6 +449,7 @@ impl TestEndpoint { capture_inbound_packets: false, handle_incoming: Box::new(|_| IncomingConnectionBehavior::Accept), waiting_incoming: Vec::new(), + lifecycle: Vec::new(), } } @@ -536,6 +545,12 @@ impl TestEndpoint { } for (ch, event) in endpoint_events { + if event.is_draining() { + self.lifecycle.push((ch, Lifecycle::Draining)); + } + if event.is_drained() { + self.lifecycle.push((ch, Lifecycle::Drained)); + } if let Some(event) = self.handle_event(ch, event) && let Some(conn) = self.connections.get_mut(&ch) { diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 8b1a73eb743..7791f785946 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,12 +25,12 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] -default = ["cluster-lan", "noq", "websocket", "uds"] +default = ["cluster-lan", "quic", "websocket", "uds"] iroh = ["moq-tokio/iroh"] # Discover cluster peers on the LAN with mDNS (`--cluster-lan`). cluster-lan = ["moq-tokio/mdns"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["_quic", "moq-tokio/noq"] +quic = ["_quic", "moq-tokio/quic"] # The tokio QUIC listener, and the thread-per-core worker group behind # `runtime.workers`. Not one to enable on its own: it names no backend, so # moq-tokio would have nothing to bind a socket with. @@ -46,7 +46,7 @@ websocket = ["moq-tokio/websocket", "dep:qmux", "axum/ws"] uds = ["moq-tokio/uds"] # Serve QUIC from io_uring workers (`--runtime-io-uring`), Linux-only. # This is off by default: the relay is useful without io_uring. -io-uring = ["_uring", "moq-uring/noq"] +io-uring = ["_uring", "moq-uring/quic"] # The listener machinery shared by the feature above. _uring = ["dep:moq-sock", "dep:moq-uring"] @@ -101,7 +101,6 @@ rcgen = "0.14" tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } -web-transport-moq = { workspace = true } # Which sockets the process still holds, so the embed tests can prove a stopped # relay closed its own without rebinding a port another test could take. diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 2750f1c024a..707ffbc3795 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1570,7 +1570,7 @@ uid = [1001] "cannot be combined", ); - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] { let path = dir.join("typed.toml"); std::fs::write(&path, "[listen]\nload_balancer = { id = \"ab\", nonce = 8 }\n").unwrap(); diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index fdf4908f6d0..8f463f4c3a0 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -171,7 +171,7 @@ impl Relay { #[cfg(not(feature = "_quic"))] anyhow::ensure!( io_uring || config.runtime.workers().is_none(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); // Workers and the stream server split one relay across listeners, so they share @@ -552,7 +552,7 @@ impl Relay { // Before the readiness notify, for the same reason the cluster starts // before it: an unsupported kernel, a refused ring, or a certificate - // certificate identity noq will not load fails here, and reporting ready first would + // certificate identity rustls will not load fails here, and reporting ready first would // release the units depending on a relay that is about to exit. #[cfg(all(target_os = "linux", feature = "_uring"))] let mut uring = uring; @@ -929,7 +929,7 @@ mod tests { }; assert_eq!( error.to_string(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); } } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 12476d1453a..1cc1461dc8a 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -285,7 +285,7 @@ mod tests { /// A WebSocket peer that upgrades and never sends SETUP ends at the drain /// deadline even with the SETUP timeout disabled, so its task does not /// outlive the relay. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test] async fn withheld_websocket_setup_ends_at_the_deadline() { let window = Duration::from_secs(1); diff --git a/rs/moq-relay/tests/cluster_idle.rs b/rs/moq-relay/tests/cluster_idle.rs index fc90ee1b7eb..981fb408fe1 100644 --- a/rs/moq-relay/tests/cluster_idle.rs +++ b/rs/moq-relay/tests/cluster_idle.rs @@ -2,7 +2,7 @@ //! //! A crashed or partitioned peer sends no close, so the idle timeout is the whole //! outage window for subscribes routed through it. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::time::{Duration, Instant}; diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index e699dc9c4e4..bce5ce6144f 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -23,7 +23,7 @@ //! See `test/drill/README.md` for the recipe, the sensitivity proof that each //! drill fails when its recovery behavior is removed, and the loom/fuzz cases //! covering the primitives underneath. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 726479103b0..c7c0f47ac19 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -33,7 +33,7 @@ async fn within(step: &str, fut: impl std::future::Future) -> T { /// Run an integration-test future on a dedicated thread with a large stack. /// /// Under `--all-features`, `moq-tokio` compiles every transport -/// (noq, iroh, websocket, and qmux) into its `Session`/`Client` types. +/// (QUIC, iroh, websocket, and qmux) into its `Session`/`Client` types. /// These multi-relay tests hold several such values live across await points, /// so the single test future's state machine is large, and in an unoptimized /// build it overflows libtest's default 2 MiB per-test thread stack (a SIGABRT diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 5494fc4eeaa..5d66675801d 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -763,10 +763,10 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { /// `listen.timeout` bounds every handshake phase from the first packet: TLS, /// HTTP/3 before CONNECT, and MoQ SETUP over raw QUIC and WebTransport. Each /// phase closes in its own error space. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] async fn uring_workers_bound_stalled_handshakes() { - use web_transport_moq::noq; + use moq_tokio::quic::{driver, web}; use web_transport_trait::Error as _; let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -788,10 +788,10 @@ async fn uring_workers_bound_stalled_handshakes() { let crypto = |alpn: &[u8]| { let mut tls = tls.clone(); tls.alpn_protocols = vec![alpn.to_vec()]; - let crypto = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); - noq::ClientConfig::new(std::sync::Arc::new(crypto)) + let crypto = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); + driver::ClientConfig::new(std::sync::Arc::new(crypto)) }; - let client = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); + let client = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); // TLS: the relay never hears more than the ClientHello, so only its own // deadline can end the handshake before the idle timeout does. @@ -806,7 +806,7 @@ async fn uring_workers_bound_stalled_handshakes() { .await .expect("listener did not bound the QUIC handshake"); assert!( - !matches!(error, noq::ConnectionError::TimedOut), + !matches!(error, driver::ConnectionError::TimedOut), "the relay must close a stalled QUIC handshake: {error}" ); @@ -823,11 +823,10 @@ async fn uring_workers_bound_stalled_handshakes() { .unwrap(); if scheme == "https" { let url: url::Url = format!("https://{addr}/stalled").parse().unwrap(); - let request = - web_transport_moq::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); - web_transport_moq::Session::connect(conn, request).await.unwrap() + let request = web::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); + web::Session::connect(conn, request).await.unwrap() } else { - web_transport_moq::Session::raw(conn) + web::Session::raw(conn) } }) .await @@ -842,7 +841,7 @@ async fn uring_workers_bound_stalled_handshakes() { "{scheme}: {error}" ); } - client.close(noq::VarInt::from_u32(0), b"test complete"); + client.close(driver::VarInt::from_u32(0), b"test complete"); client.wait_idle().await; running.abort(); let _ = running.await; @@ -851,7 +850,7 @@ async fn uring_workers_bound_stalled_handshakes() { /// A UDP hop to `server` that forwards one client's datagrams until the server /// first answers, and every answer back, so the server never sees the client /// finish its handshake. Returns the address to dial. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn stall_after_hello(server: SocketAddr) -> SocketAddr { let socket = tokio::net::UdpSocket::bind("127.0.0.1:0").await.expect("bind hop"); let addr = socket.local_addr().expect("hop address"); diff --git a/rs/moq-sock/src/bind.rs b/rs/moq-sock/src/bind.rs index ebd1e30851d..13471fb997b 100644 --- a/rs/moq-sock/src/bind.rs +++ b/rs/moq-sock/src/bind.rs @@ -27,7 +27,7 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); /// UDP socket buffer size requested in each direction, in bytes. /// /// A QUIC stack absorbs bursts in the kernel socket buffer: whatever doesn't fit -/// while the process is off the CPU is dropped before noq ever sees it, and +/// while the process is off the CPU is dropped before QUIC ever sees it, and /// congestion control reads those drops as congestion. The OS defaults are sized /// for a chatty TCP-era socket (208 KiB on Linux), which a single relay socket /// carrying every connection blows through in milliseconds. 8 MiB is roughly 64ms @@ -317,7 +317,7 @@ pub fn tcp(addr: SocketAddr) -> io::Result { /// A specific IPv6 address such as `[::1]` can never carry IPv4, so it is set /// v6-only instead, which Linux does on bind anyway. Windows keeps the cleared /// flag, so the socket claims an IPv4 side it lacks and then rejects the -/// IPv4-level options noq sets on a dual-stack socket with `WSAEINVAL`, failing +/// IPv4-level options the QUIC driver sets on a dual-stack socket with `WSAEINVAL`, failing /// endpoint setup. fn make_dual_stack(socket: &Socket, addr: SocketAddr) { let SocketAddr::V6(addr) = addr else { return }; @@ -364,7 +364,7 @@ mod tests { } /// A specific IPv6 address can't carry IPv4, so the socket must not claim to: - /// Windows rejects noq's IPv4 socket options on a dual-stack `[::1]` socket. + /// Windows rejects the QUIC driver's IPv4 socket options on a dual-stack `[::1]` socket. /// Linux forces v6-only on such a bind itself, so only Windows can regress. #[test] fn udp_ipv6_loopback_is_v6_only() { diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 45ad002eabc..9664cb7787d 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -13,8 +13,24 @@ keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] [features] -default = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] -noq = ["_transport", "_certs", "dep:web-transport-moq", "dep:reqwest", "watch", "dep:dns-lookup"] +default = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] +# QUIC and WebTransport over moq-quic, for `https://`, `moqt://`, and `moql://`. +quic = [ + "_transport", + "_certs", + "dep:moq-quic", + "dep:moq-clock", + "dep:http", + "dep:kio", + "dep:pin-project-lite", + "dep:rustc-hash", + "dep:rustls-native-certs", + "dep:socket2", + "dep:web-transport-proto", + "dep:reqwest", + "watch", + "dep:dns-lookup", +] # Server-side certificate handling: chains loaded from disk, self-signed generation, # in-memory identities, and per-SNI selection. Not one to enable on its own, since # nothing turns it into a listener: every QUIC backend needs it for its own TLS, and @@ -28,12 +44,12 @@ _transport = [] watch = ["dep:notify"] # Capture qlog traces (`--*-quic-qlog `). Off by default so production # builds do not pull in the qlog crate and its serde_json encoder. -qlog = ["web-transport-moq?/qlog"] +qlog = ["moq-quic?/qlog"] # The QUIC backend pulls in its crypto provider from these features (default-features # are off on the dependency), so a backend without aws-lc-rs/ring has no initial cipher. # A provider is also all it takes to sign and serve certificates, so each turns on # `_certs` even where there is no QUIC backend to serve them for. -aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "web-transport-moq?/aws-lc-rs"] +aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "moq-quic?/rustls-aws-lc-rs"] iroh = ["_transport", "dep:web-transport-iroh", "dep:web-transport-proto", "dep:noq-proto"] # LAN peer discovery over mDNS (the `mdns` module). mdns = ["dep:mdns-sd", "dep:hmac", "dep:sha2", "dep:subtle"] @@ -44,7 +60,7 @@ tcp = ["_transport", "dep:qmux", "dep:dns-lookup"] # Unix-domain-socket qmux transport (`unix://`), unix-only. Adds peer-credential # inspection on accept. Pulls in `tcp` (shares qmux's stream transport). uds = ["tcp", "qmux/uds"] -ring = ["_certs", "rustls/ring", "rcgen/ring", "web-transport-moq?/ring"] +ring = ["_certs", "rustls/ring", "rcgen/ring", "moq-quic?/rustls-ring"] [dependencies] bytes = "1" @@ -54,27 +70,37 @@ dns-lookup = { version = "4", optional = true } futures = "0.3" hex = { workspace = true } hmac = { workspace = true, optional = true } +http = { version = "1", optional = true } humantime = "2.3" +kio = { workspace = true, optional = true } mdns-sd = { workspace = true, optional = true } +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } +# The QUIC state machine the `quic` module drives; the crypto-provider features pick its provider. +moq-quic = { workspace = true, features = ["bloom"], optional = true } moq-sock = { workspace = true } -# iroh runs on upstream noq, not the moq-noq fork, and re-exports only the +# iroh runs on upstream noq rather than moq-quic, and re-exports only the # ControllerFactory trait, not the concrete congestion configs. Version matched to # the copy iroh pulls in. noq-proto = { workspace = true, optional = true } notify = { version = "8", optional = true } +pin-project-lite = { version = "0.2", optional = true } qmux = { workspace = true, features = ["wss", "tls", "tcp"], optional = true } rand = "0.10.1" rcgen = { version = "0.14", default-features = false, optional = true } reqwest = { version = "0.13", default-features = false, optional = true } +rustc-hash = { workspace = true, optional = true } rustls = "0.23" +# The WebTransport client trusts the OS store alongside its custom roots. +rustls-native-certs = { version = "0.8", optional = true } # OS-native server-cert verifier for the rustls backend. rustls-platform-verifier = "0.7" rustls-webpki = { version = "0.103", features = ["aws-lc-rs"], optional = true } serde = { version = "1", features = ["derive"] } serde_with = { version = "3", features = ["hex"] } sha2 = { workspace = true, optional = true } +socket2 = { workspace = true, optional = true } subtle = { workspace = true, optional = true } thiserror = "2" tikv-jemalloc-ctl = { version = "0.6", optional = true } @@ -87,7 +113,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = { version = "2", features = ["serde"] } usage = { workspace = true, features = ["config"] } web-transport-iroh = { workspace = true, optional = true } -web-transport-moq = { workspace = true, optional = true } web-transport-proto = { workspace = true, optional = true } web-transport-trait = { workspace = true } x509-parser = "0.18" diff --git a/rs/moq-tokio/README.md b/rs/moq-tokio/README.md index 4802a6760b6..3ae2db1988d 100644 --- a/rs/moq-tokio/README.md +++ b/rs/moq-tokio/README.md @@ -12,7 +12,7 @@ Tokio-based connection helpers for native [Media over QUIC](https://moq.dev) app Establishes MoQ connections over a few different transports, selectable via cargo features: -- **WebTransport** (HTTP/3) via [noq](https://crates.io/crates/noq) +- **WebTransport** (HTTP/3) via [moq-quic](https://crates.io/crates/moq-quic) - **Raw QUIC** with ALPN negotiation - **WebSocket** as a fallback when QUIC isn't available - **Iroh** P2P (`iroh` feature) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 20c71f8b585..66166b129bc 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -6,7 +6,7 @@ use crate::connection::Goaway; use crate::{Addrs, Backoff, Connection, Error}; -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] use std::future::Future; use url::Url; @@ -47,7 +47,7 @@ pub struct Client { /// raw TCP/UDS qmux and WebSocket). Resolved once in [`Client::new`] so the ALPN list /// can't diverge between transports. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -81,10 +81,10 @@ pub struct Client { #[cfg(feature = "websocket")] tls_host_name: Option, /// Only the TLS-based dials read this; the plaintext qmux transports have none. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls: rustls::ClientConfig, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "iroh")] iroh: Option, #[cfg(feature = "iroh")] @@ -98,7 +98,7 @@ impl Client { #[cfg(not(feature = "_transport"))] pub fn new(_config: Config) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -121,11 +121,11 @@ impl Client { // Only the rustls-backed transports use this. Iroh and the plaintext // qmux transports must not require a rustls crypto provider. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] let tls = config.tls.build()?; - #[cfg(feature = "noq")] - let noq = Some(crate::noq::NoqClient::new(&config, &quic)?); + #[cfg(feature = "quic")] + let quic = Some(crate::quic::Client::new(&config, &quic)?); let versions = config.versions(); // Read before the struct literal below moves fields out of `config`. @@ -145,7 +145,7 @@ impl Client { .with_versions(versions.clone()) .with_extensions(config.extensions), #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -168,10 +168,10 @@ impl Client { websocket: config.websocket, #[cfg(feature = "websocket")] tls_host_name, - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "iroh")] iroh: None, #[cfg(feature = "iroh")] @@ -278,13 +278,13 @@ impl Client { /// This closes at once, discarding stream data the peer has not acknowledged yet. /// Call [`Connection::close`] on each connection first to deliver it. /// - /// Only the noq endpoint is closed. WebSocket, TCP, and UDS sessions end when their + /// Only the QUIC endpoint is closed. WebSocket, TCP, and UDS sessions end when their /// [`Connection`] is dropped (the kernel closes the socket on exit), and an iroh /// endpoint passed to `with_iroh` is closed by its owner. pub async fn close(self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq { - noq.close().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic { + quic.close().await; } } @@ -315,7 +315,7 @@ impl Client { /// /// Errors if no transport feature is compiled in. #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -323,7 +323,7 @@ impl Client { )))] pub(crate) async fn dial(&self, _addr: crate::connect::Addr) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -334,7 +334,7 @@ impl Client { /// protocol driver is spawned on the current tokio runtime; the session /// closes once the last returned handle drops. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -362,7 +362,7 @@ impl Client { /// The moq client builder, advertising `path` in the SETUP when there is one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -376,7 +376,7 @@ impl Client { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -472,15 +472,15 @@ impl Client { return Ok(Dialed::new(session, crate::Transport::Iroh)); } - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { let quic_addr = addr.clone(); let binding = match quic_transport(&url) { crate::Transport::Quic => moq_net::transport::Binding::InBand, _ => moq_net::transport::Binding::Uri, }; let quic_handle = async { - noq.connect(&self.tls, quic_addr, &self.versions) + quic.connect(&self.tls, quic_addr, &self.versions) .await .map(|session| crate::transport::Session::new(session, binding)) .map_err(Error::from) @@ -530,7 +530,7 @@ impl Client { /// /// Only compiled when there is a QUIC dial to race: a WebSocket-only build connects /// over the fallback directly. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] async fn race_moq_connect( &self, moq: &moq_net::Client, @@ -574,7 +574,7 @@ impl Client { /// `None` when the result is empty, which means the same as omitting the parameter: the /// server's default path. A peer on published lite-05 rejects an empty value outright. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -600,7 +600,7 @@ fn request_target(url: &Url) -> Option { /// picked by ALPN negotiation rather than by the scheme; that dial reads the negotiated /// [`crate::iroh::Binding`] and calls [`request_target`] itself. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -623,9 +623,9 @@ fn setup_path(url: &Url) -> Option { } } -/// What a noq dial to `url` runs on: WebTransport for `https://` (and the `http://` +/// What a QUIC dial to `url` runs on: WebTransport for `https://` (and the `http://` /// bootstrap), raw QUIC for `moqt://` and `moql://`. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn quic_transport(url: &Url) -> crate::Transport { match url.scheme() { "moqt" | "moql" => crate::Transport::Quic, @@ -643,7 +643,7 @@ pub(crate) struct Dialed { impl Dialed { #[cfg_attr( not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -662,7 +662,7 @@ impl Dialed { /// from the host and port so URL userinfo never goes on the wire. `None` for every other /// scheme, and when the URL has no host, since an empty authority is meaningless. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -681,7 +681,7 @@ fn setup_authority(url: &Url) -> Option { } /// The arm that won a [`race_transport_connect`]; the other was dropped, cancelling it. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] enum TransportRace { Quic(Q), WebSocket(W), @@ -689,7 +689,7 @@ enum TransportRace { /// Race the QUIC dial to `url` against the WebSocket fallback, returning whichever /// transport comes up first. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] async fn race_transport_connect(url: &Url, quic: Q, websocket: W) -> crate::Result> where Q: Future>, @@ -751,7 +751,7 @@ where /// Auth is terminal only when both arms refused. A WebTransport-only endpoint /// answers the fallback with 403 while QUIC is still in flight, and reconnect /// treats is_auth() as terminal, so a mixed pair reports the retryable error. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] fn race_error(quic: Error, websocket: Error) -> Error { match (quic.is_auth(), websocket.is_auth()) { (false, false) => Error::TransportRace { @@ -764,7 +764,7 @@ fn race_error(quic: Error, websocket: Error) -> Error { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -791,25 +791,25 @@ mod tests { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn fixed_target_preserves_request_and_refuses_redirect() { check_fixed_redirect(false, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn one_shot_fixed_target_refuses_redirect() { check_fixed_redirect(true, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn unused_fixed_fallback_does_not_restrict_connected_target() { check_fixed_redirect(true, false).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] async fn check_fixed_redirect(once: bool, pinned: bool) { let mut listen = crate::listen::Config { bind: Some("127.0.0.1:0".parse().unwrap()), @@ -886,7 +886,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -937,7 +937,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -976,7 +976,7 @@ mod tests { /// The iroh dial derives its target here rather than through [`setup_path`], since /// only the negotiated binding says whether to send one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1351,7 +1351,7 @@ mod tests { assert_eq!(config.versions().alpns().len(), moq_net::ALPNS.len()); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_auth_error() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1367,7 +1367,7 @@ mod tests { assert!(matches!(value, super::TransportRace::WebSocket(1))); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_quic_after_websocket_forbidden() { let quic = async { @@ -1389,7 +1389,7 @@ mod tests { /// Inline rather than in `tests/` so each arm dials its own ephemeral port: the /// public connect sends the fallback to the QUIC port, which nothing reserves over /// TCP as well. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn websocket_forbidden_does_not_end_a_quic_connect() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -1431,14 +1431,14 @@ mod tests { // The same race `connect_inner` runs, except the fallback dials its own port, // as plain ws:// so the listener above can answer without TLS. - let noq = client.noq.clone().unwrap(); + let quic = client.quic.clone().unwrap(); let (tls, versions) = (client.tls.clone(), client.versions.clone()); let quic_addr: crate::connect::Addr = Url::parse(&format!("https://localhost:{quic_port}")).unwrap().into(); let ws_addr: crate::connect::Addr = Url::parse(&format!("http://localhost:{ws_port}")).unwrap().into(); // Hold QUIC until the fallback has been refused, so the 403 is always exercised. let quic = Box::pin(async move { forbid.await.unwrap().expect("fallback listener failed"); - noq.connect(&tls, quic_addr, &versions) + quic.connect(&tls, quic_addr, &versions) .await .map(|session| crate::transport::Session::new(session, moq_net::transport::Binding::Uri)) .map_err(Error::from) @@ -1460,7 +1460,7 @@ mod tests { /// QUIC wins while the fallback handshake is held until QUIC connects. /// Each transport owns its ephemeral port; this exercises the same race as /// connect_inner without depending on which real handshake runs faster. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tracing_test::traced_test] #[tokio::test] async fn broadcast_race_quic_wins() { @@ -1530,8 +1530,8 @@ mod tests { let client = client.with_subscriber(sub_origin); let dialer = client.clone(); let quic = Box::pin(async move { - let noq = dialer.noq.as_ref().expect("QUIC backend"); - let session = noq.connect(&dialer.tls, quic_addr, &dialer.versions).await?; + let quic = dialer.quic.as_ref().expect("QUIC backend"); + let session = quic.connect(&dialer.tls, quic_addr, &dialer.versions).await?; connected.send(()).expect("fallback listener alive"); Ok::<_, crate::Error>(crate::transport::Session::new( session, @@ -1577,7 +1577,7 @@ mod tests { .expect("server task failed"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_auth_when_both_refuse() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1589,7 +1589,7 @@ mod tests { assert!(err.is_auth(), "unexpected error: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_quic_error_when_websocket_forbidden() { let quic = async { @@ -1605,7 +1605,7 @@ mod tests { assert!(!err.is_auth(), "mixed auth/non-auth must stay retryable: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_non_auth_error() { let quic = async { Err::(Error::ConnectFailed) }; @@ -1619,14 +1619,14 @@ mod tests { /// The URL a race unit test dials. Each test runs in its own process under nextest, /// so the process-wide [`crate::websocket::won`] memo starts empty. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn race_url() -> Url { Url::parse("https://race.invalid/").unwrap() } /// QUIC winning a race clears WebSocket's earlier win, so the next dial gives QUIC /// its head start again. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_restores_its_head_start() { crate::websocket::remember(&race_url()); @@ -1644,7 +1644,7 @@ mod tests { } /// A race arm that never finishes, and reports through `dropped` once it is dropped. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn loser() -> ( impl Future, tokio::sync::oneshot::Receiver, @@ -1659,7 +1659,7 @@ mod tests { /// WebSocket winning cancels the QUIC dial before the race returns, so no QUIC /// session can come up behind the one WebSocket serves. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn websocket_winning_cancels_the_quic_dial() { let (quic, mut dropped) = loser::>(); @@ -1677,7 +1677,7 @@ mod tests { } /// QUIC winning cancels the WebSocket dial before the race returns. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_cancels_the_websocket_dial() { let quic = async { Ok("quic") }; @@ -1694,7 +1694,7 @@ mod tests { ); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_returns_when_quic_transport_connects() { let quic = async { Ok("quic") }; @@ -1721,7 +1721,7 @@ mod tests { /// Iroh carries no rustls state, so constructing its client must not require an /// application-installed crypto provider. - #[cfg(all(feature = "iroh", not(any(feature = "noq", feature = "websocket"))))] + #[cfg(all(feature = "iroh", not(any(feature = "quic", feature = "websocket"))))] #[test] fn iroh_only_client_does_not_require_a_tls_provider() { crate::connect::Config::default().init().expect("iroh-only client"); diff --git a/rs/moq-tokio/src/connect.rs b/rs/moq-tokio/src/connect.rs index 789efcb0df5..cd67fa66785 100644 --- a/rs/moq-tokio/src/connect.rs +++ b/rs/moq-tokio/src/connect.rs @@ -341,7 +341,7 @@ pub enum ConnectError { impl ConnectError { /// Only the transports that carry an HTTP status (WebTransport, WebSocket) can /// classify one; qmux over tcp/unix has no such response. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] pub(crate) fn from_status_u16(status: u16) -> Option { match status { 401 => Some(Self::Unauthorized), @@ -357,7 +357,7 @@ impl ConnectError { } } -#[cfg(all(test, any(feature = "noq", feature = "websocket")))] +#[cfg(all(test, any(feature = "quic", feature = "websocket")))] mod tests { use super::*; diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f98e88f8e1c..e44fbaa7600 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2070,7 +2070,7 @@ mod tests { } /// Invalid local configuration must end an unlimited reconnect loop without waiting. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_unsupported_scheme_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2088,7 +2088,7 @@ mod tests { } /// A URL host that parses but can never be a TLS server name ends an unlimited reconnect loop. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_invalid_tls_server_name_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2200,12 +2200,12 @@ mod tests { /// /// Holding is what decides the transport race without depending on timing: the /// QUIC dial cannot finish until the test says so. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] struct Forwarder { gate: tokio::sync::watch::Sender, } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] impl Forwarder { async fn start(front: tokio::net::UdpSocket, server: std::net::SocketAddr) -> Self { use std::sync::Arc; @@ -2264,7 +2264,7 @@ mod tests { /// When QUIC wins the race the WebSocket dial is cancelled at once: its socket /// closes without carrying a session, and the one session runs over QUIC. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn quic_winning_cancels_the_websocket_dial() { use tokio::io::AsyncReadExt; diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index e1dbb140643..21aff0f4dbe 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -206,10 +206,10 @@ pub enum Error { #[error(transparent)] Tls(Arc), - /// The noq backend failed. - #[cfg(feature = "noq")] + /// The QUIC backend failed. + #[cfg(feature = "quic")] #[error(transparent)] - Noq(Arc), + Quic(Arc), /// The Iroh backend failed. #[cfg(feature = "iroh")] @@ -252,8 +252,8 @@ impl Error { | Self::WindowRange(_) | Self::QlogUnsupported | Self::Connect(_) => true, - #[cfg(feature = "noq")] - Self::Noq(err) => err.is_terminal(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.is_terminal(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.is_terminal(), #[cfg(feature = "tcp")] @@ -285,8 +285,8 @@ impl Error { Self::MoqNet( moq_net::Error::Unauthorized | moq_net::Error::Session(moq_net::SessionError::Unauthorized), ) => Some(crate::ConnectError::Unauthorized), - #[cfg(feature = "noq")] - Self::Noq(err) => err.connect_error(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.connect_error(), #[cfg(feature = "websocket")] Self::TransportRace { quic, websocket } => quic.connect_error().or_else(|| websocket.connect_error()), #[cfg(feature = "websocket")] @@ -318,8 +318,8 @@ impl Error { _ => None, }, - #[cfg(feature = "noq")] - Self::Noq(err) => err.status(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.status(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.status(), _ => None, @@ -348,14 +348,14 @@ impl From for Error { } } -#[cfg(feature = "noq")] -impl From for Error { - fn from(err: crate::noq::Error) -> Self { +#[cfg(feature = "quic")] +impl From for Error { + fn from(err: crate::quic::Error) -> Self { if let Some(err) = err.connect_error() { return Self::Connect(err); } - Self::Noq(Arc::new(err)) + Self::Quic(Arc::new(err)) } } diff --git a/rs/moq-tokio/src/iroh.rs b/rs/moq-tokio/src/iroh.rs index 9e7350d1ade..058f33ca0c9 100644 --- a/rs/moq-tokio/src/iroh.rs +++ b/rs/moq-tokio/src/iroh.rs @@ -9,7 +9,7 @@ use std::{net, path::PathBuf, str::FromStr, sync::Arc}; use crate::quic::CongestionControl; use url::Url; use web_transport_iroh::iroh::{self, SecretKey}; -// NOTE: web-transport-iroh should re-export proto like web-transport-moq does. +// NOTE: web-transport-iroh should re-export proto like `crate::quic::web` does. use web_transport_proto::{ConnectRequest, ConnectResponse}; pub use iroh::Endpoint; @@ -112,7 +112,7 @@ pub enum Error { RecvRequest(String), /// GSO is always on for iroh, so `--quic-gso=false` cannot be honored. - #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the noq backend")] + #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the quic backend")] GsoUnsupported, } @@ -531,7 +531,7 @@ mod tests { /// Build a controller from each family's factory and downcast it to the /// concrete implementation it must map to. iroh runs on noq, so this is the - /// same pairing as the noq backend. + /// same pairing as the quic backend. #[test] fn congestion_factory_maps_each_family() { let now = std::time::Instant::now(); diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index 29a9b356e95..1dc43f9694a 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -11,7 +11,7 @@ //! See [`Client`] for connecting to relays and [`Server`] for accepting //! connections. The `mdns` feature finds peers to connect to on the local network. //! -//! With `default-features = false`, the `noq` backend must be paired +//! With `default-features = false`, the `quic` backend must be paired //! with the `aws-lc-rs` or `ring` crypto-provider feature. Every other subset //! compiles, including no transport at all: such a build cannot connect to //! anything, though a crypto provider on its own is still enough to configure TLS @@ -22,7 +22,7 @@ // The protocol crates need a compiled provider for reset and retry-token keys. A rustls provider // installed at runtime cannot supply constructors removed by their compile-time feature gates. -#[cfg(all(feature = "noq", not(any(feature = "aws-lc-rs", feature = "ring"))))] +#[cfg(all(feature = "quic", not(any(feature = "aws-lc-rs", feature = "ring"))))] compile_error!("a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature"); mod abort; @@ -34,18 +34,16 @@ pub mod connect; pub mod connection; pub mod crypto; mod error; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] pub mod failover; #[cfg(feature = "jemalloc")] pub mod jemalloc; pub mod listen; mod log; -#[cfg(feature = "noq")] -pub mod noq; pub mod origin; pub mod panic; pub mod quic; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] mod resolve; #[cfg(feature = "_transport")] pub mod server; @@ -60,7 +58,7 @@ pub mod unix; pub mod worker; // Resolving a `host:port` bind string is a QUIC-listener concern; the stream // listeners take a `SocketAddr`/path straight from their config. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod util; #[cfg(feature = "watch")] pub mod watch; diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 20e6fa4f7ab..16959fd75cd 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -14,7 +14,7 @@ pub enum Bind { } impl Bind { - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn resolve(&self) -> std::io::Result { match self { Self::Addr(addr) => Ok(*addr), @@ -186,11 +186,11 @@ pub struct Config { /// IPv4 address advertised as the QUIC preferred_address. /// - /// Supporting clients (Chrome M131+, native noq) migrate to this address + /// Supporting clients (Chrome M131+, this crate) migrate to this address /// shortly after the handshake completes. Typical use: handshake on an /// anycast IP, steady-state on this host's unicast IP. /// - /// Honored by noq. Accept-only, which is why it lives + /// Honored by the QUIC backend. Accept-only, which is why it lives /// here rather than in the shared [`crate::quic::Config`]. #[usage( name = "listen-preferred-v4", @@ -292,7 +292,7 @@ impl Default for Config { } } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) use moq_sock::shard::Shard; /// One server's socket in a complete `SO_REUSEPORT` group. /// @@ -527,7 +527,7 @@ impl Config { (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { self.lb_id @@ -666,7 +666,7 @@ mod tests { assert!("relay.example.com:443:8443".parse::().is_err()); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn cli_load_balancer_survives_the_merge_round_trip() { let config = config_from(["test", "--listen-quic-lb-id", "ab", "--listen-quic-lb-nonce", "9"]); @@ -726,7 +726,7 @@ load_balancer = { id = "cd", nonce = 4 } } /// Programmatic configuration keeps the QUIC-LB id and nonce paired. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn load_balancer_is_a_single_typed_value() { assert_eq!(Config::default().load_balancer(), None); diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index ddf33bbcbe1..5d8a5e4f198 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,7 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) - .add_directive("noq=info".parse()?) + .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) .add_directive("tracing::span=off".parse()?) diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 4935765246d..4b28ac64ef4 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -14,6 +14,29 @@ use std::time::Duration; use crate::cli::Duration as CliDuration; +#[cfg(feature = "quic")] +mod backend; +#[cfg(feature = "quic")] +pub use backend::Error; +#[cfg(feature = "quic")] +pub(crate) use backend::{Client, Options, Server, accept}; + +// The async QUIC layer driving `moq-quic` on tokio, forked from quinn (see its module docs). +// Public only so the workspace's tests can dial a bare endpoint; not a stable API. +// cargo fmt formats submodules with the crate root's config, so `just rs fix` +// formats this one separately with its own (quinn's). +#[cfg(feature = "quic")] +#[doc(hidden)] +#[rustfmt::skip] +pub mod driver; + +/// WebTransport over [`driver`], formerly the `web-transport-moq` crate. +/// +/// Public only so the workspace's tests can dial a bare session; not a stable API. +#[cfg(feature = "quic")] +#[doc(hidden)] +pub mod web; + /// The routable server ID a QUIC-LB load balancer encodes into connection IDs. /// /// Parsed from, and serialized as, a hex string. Its length must match the load @@ -55,7 +78,7 @@ pub struct LoadBalancer { /// The congestion control family for a QUIC connection. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. #[derive(Clone, Copy, Debug, PartialEq, Eq, usage::ValueEnum, serde::Serialize, serde::Deserialize)] #[usage(ignore_case)] #[serde(rename_all = "kebab-case")] @@ -87,7 +110,7 @@ pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Bounds how much unread data one peer can make us buffer across all of its streams, /// which the backends otherwise leave unlimited. Sized so a relay-to-relay session, which /// shares one connection, can still sustain several Gbps at intercontinental RTTs. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) const DEFAULT_RECEIVE_WINDOW: u64 = 64 << 20; /// Default idle timeout before an inactive connection is dropped. @@ -250,7 +273,7 @@ pub struct Config { /// Write qlog traces into this directory, which must already exist. /// - /// Noq writes one file per connection. + /// Each endpoint writes one file, shared by its connections. /// /// Requires the `qlog` feature; setting it errors at init otherwise. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -648,7 +671,7 @@ impl Resolved { /// Delay-based unless the operator says otherwise: BBR keeps queues short and the /// send rate steady enough for a live encoder to track, which is what this stack /// carries. - #[cfg_attr(not(any(feature = "noq", feature = "iroh")), allow(dead_code))] + #[cfg_attr(not(any(feature = "quic", feature = "iroh")), allow(dead_code))] pub(crate) fn congestion(&self) -> CongestionControl { self.congestion_control.unwrap_or(CongestionControl::Delay) } @@ -657,7 +680,7 @@ impl Resolved { /// /// Only meaningful once [`Config::validate`] has passed; a build without the /// `qlog` feature never gets here with a directory set. - #[cfg_attr(not(feature = "noq"), allow(dead_code))] + #[cfg_attr(not(feature = "quic"), allow(dead_code))] pub(crate) fn qlog_dir(&self) -> Option<&std::path::Path> { self.qlog.as_deref() } diff --git a/rs/moq-tokio/src/noq.rs b/rs/moq-tokio/src/quic/backend.rs similarity index 79% rename from rs/moq-tokio/src/noq.rs rename to rs/moq-tokio/src/quic/backend.rs index 27634e16481..b0b5b084148 100644 --- a/rs/moq-tokio/src/noq.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -1,5 +1,6 @@ -//! The noq QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +//! The QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +use super::{driver, web}; use crate::RedactedUrl; use crate::connect; use crate::listen; @@ -12,13 +13,12 @@ use std::pin::Pin; use std::sync::{Arc, Mutex, RwLock}; use std::task::{Context, Poll, Waker, ready}; use std::time::Duration; -use web_transport_moq::noq; -/// Attach a qlog factory writing into the configured directory, if any. +/// Attach a qlog stream writing into the configured directory, if any. /// -/// noq's factory opens one file per connection, named after its initial destination -/// connection ID, so nothing needs to be unique per endpoint here. -fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { +/// A [`driver::QlogStream`] is a shared handle: every connection on the endpoint writes into +/// it, tagged with its own qlog `group_id`, so this is one file per endpoint. +fn apply_qlog(transport: &mut driver::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { // `Client::validate` already rejected a directory this build can't honor, so the // block below is only reached where capture actually works. let Some(dir) = quic.qlog_dir() else { @@ -27,8 +27,21 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) #[cfg(feature = "qlog")] { - transport.qlog_from_path(dir, &format!("moq-{role}")); - tracing::info!(dir = %dir.display(), "writing qlog"); + // The pid and sequence keep endpoints sharing a directory from clobbering each other. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = dir.join(format!("moq-{role}-{}-{seq}.sqlog", std::process::id())); + let file = std::fs::File::create(&path).map_err(Error::CreateQlog)?; + + // Deliberately unbuffered: qlog's streamer only flushes on `finish_log`, which is + // never called per event, so a BufWriter would hold every trace in memory until the + // endpoint drops and lose the lot if the process is killed. Killing a stuck process + // is exactly when these traces are worth having. + let mut config = driver::QlogConfig::default(); + config.writer(Box::new(file)).title(Some(format!("moq-tokio {role}"))); + + transport.qlog_stream(config.into_stream()); + tracing::info!(path = %path.display(), "writing qlog"); } #[cfg(not(feature = "qlog"))] @@ -37,17 +50,17 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) Ok(()) } -/// Apply the resolved quic knobs to a noq transport config. -fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { +/// Apply the resolved quic knobs to a transport config. +fn apply_transport(transport: &mut driver::TransportConfig, quic: &Resolved) { transport.max_idle_timeout(Some(quic.idle_timeout.try_into().expect("idle timeout out of range"))); transport.keep_alive_interval(quic.keep_alive); - // noq enables MTU discovery by default; disable it unless asked. + // MTU discovery is enabled by default; disable it unless asked. if !quic.mtu_discovery { transport.mtu_discovery_config(None); } - let max_streams = noq::VarInt::from_u64(quic.max_streams).unwrap_or(noq::VarInt::MAX); + let max_streams = driver::VarInt::from_u64(quic.max_streams).unwrap_or(driver::VarInt::MAX); transport.max_concurrent_bidi_streams(max_streams); transport.max_concurrent_uni_streams(max_streams); @@ -62,28 +75,28 @@ fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { /// Apply the flow-control windows. The connection window defaults to ours, since the /// backend's is unlimited; the others stay at the backend default when unset. -fn apply_windows(transport: &mut noq::TransportConfig, quic: &Resolved) { +fn apply_windows(transport: &mut driver::TransportConfig, quic: &Resolved) { // Saturating rather than erroring: `Config::validate` already rejects a window // past the varint, so this only bites a `Resolved` built without it. let window = quic.receive_window.unwrap_or(crate::quic::DEFAULT_RECEIVE_WINDOW); - transport.receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); if let Some(window) = quic.stream_receive_window { - transport.stream_receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.stream_receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); } if let Some(window) = quic.send_window { transport.send_window(window); } } -/// The noq controller factory for a congestion control family. noq's BBR is v3. -fn congestion_factory(family: CongestionControl) -> Arc { +/// The controller factory for a congestion control family. moq-quic's BBR is v3. +fn congestion_factory(family: CongestionControl) -> Arc { match family { - CongestionControl::Loss => Arc::new(noq::congestion::CubicConfig::default()), - CongestionControl::Delay => Arc::new(noq::congestion::Bbr3Config::default()), + CongestionControl::Loss => Arc::new(driver::congestion::CubicConfig::default()), + CongestionControl::Delay => Arc::new(driver::congestion::Bbr3Config::default()), } } -/// Errors specific to the noq QUIC backend. +/// Errors specific to the QUIC backend. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -95,6 +108,11 @@ pub enum Error { #[error("failed to create QUIC endpoint")] CreateEndpoint(#[source] std::io::Error), + /// The qlog trace file couldn't be created, usually a missing directory. + #[cfg(feature = "qlog")] + #[error("failed to create qlog file")] + CreateQlog(#[source] std::io::Error), + /// No async runtime was found. Construct the client or server from within a tokio runtime. #[error("no async runtime")] NoRuntime, @@ -203,7 +221,7 @@ pub enum Error { message: String, /// The HTTP status the server answered the CONNECT with, when it answered with one. /// - /// Read at conversion time rather than kept as a `web-transport-moq` error, so the + /// Read at conversion time rather than kept as a [`web`] error, so the /// classification survives without that crate appearing in this crate's public API. status: Option, }, @@ -251,20 +269,20 @@ impl crate::failover::Aggregate for Error { } crate::error::from_message! { - noq::ConnectError => Connect, - noq::ConnectionError => Connection, - web_transport_moq::ServerError => Server, + driver::ConnectError => Connect, + driver::ConnectionError => Connection, + web::ServerError => Server, hex::FromHexError => InvalidFingerprint, } -impl From for Error { - fn from(_: noq::crypto::rustls::NoInitialCipherSuite) -> Self { +impl From for Error { + fn from(_: driver::crypto::rustls::NoInitialCipherSuite) -> Self { Self::NoInitialCipherSuite } } -impl From for Error { - fn from(err: web_transport_moq::ClientError) -> Self { +impl From for Error { + fn from(err: web::ClientError) -> Self { Self::Client { status: client_status(&err), message: crate::error::message(err), @@ -277,9 +295,9 @@ type Result = std::result::Result; // ── Client ────────────────────────────────────────────────────────── #[derive(Clone)] -pub(crate) struct NoqClient { - pub quic: noq::Endpoint, - pub transport: Arc, +pub(crate) struct Client { + pub quic: driver::Endpoint, + pub transport: Arc, /// Whether an `http://` URL may bootstrap a pin (see [crate::tls::Connect::allows_http_bootstrap]). pub http_bootstrap: bool, /// Optional TLS SNI / verification hostname override (from config). @@ -295,24 +313,25 @@ pub(crate) struct NoqClient { dual_stack: bool, } -impl NoqClient { +impl Client { pub fn new(config: &connect::Config, quic: &crate::quic::Config) -> Result { let resolved = config.resolve(); let socket = crate::bind::udp(crate::bind::Udp::new(resolved.bind)).map_err(Error::BindSocket)?; let dual_stack = crate::bind::udp_is_dual_stack(&socket); - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "client")?; let transport = Arc::new(transport); // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; - let endpoint_config = noq::EndpointConfig::default(); + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; + let endpoint_config = driver::EndpointConfig::default(); // Create the generic QUIC endpoint. - let quic = noq::Endpoint::new(endpoint_config, None, socket, runtime).map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new(endpoint_config, None, socket).map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -327,7 +346,7 @@ impl NoqClient { /// Close every connection, then wait until each has sent its close to the peer. pub async fn close(self) { - self.quic.close(noq::VarInt::from_u32(0), b"client shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"client shutdown"); // Not `wait_idle`, which also sits out each connection's 3 PTO closing // period: that only repeats the close to a peer that already has it. self.quic.wait_all_draining().await; @@ -338,7 +357,7 @@ impl NoqClient { tls: &rustls::ClientConfig, addr: crate::connect::Addr, versions: &moq_net::Versions, - ) -> Result { + ) -> Result { let mut url = addr.url().clone(); let mut config = tls.clone(); @@ -401,7 +420,7 @@ impl NoqClient { } let alpns: Vec> = match url.scheme() { - "https" => vec![web_transport_moq::ALPN.as_bytes().to_vec()], + "https" => vec![web::ALPN.as_bytes().to_vec()], "moqt" | "moql" => versions.alpns().iter().map(|alpn| alpn.as_bytes().to_vec()).collect(), _ => return Err(Error::InvalidScheme), }; @@ -409,8 +428,8 @@ impl NoqClient { config.alpn_protocols = alpns; config.key_log = Arc::new(rustls::KeyLogFile::new()); - let config: noq::crypto::rustls::QuicClientConfig = config.try_into()?; - let mut config = noq::ClientConfig::new(Arc::new(config)); + let config: driver::crypto::rustls::QuicClientConfig = config.try_into()?; + let mut config = driver::ClientConfig::new(Arc::new(config)); config.transport_config(self.transport.clone()); tracing::debug!(peer = %crate::connect::Endpoint(&url), "connecting"); @@ -429,15 +448,15 @@ impl NoqClient { let session = match url.scheme() { "https" => { - let mut request = web_transport_moq::proto::ConnectRequest::new(url.clone()); + let mut request = web::proto::ConnectRequest::new(url.clone()); for alpn in versions.alpns() { request = request.with_protocol(alpn.to_string()); } - web_transport_moq::Session::connect(connection, request) + web::Session::connect(connection, request) .await .map_err(map_client_error)? } - "moqt" | "moql" => web_transport_moq::Session::raw(connection), + "moqt" | "moql" => web::Session::raw(connection), _ => return Err(Error::UnsupportedScheme(url.scheme().to_string())), }; @@ -502,7 +521,7 @@ impl Error { } } -fn map_client_error(err: web_transport_moq::ClientError) -> Error { +fn map_client_error(err: web::ClientError) -> Error { match client_status(&err).and_then(crate::ConnectError::from_status_u16) { Some(rejected) => rejected.into(), None => err.into(), @@ -516,48 +535,49 @@ fn map_client_error(err: web_transport_moq::ClientError) -> Error { /// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults /// the status. A `404` or `405` is the server's settled answer, so retrying /// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_moq::ClientError) -> Option { +fn client_status(err: &web::ClientError) -> Option { match err { - web_transport_moq::ClientError::HttpError(err) => connect_status(err), + web::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn connect_status(err: &web_transport_moq::ConnectError) -> Option { +fn connect_status(err: &web::ConnectError) -> Option { match err { - web_transport_moq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), - web_transport_moq::ConnectError::ProtoError(err) => proto_status(err), + web::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn proto_status(err: &web_transport_moq::proto::ConnectError) -> Option { +fn proto_status(err: &web::proto::ConnectError) -> Option { match err { - web_transport_moq::proto::ConnectError::ErrorStatus(status) - | web_transport_moq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + web::proto::ConnectError::ErrorStatus(status) | web::proto::ConnectError::WrongStatus(Some(status)) => { + Some(status.as_u16()) + } _ => None, } } // ── Server ────────────────────────────────────────────────────────── -pub(crate) struct NoqServer { - pub quic: noq::Endpoint, +pub(crate) struct Server { + pub quic: driver::Endpoint, tls: crate::tls::Serve, socket: Socket, } -/// The server's UDP socket, which [`NoqServer::shutdown`] closes even while noq +/// The server's UDP socket, which [`Server::shutdown`] closes even while the driver /// still holds references to it. /// -/// noq gives each connection a sender sharing the socket, and a connection lives +/// The driver gives each connection a sender sharing the socket, and a connection lives /// as long as anything holds its handle, so dropping the endpoint leaves the port /// bound until every accepted session is gone too. Taking the socket out of this /// slot closes it regardless. #[derive(Clone, Debug)] struct Socket { io: Arc>>>, - state: Arc, + state: Arc, local: net::SocketAddr, /// Wakes every sender blocked on a full send buffer from the socket's single /// write-readiness registration. @@ -566,7 +586,7 @@ struct Socket { impl Socket { fn new(socket: std::net::UdpSocket) -> std::io::Result { - let state = noq::udp::UdpSocketState::new((&socket).into())?; + let state = moq_sock::udp::UdpSocketState::new((&socket).into())?; let local = socket.local_addr()?; let io = tokio::net::UdpSocket::from_std(socket)?; Ok(Self { @@ -588,8 +608,8 @@ impl Socket { } } -impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { +impl driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { Box::pin(self.clone()) } @@ -597,7 +617,7 @@ impl noq::AsyncUdpSocket for Socket { &mut self, cx: &mut Context<'_>, bufs: &mut [std::io::IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { let io = self.io.read().unwrap(); // Released: nothing arrives again, and the endpoint stops through its close. @@ -617,7 +637,7 @@ impl noq::AsyncUdpSocket for Socket { Ok(self.local) } - fn max_receive_segments(&self) -> std::num::NonZeroUsize { + fn max_receive_segments(&self) -> usize { self.state.gro_segments() } @@ -626,10 +646,10 @@ impl noq::AsyncUdpSocket for Socket { } } -impl noq::UdpSender for Socket { +impl driver::UdpSender for Socket { fn poll_send( self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, + transmit: &moq_sock::udp::Transmit<'_>, cx: &mut Context<'_>, ) -> Poll> { let io = self.io.read().unwrap(); @@ -638,6 +658,9 @@ impl noq::UdpSender for Socket { return Poll::Ready(Ok(())); }; loop { + // `send` logs and drops a failed datagram rather than failing the connection, as + // UDP loss would; only `WouldBlock` comes back. + #[allow(deprecated)] match io.try_io(tokio::io::Interest::WRITABLE, || self.state.send(io.into(), transmit)) { Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} res => return Poll::Ready(res), @@ -650,7 +673,7 @@ impl noq::UdpSender for Socket { } } - fn max_transmit_segments(&self) -> std::num::NonZeroUsize { + fn max_transmit_segments(&self) -> usize { self.state.max_gso_segments() } } @@ -687,7 +710,7 @@ pub(crate) struct Options { pub tls: crate::tls::Serve, } -impl NoqServer { +impl Server { pub fn new(options: Options) -> Result { let Options { config, @@ -695,7 +718,7 @@ impl NoqServer { member, tls: served, } = options; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "server")?; @@ -708,17 +731,17 @@ impl NoqServer { .iter() .map(|alpn| alpn.as_bytes().to_vec()) .collect(); - alpns.push(web_transport_moq::ALPN.as_bytes().to_vec()); + alpns.push(web::ALPN.as_bytes().to_vec()); let mut tls = (*served.config(alpns)?).clone(); tls.key_log = Arc::new(rustls::KeyLogFile::new()); - let tls: noq::crypto::rustls::QuicServerConfig = tls.try_into()?; - let mut tls = noq::ServerConfig::with_crypto(Arc::new(tls)); + let tls: driver::crypto::rustls::QuicServerConfig = tls.try_into()?; + let mut tls = driver::ServerConfig::with_crypto(Arc::new(tls)); tls.transport_config(transport); // Advertise the preferred_address transport parameter (RFC 9000 §9.6). - // noq allocates a fresh CID + reset token for the address during the handshake. + // The driver allocates a fresh CID + reset token for the address during the handshake. if let Some(addr) = config.preferred_v4 { tls.preferred_address_v4(Some(addr)); } @@ -727,7 +750,8 @@ impl NoqServer { } // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; let listen = config .bind @@ -739,7 +763,7 @@ impl NoqServer { let load_balancer = config.load_balancer(); // Configure connection ID generator with server ID if provided - let mut endpoint_config = noq::EndpointConfig::default(); + let mut endpoint_config = driver::EndpointConfig::default(); if let Some(shard) = member.as_ref().map(listen::Socket::shard) { if load_balancer.is_some() { return Err(Error::ShardWithQuicLb); @@ -781,9 +805,8 @@ impl NoqServer { // Create the generic QUIC endpoint. let socket = Socket::new(socket).map_err(Error::CreateEndpoint)?; - let quic = - noq::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone()), runtime) - .map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone())) + .map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -792,7 +815,7 @@ impl NoqServer { }) } - pub fn accept(&self) -> impl std::future::Future> + '_ { + pub fn accept(&self) -> impl std::future::Future> + '_ { self.quic.accept() } @@ -805,7 +828,7 @@ impl NoqServer { } pub fn close(&self) { - self.quic.close(noq::VarInt::from_u32(0), b"server shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"server shutdown"); } /// Close every connection, wait until each has sent its close to the peer, @@ -824,32 +847,34 @@ impl NoqServer { } } -// ── NoqRequest ────────────────────────────────────────────────────── +// ── QuicRequest ───────────────────────────────────────────────────── -/// A raw QUIC connection request without WebTransport framing (noq backend). +/// A raw QUIC connection request without WebTransport framing (QUIC backend). /// Accept a QUIC connection, negotiate WebTransport or raw moq, and complete the /// handshake (a `200 OK` for WebTransport). Returns the established session, the request /// URL and validated mTLS identity (both captured before the response consumes the /// request), and the dialed authority (the CONNECT authority on WebTransport, the TLS /// SNI on raw QUIC). Raw QUIC carries no request URL (the path rides the SETUP instead). pub(crate) async fn accept( - conn: noq::Incoming, + conn: driver::Incoming, alpns: Vec<&'static str>, -) -> Result> { +) -> Result> { let mut conn = conn.accept()?; - let handshake = conn - .handshake_data() - .await? - .downcast::() - .unwrap(); + // Scoped so the `!Send` boxed handshake data is gone before the next await. + let (alpn, host) = { + let handshake = conn + .handshake_data() + .await? + .downcast::() + .unwrap(); + (handshake.protocol, handshake.server_name) + }; - let alpn = handshake.protocol.ok_or(Error::MissingAlpn)?; + let alpn = alpn.ok_or(Error::MissingAlpn)?; let alpn = String::from_utf8(alpn)?; - let host = handshake.server_name.unwrap_or_default(); + let host = host.unwrap_or_default(); - // The established Connection no longer exposes a single peer address (noq 1.0 - // supports multipath), so capture it from the Connecting before awaiting. let remote = conn.remote_address(); tracing::debug!(%host, ip = %remote, %alpn, "accepting"); @@ -868,10 +893,10 @@ pub(crate) async fn accept( }; match alpn.as_str() { - web_transport_moq::ALPN => { + web::ALPN => { // Wait for the CONNECT request, then capture its URL and mTLS identity before // the response consumes it. - let request = web_transport_moq::Request::accept(conn) + let request = web::Request::accept(conn) .await .map_err(|err| Error::RecvRequest(crate::error::message(err)))?; let url = Some(request.url.clone()); @@ -879,7 +904,7 @@ pub(crate) async fn accept( // The authority the client put in its CONNECT URL. let authority = request.url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); - let mut response = web_transport_moq::proto::ConnectResponse::OK; + let mut response = web::proto::ConnectResponse::OK; let mut link = link; if let Some(protocol) = request.protocols.iter().find(|p| alpns.contains(&p.as_str())) { response = response.with_protocol(protocol); @@ -906,7 +931,7 @@ pub(crate) async fn accept( // Raw QUIC carries no request URL; the path rides the SETUP. The TLS SNI is the // only authority the client can offer here, and it is optional. let authority = (!host.is_empty()).then_some(host); - let session = web_transport_moq::Session::raw(conn); + let session = web::Session::raw(conn); Ok(crate::server::Accepted { session, url: None, @@ -939,13 +964,13 @@ impl ShardIdGenerator { } } -impl noq::ConnectionIdGenerator for ShardIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ShardIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let mut cid = Vec::with_capacity(Self::LEN); cid.push(moq_sock::shard::cid_prefix(self.shard)); cid.extend(rand::rng().random_iter::().take(Self::LEN - 1)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -970,8 +995,8 @@ impl ServerIdGenerator { } } -impl noq::ConnectionIdGenerator for ServerIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ServerIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let cid_len = self.cid_len(); let mut cid = Vec::with_capacity(cid_len); @@ -979,7 +1004,7 @@ impl noq::ConnectionIdGenerator for ServerIdGenerator { cid.push((cid_len - 1) as u8); cid.extend(self.server_id.0.iter()); cid.extend(rand::rng().random_iter::().take(self.nonce_len)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -1022,7 +1047,7 @@ mod tests { ..Default::default() }; let tls = config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config, quic: quic.clone(), member: None, @@ -1040,7 +1065,7 @@ mod tests { ..Default::default() }; let tls = config.tls.build().expect("tls config"); - let client = NoqClient::new(&config, &quic).expect("client init"); + let client = Client::new(&config, &quic).expect("client init"); let url: Url = format!("moqt://{addr}/.cluster/test").parse().unwrap(); let versions = moq_net::Versions::default(); let dial = client.connect(&tls, url.into(), &versions); @@ -1054,18 +1079,20 @@ mod tests { let (_client, _server) = result.expect("pinned connection failed"); } - /// noq exposes no getters for the flow-control windows, but its `Debug` prints + /// moq-quic exposes no getters for the flow-control windows, but its `Debug` prints /// them, which is enough to prove each one reached the transport config, that an - /// unset connection window gets our finite default instead of noq's unlimited one, - /// and that the other unset knobs leave noq's own defaults in place. + /// unset connection window gets our finite default instead of moq-quic's unlimited one, + /// and that the other unset knobs leave moq-quic's own defaults in place. #[test] fn apply_windows_writes_each_field() { - let mut quic = crate::quic::Config::default(); - quic.receive_window = Some(128 << 20); - quic.stream_receive_window = Some(8 << 20); - quic.send_window = Some(32 << 20); + let quic = crate::quic::Config { + receive_window: Some(128 << 20), + stream_receive_window: Some(8 << 20), + send_window: Some(32 << 20), + ..Default::default() + }; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); apply_windows(&mut transport, &quic.resolve()); let applied = format!("{:?}", transport); @@ -1077,33 +1104,33 @@ mod tests { assert!(applied.contains(&field), "{field} missing from {applied}"); } - let mut untouched = noq::TransportConfig::default(); + let mut untouched = driver::TransportConfig::default(); apply_windows(&mut untouched, &crate::quic::Config::default().resolve()); - let mut expected = noq::TransportConfig::default(); - expected.receive_window(noq::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); + let mut expected = driver::TransportConfig::default(); + expected.receive_window(driver::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); assert_eq!(format!("{:?}", untouched), format!("{:?}", expected)); assert_ne!( format!("{:?}", untouched), - format!("{:?}", noq::TransportConfig::default()), - "noq's own connection window is unlimited, so ours must replace it" + format!("{:?}", driver::TransportConfig::default()), + "moq-quic's own connection window is unlimited, so ours must replace it" ); } /// Build a controller from each family's factory and downcast it to the - /// concrete noq implementation it must map to. + /// concrete moq-quic implementation it must map to. #[test] fn congestion_factory_maps_each_family() { - let now = std::time::Instant::now(); + let now = moq_clock::Instant::now(); let mtu = 1200; let loss = congestion_factory(CongestionControl::Loss).build(now, mtu); - assert!(loss.into_any().downcast::().is_ok()); + assert!(loss.into_any().downcast::().is_ok()); let delay = congestion_factory(CongestionControl::Delay).build(now, mtu); - assert!(delay.into_any().downcast::().is_ok()); + assert!(delay.into_any().downcast::().is_ok()); } - /// Loopback regression test: with the knob unset, live noq connections must run + /// Loopback regression test: with the knob unset, live QUIC connections must run /// BBRv3 on both ends. #[tokio::test] async fn default_reaches_the_live_connection() { @@ -1120,7 +1147,7 @@ mod tests { let quic = crate::quic::Config::default(); let tls = server_config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config: server_config, quic: quic.clone(), member: None, @@ -1146,7 +1173,7 @@ mod tests { }; let tls = client_config.tls.build().expect("tls config"); - let client = NoqClient::new(&client_config, &quic).expect("client init"); + let client = Client::new(&client_config, &quic).expect("client init"); // Dial the loopback IP directly so the system resolver is never involved. let url: Url = format!("moqt://127.0.0.1:{}", addr.port()).parse().unwrap(); @@ -1158,7 +1185,7 @@ mod tests { .await .expect("connect failed"); - // web_transport_moq::Session derefs to the noq connection. + // web::Session derefs to the driver connection. assert!(is_bbr3(&session), "client connection is not running BBRv3"); assert!( accepted.await.expect("server task panicked"), @@ -1169,15 +1196,11 @@ mod tests { .expect("test timed out"); } - /// Whether a live connection's initial path is running BBRv3. - /// - /// noq is multipath, so the controller is per path rather than per connection; - /// `PathId::ZERO` is the path the handshake came up on. - fn is_bbr3(conn: &noq::Connection) -> bool { - conn.congestion_state(noq::PathId::ZERO) - .expect("no controller on the initial path") + /// Whether a live connection is running BBRv3. + fn is_bbr3(conn: &driver::Connection) -> bool { + conn.congestion_state() .into_any() - .downcast::() + .downcast::() .is_ok() } } diff --git a/rs/moq-tokio/src/quic/driver/.rustfmt.toml b/rs/moq-tokio/src/quic/driver/.rustfmt.toml new file mode 100644 index 00000000000..daf6479bc8a --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/.rustfmt.toml @@ -0,0 +1,2 @@ +# quinn's formatting (rustfmt defaults) rather than rs/.rustfmt.toml, so +# upstream fixes cherry-pick without whitespace conflicts. diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 6ce968db476..605163a6736 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -23,15 +23,11 @@ use tokio::sync::{ }; use tracing::{Instrument, Span, debug_span}; -use crate::{ - ConnectionEvent, Duration, Instant, VarInt, - mutex::Mutex, - recv_stream::RecvStream, - runtime::{AsyncTimer, Runtime, UdpSender}, - send_stream::SendStream, - udp_transmit, +use crate::quic::driver::{ + ConnectionEvent, Duration, Instant, VarInt, mutex::Mutex, recv_stream::RecvStream, + runtime::UdpSender, send_stream::SendStream, udp_transmit, }; -use proto::{ +use moq_quic::{ ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, StreamId, TransportError, TransportErrorCode, congestion::Controller, }; @@ -53,11 +49,10 @@ pub struct Connecting { impl Connecting { pub(crate) fn new( handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, sender: Pin>, - runtime: Arc, ) -> Self { let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); @@ -69,21 +64,20 @@ impl Connecting { conn_events, on_handshake_data_send, sender, - runtime.clone(), )), shared: Shared::default(), })); let connected = Box::pin(conn.shared.connected.clone().notified_owned()); let driver = ConnectionDriver::new(conn.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {e}"); } } .instrument(Span::current()), - )); + ); Self { conn: Some(conn), @@ -104,7 +98,7 @@ impl Connecting { /// ## Outgoing /// /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends - /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig] /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may @@ -119,14 +113,14 @@ impl Connecting { /// relevant resumption state to be stored in the server, which servers may limit or lose for /// various reasons including not persisting resumption state across server restarts. /// - /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// If manually providing a [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Incoming /// /// For incoming connections, conversion to 0.5-RTT will always succeed. /// - /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// If manually providing a [`crypto::ServerConfig`][crate::quic::driver::crypto::ServerConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Security @@ -156,9 +150,9 @@ impl Connecting { /// Parameters negotiated during the handshake /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a - /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// [`crypto::rustls::HandshakeData`](crate::quic::driver::crypto::rustls::HandshakeData). /// /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. /// @@ -194,7 +188,7 @@ impl Connecting { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. /// /// Will panic if called after `poll` has returned `Ready`. @@ -341,9 +335,9 @@ impl Connection { /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing /// anything to [`SendStream`] will never succeed. /// - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream pub fn open_bi(&self) -> OpenBi<'_> { OpenBi { conn: &self.0, @@ -369,10 +363,10 @@ impl Connection { /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. /// - /// [`accept_bi()`]: crate::Connection::accept_bi - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`accept_bi()`]: crate::quic::driver::Connection::accept_bi + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream /// /// # Cancel safety /// @@ -462,8 +456,8 @@ impl Connection { /// undelivered to the application, including data that was acknowledged as received to /// the local endpoint. /// - /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed - /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`ConnectionError::LocallyClosed`]: crate::quic::driver::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::quic::driver::Endpoint::wait_idle /// [`close()`]: Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let conn = &mut *self.0.state.lock("close"); @@ -515,7 +509,7 @@ impl Connection { if let Some(x) = &conn.error { return Err(SendDatagramError::ConnectionLost(x.clone())); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match conn.inner.datagrams().send(data, true) { Ok(()) => { conn.wake(); @@ -614,7 +608,7 @@ impl Connection { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. pub fn local_ip(&self) -> Option { self.0.state.lock("local_ip").inner.local_ip() @@ -678,7 +672,7 @@ impl Connection { /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on /// the returned value. /// - /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + /// [`Connection::handshake_data()`]: crate::quic::driver::Connecting::handshake_data pub fn handshake_data(&self) -> Option> { self.0 .state @@ -691,7 +685,7 @@ impl Connection { /// Cryptographic identity of the peer /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. /// /// For custom asynchronous validation (e.g. checking a certificate against an external @@ -738,7 +732,7 @@ impl Connection { output: &mut [u8], label: &[u8], context: &[u8], - ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + ) -> Result<(), moq_quic::crypto::ExportKeyingMaterialError> { self.0 .state .lock("export_keying_material") @@ -758,14 +752,14 @@ impl Connection { conn.wake(); } - /// See [`proto::TransportConfig::send_window()`] + /// See [`moq_quic::TransportConfig::send_window()`] pub fn set_send_window(&self, send_window: u64) { let mut conn = self.0.state.lock("set_send_window"); conn.inner.set_send_window(send_window); conn.wake(); } - /// See [`proto::TransportConfig::receive_window()`] + /// See [`moq_quic::TransportConfig::receive_window()`] pub fn set_receive_window(&self, receive_window: VarInt) { let mut conn = self.0.state.lock("set_receive_window"); conn.inner.set_receive_window(receive_window); @@ -970,7 +964,7 @@ impl Future for SendDatagram<'_> { if let Some(e) = &state.error { return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match state .inner .datagrams() @@ -1084,13 +1078,13 @@ pub(crate) struct Shared { } pub(crate) struct State { - pub(crate) inner: proto::Connection, + pub(crate) inner: moq_quic::Connection, driver: Option, handle: ConnectionHandle, on_handshake_data: Option>, connected: bool, handshake_confirmed: bool, - timer: Option>>, + timer: Option, timer_deadline: Option, conn_events: mpsc::UnboundedReceiver, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, @@ -1100,23 +1094,21 @@ pub(crate) struct State { /// Always set to Some before the connection becomes drained pub(crate) error: Option, sender: Pin>, - runtime: Arc, send_buffer: Vec, /// We buffer a transmit when the underlying I/O would block - buffered_transmit: Option, + buffered_transmit: Option, remote_address: watch::Sender, } impl State { #[allow(clippy::too_many_arguments)] fn new( - inner: proto::Connection, + inner: moq_quic::Connection, handle: ConnectionHandle, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, on_handshake_data: oneshot::Sender<()>, sender: Pin>, - runtime: Arc, ) -> Self { let remote_address = inner.remote_address(); @@ -1136,7 +1128,6 @@ impl State { stopped: FxHashMap::default(), error: None, sender, - runtime, send_buffer: Vec::new(), buffered_transmit: None, remote_address: watch::Sender::new(remote_address), @@ -1144,7 +1135,7 @@ impl State { } fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { - let now = self.runtime.now(); + let now = Instant::now(); let mut transmits = 0; let max_datagrams = self @@ -1241,7 +1232,7 @@ impl State { fn forward_app_events(&mut self, shared: &Shared) { while let Some(event) = self.inner.poll() { - use proto::Event::*; + use moq_quic::Event::*; match event { HandshakeDataReady => { if let Some(x) = self.on_handshake_data.take() { @@ -1310,7 +1301,7 @@ impl State { // Use the clock rather than the async timer to detect expiry: Sleep::poll // respects Tokio's cooperative budget and can return Pending for elapsed // deadlines. - let now = self.runtime.now(); + let now = Instant::now(); if now >= deadline { self.inner.handle_timeout(now); self.timer_deadline = None; @@ -1320,26 +1311,22 @@ impl State { match &mut self.timer { // Avoid resetting the timer when the deadline is unchanged. Some(delay) if self.timer_deadline != Some(deadline) => { - delay.as_mut().reset(deadline); + delay.reset(deadline); } None => { - self.timer = Some(self.runtime.new_timer(deadline)); + self.timer = Some(moq_clock::sleep_until(deadline)); } _ => {} } self.timer_deadline = Some(deadline); - let delay = self - .timer - .as_mut() - .expect("timer must exist in this state") - .as_mut(); - if delay.poll(cx).is_pending() { + let delay = self.timer.as_mut().expect("timer must exist in this state"); + if Pin::new(delay).poll(cx).is_pending() { return false; } // The deadline elapsed in the window between the clock check and poll. - self.inner.handle_timeout(self.runtime.now()); + self.inner.handle_timeout(Instant::now()); self.timer_deadline = None; true } @@ -1372,7 +1359,7 @@ impl State { } fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { - self.inner.close(self.runtime.now(), error_code, reason); + self.inner.close(Instant::now(), error_code, reason); self.terminate(ConnectionError::LocallyClosed, shared); self.wake(); } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2012e415ff6..2bae9031dc0 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -14,35 +14,24 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] -use crate::runtime::default_runtime; -use crate::{ +use crate::quic::driver::{ Instant, - runtime::{AsyncUdpSocket, Runtime, UdpSender}, + runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}, udp_transmit, }; use bytes::{Bytes, BytesMut}; -use pin_project_lite::pin_project; -use proto::{ - self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, - EndpointEvent, ServerConfig, +use moq_quic::{ + ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, EndpointEvent, + ServerConfig, }; -use rustc_hash::FxHashMap; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] +use moq_sock::udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; +use pin_project_lite::pin_project; +use rustc_hash::{FxHashMap, FxHashSet}; use socket2::{Domain, Protocol, Socket, Type}; use tokio::sync::{Notify, futures::Notified, mpsc}; use tracing::{Instrument, Span}; -use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; -use crate::{ +use crate::quic::driver::{ ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, }; @@ -56,7 +45,6 @@ use crate::{ #[derive(Debug, Clone)] pub struct Endpoint { pub(crate) inner: EndpointRef, - runtime: Arc, } impl Endpoint { @@ -77,11 +65,6 @@ impl Endpoint { /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn client(addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -90,13 +73,10 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), None, - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } @@ -119,11 +99,6 @@ impl Endpoint { /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never /// dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -132,26 +107,21 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), Some(config), - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } /// Construct an endpoint with arbitrary configuration and socket - #[cfg(not(wasm_browser))] pub fn new( config: EndpointConfig, server_config: Option, socket: std::net::UdpSocket, - runtime: Arc, ) -> io::Result { - let socket = runtime.wrap_udp_socket(socket)?; - Self::new_with_abstract_socket(config, server_config, socket, runtime) + let socket = wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket) } /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket @@ -162,32 +132,30 @@ impl Endpoint { config: EndpointConfig, server_config: Option, socket: Box, - runtime: Arc, ) -> io::Result { let addr = socket.local_addr()?; let allow_mtud = !socket.may_fragment(); let rc = EndpointRef::new( socket, - proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + moq_quic::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), addr.is_ipv6(), - runtime.clone(), ); let driver = EndpointDriver(rc.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {}", e); } } .instrument(Span::current()), - )); - Ok(Self { inner: rc, runtime }) + ); + Ok(Self { inner: rc }) } /// Get the next incoming connection attempt from a client /// /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] - /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// can be `await`ed to obtain the final [`Connection`](crate::quic::driver::Connection), or used to e.g. /// filter connection attempts or force address validation, or converted into an intermediate /// `Connecting` future which can be used to e.g. send 0.5-RTT data. pub fn accept(&self) -> Accept<'_> { @@ -252,22 +220,21 @@ impl Endpoint { let (ch, conn) = endpoint .inner - .connect(self.runtime.now(), config, addr, server_name)?; + .connect(Instant::now(), config, addr, server_name)?; let sender = endpoint.socket.create_sender(); endpoint.stats.outgoing_handshakes += 1; Ok(endpoint .recv_state .connections - .insert(ch, conn, sender, false, self.runtime.clone())) + .insert(ch, conn, sender, false)) } /// Switch to a new UDP socket /// /// See [`Endpoint::rebind_abstract()`] for details. - #[cfg(not(wasm_browser))] pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { - self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + self.rebind_abstract(wrap_udp_socket(socket)?) } /// Switch to a new UDP socket @@ -322,7 +289,7 @@ impl Endpoint { /// /// See [`Connection::close()`] for details. /// - /// [`Connection::close()`]: crate::Connection::close + /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); let mut endpoint = self.inner.state.lock().unwrap(); @@ -360,6 +327,32 @@ impl Endpoint { .await; } } + + /// Wait until no connection on the endpoint is still active + /// + /// A connection stops being active once its peer's close arrives, or once its closing + /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out + /// the closing period for a peer that already has the close, nor wait for the + /// [`Connection`](crate::quic::driver::Connection) handles to drop. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + /// [`wait_idle()`]: Endpoint::wait_idle + pub async fn wait_all_draining(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.recv_state.connections.all_draining() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.all_draining.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -400,7 +393,7 @@ impl Future for EndpointDriver { None => endpoint.driver = Some(cx.waker().clone()), } - let now = endpoint.runtime.now(); + let now = Instant::now(); let mut keep_going = false; keep_going |= endpoint.drive_recv(cx, now)?; keep_going |= endpoint.handle_events(cx, &self.0.shared); @@ -432,6 +425,8 @@ impl Drop for EndpointDriver { // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); + endpoint.recv_state.connections.draining.clear(); + self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. @@ -450,7 +445,7 @@ pub(crate) struct EndpointInner { impl EndpointInner { pub(crate) fn accept( &self, - incoming: proto::Incoming, + incoming: moq_quic::Incoming, server_config: Option>, ) -> Result { let mut response_buffer = Vec::new(); @@ -458,7 +453,7 @@ impl EndpointInner { // Phase 1: reserve endpoint state for the connection under the lock. let accepting = { let mut state = self.state.lock().unwrap(); - let now = state.runtime.now(); + let now = Instant::now(); match state .inner .start_accept(incoming, now, &mut response_buffer, server_config) @@ -487,12 +482,11 @@ impl EndpointInner { Ok((handle, conn)) => { state.stats.accepted_handshakes += 1; let sender = state.socket.create_sender(); - let runtime = state.runtime.clone(); let driver_lost = state.driver_lost; Ok(state .recv_state .connections - .insert(handle, conn, sender, driver_lost, runtime)) + .insert(handle, conn, sender, driver_lost)) } Err(error) => { if let Some(transmit) = error.response { @@ -509,7 +503,7 @@ impl EndpointInner { result } - pub(crate) fn refuse(&self, incoming: proto::Incoming) { + pub(crate) fn refuse(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.refused_handshakes += 1; let mut response_buffer = Vec::new(); @@ -517,7 +511,7 @@ impl EndpointInner { respond(transmit, &response_buffer, &mut state.sender); } - pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + pub(crate) fn retry(&self, incoming: moq_quic::Incoming) -> Result<(), moq_quic::RetryError> { let mut state = self.state.lock().unwrap(); let mut response_buffer = Vec::new(); let transmit = state.inner.retry(incoming, &mut response_buffer)?; @@ -525,7 +519,7 @@ impl EndpointInner { Ok(()) } - pub(crate) fn ignore(&self, incoming: proto::Incoming) { + pub(crate) fn ignore(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.ignored_handshakes += 1; state.inner.ignore(incoming); @@ -539,13 +533,12 @@ pub(crate) struct State { /// During an active migration, abandoned_socket receives traffic /// until the first packet arrives on the new socket. prev_socket: Option>, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, recv_state: RecvState, driver: Option, ipv6: bool, events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, driver_lost: bool, - runtime: Arc, stats: EndpointStats, default_client_config: Option, /// Connections in the process of being accepted @@ -556,13 +549,14 @@ pub(crate) struct State { pub(crate) struct Shared { incoming: Notify, idle: Notify, + all_draining: Notify, /// Number of live handles that can be used to initiate or handle I/O; excludes the driver ref_count: AtomicUsize, } impl State { fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { - let get_time = || self.runtime.now(); + let get_time = Instant::now; self.recv_state.recv_limiter.start_cycle(get_time); if let Some(socket) = &mut self.prev_socket { // We don't care about the `PollProgress` from old sockets. @@ -571,7 +565,6 @@ impl State { &mut self.inner, &mut **socket, &mut self.sender, - &*self.runtime, now, ); if poll_res.is_err() { @@ -583,7 +576,6 @@ impl State { &mut self.inner, &mut *self.socket, &mut self.sender, - &*self.runtime, now, ); self.recv_state.recv_limiter.finish_cycle(get_time); @@ -606,12 +598,22 @@ impl State { } }; + let connections = &mut self.recv_state.connections; + if event.is_draining() && connections.senders.contains_key(&ch) { + connections.draining.insert(ch); + } if event.is_drained() { - self.recv_state.connections.senders.remove(&ch); + connections.senders.remove(&ch); + connections.draining.remove(&ch); if self.is_idle() { shared.idle.notify_waiters(); } } + if (event.is_draining() || event.is_drained()) + && self.recv_state.connections.all_draining() + { + shared.all_draining.notify_waiters(); + } let Some(event) = self.inner.handle_event(ch, event) else { continue; }; @@ -642,7 +644,7 @@ impl Drop for State { } fn respond( - transmit: proto::Transmit, + transmit: moq_quic::Transmit, response_buffer: &[u8], sender: &mut Pin>, ) { @@ -692,11 +694,11 @@ fn respond( } #[inline] -fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { +fn proto_ecn(ecn: moq_sock::udp::EcnCodepoint) -> moq_quic::EcnCodepoint { match ecn { - udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, - udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, - udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + moq_sock::udp::EcnCodepoint::Ect0 => moq_quic::EcnCodepoint::Ect0, + moq_sock::udp::EcnCodepoint::Ect1 => moq_quic::EcnCodepoint::Ect1, + moq_sock::udp::EcnCodepoint::Ce => moq_quic::EcnCodepoint::Ce, } } @@ -704,6 +706,9 @@ fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { struct ConnectionSet { /// Senders for communicating with the endpoint's connections senders: FxHashMap>, + /// The connections in `senders` that are no longer active: closed by the peer, or past + /// their closing period + draining: FxHashSet, /// Stored to give out clones to new ConnectionInners sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, /// Set if the endpoint has been manually closed @@ -714,10 +719,9 @@ impl ConnectionSet { fn insert( &mut self, handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, sender: Pin>, driver_lost: bool, - runtime: Arc, ) -> Connecting { let (send, recv) = mpsc::unbounded_channel(); if let Some((error_code, reason)) = &self.close { @@ -735,12 +739,17 @@ impl ConnectionSet { self.senders.insert(handle, send); } } - Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + Connecting::new(handle, conn, self.sender.clone(), recv, sender) } fn is_empty(&self) -> bool { self.senders.is_empty() } + + /// Whether no connection is still active + fn all_draining(&self) -> bool { + self.senders.len() == self.draining.len() + } } fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { @@ -795,9 +804,8 @@ pub(crate) struct EndpointRef(Arc); impl EndpointRef { pub(crate) fn new( socket: Box, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, ipv6: bool, - runtime: Arc, ) -> Self { let (sender, events) = mpsc::unbounded_channel(); let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); @@ -806,6 +814,7 @@ impl EndpointRef { shared: Shared { incoming: Notify::new(), idle: Notify::new(), + all_draining: Notify::new(), ref_count: AtomicUsize::new(0), }, state: Mutex::new(State { @@ -818,7 +827,6 @@ impl EndpointRef { driver: None, driver_lost: false, recv_state, - runtime, stats: EndpointStats::default(), default_client_config: None, pending_accepts: 0, @@ -858,7 +866,7 @@ impl std::ops::Deref for EndpointRef { /// State directly involved in handling incoming packets struct RecvState { - incoming: VecDeque, + incoming: VecDeque, connections: ConnectionSet, recv_buf: Box<[u8]>, /// Reused buffer that received datagrams are copied into and split into owned slices. @@ -870,7 +878,7 @@ impl RecvState { fn new( sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, max_receive_segments: usize, - endpoint: &proto::Endpoint, + endpoint: &moq_quic::Endpoint, ) -> Self { let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) as usize @@ -879,6 +887,7 @@ impl RecvState { Self { connections: ConnectionSet { senders: FxHashMap::default(), + draining: FxHashSet::default(), sender, close: None, }, @@ -892,10 +901,9 @@ impl RecvState { fn poll_socket( &mut self, cx: &mut Context<'_>, - endpoint: &mut proto::Endpoint, + endpoint: &mut moq_quic::Endpoint, socket: &mut dyn AsyncUdpSocket, sender: &mut Pin>, - runtime: &dyn Runtime, now: Instant, ) -> Result { let mut received_connection_packet = false; @@ -990,7 +998,7 @@ impl RecvState { return Err(e); } } - if !self.recv_limiter.allow_work(|| runtime.now()) { + if !self.recv_limiter.allow_work(Instant::now) { return Ok(PollProgress { received_connection_packet, keep_going: true, diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs index b5d60a08ecc..64a08e493c6 100644 --- a/rs/moq-tokio/src/quic/driver/incoming.rs +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -6,10 +6,10 @@ use std::{ task::{Context, Poll}, }; -use proto::{ConnectionError, ConnectionId, ServerConfig}; +use moq_quic::{ConnectionError, ConnectionId, ServerConfig}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ connection::{Connecting, Connection}, endpoint::EndpointRef, }; @@ -19,7 +19,7 @@ use crate::{ pub struct Incoming(Option); impl Incoming { - pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + pub(crate) fn new(inner: moq_quic::Incoming, endpoint: EndpointRef) -> Self { Self(Some(State { inner, endpoint })) } @@ -111,7 +111,7 @@ impl Drop for Incoming { #[derive(Debug)] struct State { - inner: proto::Incoming, + inner: moq_quic::Incoming, endpoint: EndpointRef, } diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index 649801fff81..d758cc9c7c6 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -50,14 +50,11 @@ mod runtime; mod send_stream; mod work_limiter; -#[cfg(not(wasm_browser))] -pub(crate) use std::time::{Duration, Instant}; -#[cfg(wasm_browser)] -pub(crate) use web_time::{Duration, Instant}; +pub(crate) use moq_clock::Instant; +pub(crate) use std::time::Duration; -#[cfg(feature = "bloom")] -pub use proto::BloomTokenLog; -pub use proto::{ +pub use moq_quic::BloomTokenLog; +pub use moq_quic::{ AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, @@ -67,42 +64,37 @@ pub use proto::{ VarIntBoundsExceeded, Written, congestion, crypto, }; #[cfg(feature = "qlog")] -pub use proto::{QlogConfig, QlogStream}; -#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] -pub use rustls; -pub use udp; +pub use moq_quic::{QlogConfig, QlogStream}; -pub use crate::connection::{ +pub use crate::quic::driver::connection::{ AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, RemoteAddressWatcher, SendDatagram, SendDatagramError, }; -pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; -pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; -pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; -#[cfg(feature = "runtime-smol")] -pub use crate::runtime::SmolRuntime; -#[cfg(feature = "runtime-tokio")] -pub use crate::runtime::TokioRuntime; -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -pub use crate::runtime::default_runtime; -pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; -pub use crate::send_stream::{SendStream, StoppedError, WriteError}; +pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::quic::driver::recv_stream::{ + ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, +}; +#[cfg(test)] +pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; +pub use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender}; +pub use crate::quic::driver::send_stream::{SendStream, StoppedError, WriteError}; #[cfg(test)] mod tests; #[derive(Debug)] -enum ConnectionEvent { +pub(crate) enum ConnectionEvent { Close { error_code: VarInt, reason: bytes::Bytes, }, - Proto(proto::ConnectionEvent), + Proto(moq_quic::ConnectionEvent), Rebind(Pin>), } -fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { - udp::Transmit { +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> moq_sock::udp::Transmit<'a> { + moq_sock::udp::Transmit { destination: t.destination, ecn: t.ecn.map(udp_ecn), contents: buffer, @@ -111,11 +103,11 @@ fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { } } -fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { +fn udp_ecn(ecn: EcnCodepoint) -> moq_sock::udp::EcnCodepoint { match ecn { - EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, - EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, - EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + EcnCodepoint::Ect0 => moq_sock::udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => moq_sock::udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => moq_sock::udp::EcnCodepoint::Ce, } } diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs index 91962cb072f..d88d40e32a8 100644 --- a/rs/moq-tokio/src/quic/driver/mutex.rs +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -3,116 +3,6 @@ use std::{ ops::{Deref, DerefMut}, }; -#[cfg(feature = "lock_tracking")] -mod tracking { - use super::*; - use crate::{Duration, Instant}; - use std::collections::VecDeque; - use tracing::warn; - - #[derive(Debug)] - struct Inner { - last_lock_owner: VecDeque<(&'static str, Duration)>, - value: T, - } - - /// A Mutex which optionally allows to track the time a lock was held and - /// emit warnings in case of excessive lock times - pub(crate) struct Mutex { - inner: std::sync::Mutex>, - } - - impl Debug for Mutex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Debug::fmt(&self.inner, f) - } - } - - impl Mutex { - pub(crate) fn new(value: T) -> Self { - Self { - inner: std::sync::Mutex::new(Inner { - last_lock_owner: VecDeque::new(), - value, - }), - } - } - - /// Acquires the lock for a certain purpose - /// - /// The purpose will be recorded in the list of last lock owners - pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { - // We don't bother dispatching through Runtime::now because they're pure performance - // diagnostics. - let now = Instant::now(); - let guard = self.inner.lock().unwrap(); - - let lock_time = Instant::now(); - let elapsed = lock_time.duration_since(now); - - if elapsed > Duration::from_millis(1) { - warn!( - "Locking the connection for {} took {:?}. Last owners: {:?}", - purpose, elapsed, guard.last_lock_owner - ); - } - - MutexGuard { - guard, - start_time: lock_time, - purpose, - } - } - } - - pub(crate) struct MutexGuard<'a, T> { - guard: std::sync::MutexGuard<'a, Inner>, - start_time: Instant, - purpose: &'static str, - } - - impl Drop for MutexGuard<'_, T> { - fn drop(&mut self) { - if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { - self.guard.last_lock_owner.pop_back(); - } - - let duration = self.start_time.elapsed(); - - if duration > Duration::from_millis(1) { - warn!( - "Utilizing the connection for {} took {:?}", - self.purpose, duration - ); - } - - self.guard - .last_lock_owner - .push_front((self.purpose, duration)); - } - } - - impl Deref for MutexGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.guard.value - } - } - - impl DerefMut for MutexGuard<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.guard.value - } - } - - const MAX_LOCK_OWNERS: usize = 20; -} - -#[cfg(feature = "lock_tracking")] -pub(crate) use tracking::Mutex; - -#[cfg(not(feature = "lock_tracking"))] mod non_tracking { use super::*; @@ -159,5 +49,4 @@ mod non_tracking { } } -#[cfg(not(feature = "lock_tracking"))] pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index d3548a1a331..027349ae12d 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -6,11 +6,11 @@ use std::{ }; use bytes::Bytes; -use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; use thiserror::Error; use tokio::io::ReadBuf; -use crate::{VarInt, connection::ConnectionRef}; +use crate::quic::driver::{VarInt, connection::ConnectionRef}; /// A stream that can only be used to receive data /// @@ -42,12 +42,12 @@ use crate::{VarInt, connection::ConnectionRef}; /// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver /// will be bidirectional stream 0. /// -/// [`ReadError`]: crate::ReadError +/// [`ReadError`]: crate::quic::driver::ReadError /// [`stop()`]: RecvStream::stop -/// [`SendStream::finish`]: crate::SendStream::finish -/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`SendStream::finish`]: crate::quic::driver::SendStream::finish +/// [`WriteError::Stopped`]: crate::quic::driver::WriteError::Stopped /// [`id`]: RecvStream::id -/// [`Connection::accept_bi`]: crate::Connection::accept_bi +/// [`Connection::accept_bi`]: crate::quic::driver::Connection::accept_bi #[derive(Debug)] pub struct RecvStream { conn: ConnectionRef, @@ -104,7 +104,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read( &mut self, cx: &mut Context<'_>, @@ -126,7 +126,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read_buf( &mut self, cx: &mut Context<'_>, @@ -256,7 +256,7 @@ impl RecvStream { /// /// This operation is *not* cancel-safe. /// - /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + /// [`ReadToEndError::TooLong`]: crate::quic::driver::ReadToEndError::TooLong pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { ReadToEnd { stream: self, @@ -286,13 +286,13 @@ impl RecvStream { /// Check if this stream predates completion of the handshake on an incoming connection. /// /// True only if the stream was accepted before the handshake completed, which is only possible - /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// if you successfully called [`Connecting::into_0rtt`](crate::quic::driver::Connecting::into_0rtt) and the /// client chose to send 0-RTT data. /// /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application /// data may be a replay attack. To guard against this, applications should not execute /// non-idempotent operations until - /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + /// [`Connection::authenticated`](crate::quic::driver::Connection::authenticated) succeeds. pub fn is_0rtt(&self) -> bool { self.is_0rtt } @@ -306,7 +306,7 @@ impl RecvStream { /// /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was - /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// [`finish()`](crate::quic::driver::SendStream::finish)ed by the peer and all data has been received, after /// which it is no longer meaningful for the stream to be reset. /// /// This operation is cancel-safe. @@ -360,7 +360,7 @@ impl RecvStream { where T: FnMut(&mut Chunks<'_>) -> ReadStatus, { - use proto::ReadError::*; + use moq_quic::ReadError::*; use std::collections::hash_map::Entry; if self.all_data_read { @@ -432,11 +432,11 @@ impl RecvStream { enum ReadStatus { Readable(T), Finished(Option), - Failed(Option, proto::ReadError), + Failed(Option, moq_quic::ReadError), } -impl From<(Option, Option)> for ReadStatus { - fn from(status: (Option, Option)) -> Self { +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { match status { (read, None) => Self::Finished(read), (read, Some(e)) => Self::Failed(read, e), @@ -446,7 +446,7 @@ impl From<(Option, Option)> for ReadStatus { /// Future produced by [`RecvStream::read_to_end()`]. /// -/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +/// [`RecvStream::read_to_end()`]: crate::quic::driver::RecvStream::read_to_end struct ReadToEnd<'a> { stream: &'a mut RecvStream, buffer: ReadToEndBuffer, @@ -515,19 +515,6 @@ pub enum ReadToEndError { TooLong, } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncRead for RecvStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut [u8], - ) -> Poll> { - let mut buf = ReadBuf::new(buf); - ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; - Poll::Ready(Ok(buf.filled().len())) - } -} - impl tokio::io::AsyncRead for RecvStream { fn poll_read( self: Pin<&mut Self>, @@ -594,7 +581,7 @@ pub enum ReadError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -640,7 +627,7 @@ pub enum ResetError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -658,7 +645,7 @@ impl From for io::Error { /// Future produced by [`RecvStream::read()`]. /// -/// [`RecvStream::read()`]: crate::RecvStream::read +/// [`RecvStream::read()`]: crate::quic::driver::RecvStream::read struct Read<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -679,7 +666,7 @@ impl Future for Read<'_> { /// Future produced by [`RecvStream::read_exact()`]. /// -/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +/// [`RecvStream::read_exact()`]: crate::quic::driver::RecvStream::read_exact struct ReadExact<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -715,7 +702,7 @@ pub enum ReadExactError { /// Future produced by [`RecvStream::read_chunk()`]. /// -/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +/// [`RecvStream::read_chunk()`]: crate::quic::driver::RecvStream::read_chunk struct ReadChunk<'a> { stream: &'a mut RecvStream, max_length: usize, @@ -732,7 +719,7 @@ impl Future for ReadChunk<'_> { /// Future produced by [`RecvStream::read_chunks()`]. /// -/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +/// [`RecvStream::read_chunks()`]: crate::quic::driver::RecvStream::read_chunks struct ReadChunks<'a> { stream: &'a mut RecvStream, bufs: &'a mut [Bytes], diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs index 9898a0e879c..cc89f2f90e8 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/mod.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -1,5 +1,3 @@ -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -use std::sync::Arc; use std::{ fmt::{self, Debug}, future::Future, @@ -10,35 +8,9 @@ use std::{ time::Duration, }; -use udp::{RecvMeta, Transmit}; - -use crate::Instant; - -/// Abstracts I/O and timer operations for runtime independence -pub trait Runtime: Send + Sync + Debug + 'static { - /// Construct a timer that will expire at `i` - fn new_timer(&self, i: Instant) -> Pin>; - /// Drive `future` to completion in the background - #[track_caller] - fn spawn(&self, future: Pin + Send>>); - /// Convert `t` into the socket type used by this runtime - #[cfg(not(wasm_browser))] - fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; - /// Look up the current time - /// - /// Allows simulating the flow of time for testing. - fn now(&self) -> Instant { - Instant::now() - } -} +use moq_sock::udp::{RecvMeta, Transmit}; -/// Abstract implementation of an async timer for runtime independence -pub trait AsyncTimer: Send + Debug + 'static { - /// Update the timer to expire at `i` - fn reset(self: Pin<&mut Self>, i: Instant); - /// Check whether the timer has expired, and register to be woken if not - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; -} +use crate::quic::driver::Instant; /// Abstract implementation of a UDP socket for runtime independence pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { @@ -143,7 +115,6 @@ impl /// that resolves once the socket is write-ready. /// /// See also the bounds on this struct's [`UdpSender`] implementation. - #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { Self { socket: inner, @@ -215,7 +186,7 @@ fn log_sendmsg_error( ) { #[cfg(unix)] // Unix `EMSGSIZE` is expected for MTU probes. - if udp::is_msg_size_err(error) { + if moq_sock::udp::is_msg_size_err(error) { return; } @@ -253,41 +224,10 @@ trait UdpSenderHelperSocket: Send + Sync + 'static { fn max_transmit_segments(&self) -> usize; } -/// Automatically select an appropriate runtime from those enabled at compile time -/// -/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, -/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is -/// returned. Otherwise, `None` is returned. -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -#[allow(clippy::needless_return)] // Be sure we return the right thing -pub fn default_runtime() -> Option> { - #[cfg(feature = "runtime-tokio")] - { - if ::tokio::runtime::Handle::try_current().is_ok() { - return Some(Arc::new(TokioRuntime)); - } - } - - #[cfg(feature = "runtime-smol")] - { - return Some(Arc::new(SmolRuntime)); - } - - #[cfg(not(feature = "runtime-smol"))] - None -} - -#[cfg(feature = "runtime-tokio")] mod tokio; -#[cfg(feature = "runtime-tokio")] -pub use tokio::TokioRuntime; - -#[cfg(feature = "runtime-smol")] -mod smol; -#[cfg(feature = "runtime-smol")] -pub use smol::*; +pub(crate) use tokio::wrap_udp_socket; -#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +#[cfg(test)] mod tests { use std::{future, task::Waker}; diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs deleted file mode 100644 index b5ae3d0cf2d..00000000000 --- a/rs/moq-tokio/src/quic/driver/runtime/smol.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::{ - future::Future, - pin::Pin, - task::{Context, Poll}, - time::Instant, -}; -use std::{io, sync::Arc, task::ready}; - -use async_io::Async; -use async_io::Timer; - -use super::AsyncTimer; -use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for smol -#[derive(Debug)] -pub struct SmolRuntime; - -impl Runtime for SmolRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(Timer::at(t)) - } - - fn spawn(&self, future: Pin + Send>>) { - ::smol::spawn(future).detach(); - } - - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket::new(sock)?)) - } -} - -impl AsyncTimer for Timer { - fn reset(mut self: Pin<&mut Self>, t: Instant) { - self.set_at(t) - } - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx).map(|_| ()) - } -} - -#[derive(Debug, Clone)] -struct UdpSocket { - io: Arc>, - inner: Arc, -} - -impl UdpSocket { - fn new(sock: std::net::UdpSocket) -> io::Result { - Ok(Self { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(Async::new_nonblocking(sock)?), - }) - } -} - -impl UdpSenderHelperSocket for UdpSocket { - fn max_transmit_segments(&self) -> usize { - self.inner.max_gso_segments() - } - - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { - self.inner.try_send((&self.io).into(), transmit)?; - - Ok(()) - } -} - -impl AsyncUdpSocket for UdpSocket { - fn create_sender(&self) -> Pin> { - Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { - let socket = socket.clone(); - async move { socket.io.writable().await } - })) - } - - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], - ) -> Poll> { - loop { - ready!(self.io.poll_readable(cx))?; - match self.inner.recv((&self.io).into(), bufs, meta) { - Ok(res) => return Poll::Ready(Ok(res)), - Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} - Err(e) => return Poll::Ready(Err(e)), - } - } - } - - fn local_addr(&self) -> io::Result { - self.io.as_ref().as_ref().local_addr() - } - - fn may_fragment(&self) -> bool { - self.inner.may_fragment() - } - - fn max_receive_segments(&self) -> usize { - self.inner.gro_segments() - } -} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs index ac5dfd6f205..750bff8ac63 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -1,57 +1,26 @@ use std::{ - future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll, ready}, - time::Instant, }; -use tokio::{ - io::Interest, - time::{Sleep, sleep_until}, -}; - -use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for Tokio -#[derive(Debug)] -pub struct TokioRuntime; - -impl Runtime for TokioRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(sleep_until(t.into())) - } - - fn spawn(&self, future: Pin + Send>>) { - tokio::spawn(future); - } +use tokio::io::Interest; - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), - })) - } +use super::{AsyncUdpSocket, UdpSenderHelper, UdpSenderHelperSocket}; - fn now(&self) -> Instant { - tokio::time::Instant::now().into_std() - } -} - -impl AsyncTimer for Sleep { - fn reset(self: Pin<&mut Self>, t: Instant) { - Self::reset(self, t.into()) - } - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx) - } +/// Register `sock` with the current tokio runtime. +pub(crate) fn wrap_udp_socket(sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(moq_sock::udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) } #[derive(Debug, Clone)] struct UdpSocket { io: Arc, - inner: Arc, + inner: Arc, } impl UdpSenderHelperSocket for UdpSocket { @@ -59,7 +28,7 @@ impl UdpSenderHelperSocket for UdpSocket { self.inner.max_gso_segments() } - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + fn try_send(&self, transmit: &moq_sock::udp::Transmit<'_>) -> io::Result<()> { self.io.try_io(Interest::WRITABLE, || { self.inner.try_send((&self.io).into(), transmit) })?; @@ -80,7 +49,7 @@ impl AsyncUdpSocket for UdpSocket { &mut self, cx: &mut Context<'_>, bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { loop { ready!(self.io.poll_recv_ready(cx))?; diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 570db9ac817..e51af7cf259 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -6,10 +6,10 @@ use std::{ }; use bytes::Bytes; -use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use moq_quic::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ VarInt, connection::{ConnectionRef, State}, }; @@ -140,9 +140,9 @@ impl SendStream { write_fn: F, ) -> Poll> where - F: FnOnce(&mut proto::SendStream<'_>) -> Result, + F: FnOnce(&mut moq_quic::SendStream<'_>) -> Result, { - use proto::WriteError::*; + use moq_quic::WriteError::*; let mut conn = self.conn.state.lock("SendStream::poll_write"); if self.is_0rtt { conn.check_0rtt() @@ -315,25 +315,6 @@ fn send_stream_stopped( } } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncWrite for SendStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.poll_write(cx, buf).map_err(Into::into) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().finish().map_err(Into::into)) - } -} - impl tokio::io::AsyncWrite for SendStream { fn poll_write( self: Pin<&mut Self>, @@ -394,7 +375,7 @@ pub enum WriteError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -437,7 +418,7 @@ pub enum StoppedError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index e0c6f3ef9e0..0b284f17706 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1,8 +1,8 @@ -#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +#![cfg(any(feature = "aws-lc-rs", feature = "ring"))] -#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +#[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] use rustls::crypto::aws_lc_rs::default_provider; -#[cfg(feature = "rustls-ring")] +#[cfg(feature = "ring")] use rustls::crypto::ring::default_provider; use std::{ @@ -19,15 +19,16 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; -use crate::{Duration, Instant}; +use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}; +use crate::quic::driver::{Duration, Instant}; use bytes::Bytes; -use proto::{ +use moq_quic::{ ConnectionId, RandomConnectionIdGenerator, crypto::rustls::{QuicClientConfig, QuicServerConfig}, crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, transport_parameters::TransportParameters, }; +use moq_sock::udp::RecvMeta; use rand::{Rng, SeedableRng, rngs::StdRng}; use rustls::{ RootCertStore, @@ -41,7 +42,6 @@ use tokio::{ use tracing::instrument::Instrument as _; use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; -use udp::RecvMeta; use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; @@ -64,6 +64,7 @@ fn handshake_timeout() { let mut transport_config = TransportConfig::default(); transport_config .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .handshake_idle_timeout(IDLE_TIMEOUT) .initial_rtt(Duration::from_millis(10)); client_config.transport_config(Arc::new(transport_config)); @@ -78,7 +79,7 @@ fn handshake_timeout() { .unwrap() .await { - Err(crate::ConnectionError::TimedOut) => {} + Err(crate::quic::driver::ConnectionError::TimedOut) => {} Err(e) => panic!("unexpected error: {e:?}"), Ok(_) => panic!("unexpected success"), } @@ -119,7 +120,7 @@ async fn close_endpoint() { .unwrap(); endpoint.close(0u32.into(), &[]); match conn.await { - Err(crate::ConnectionError::LocallyClosed) => (), + Err(crate::quic::driver::ConnectionError::LocallyClosed) => (), Err(e) => panic!("unexpected error: {e}"), Ok(_) => { panic!("unexpected success"); @@ -134,7 +135,7 @@ fn local_addr() { let runtime = rt_basic(); let ep = { let _guard = runtime.enter(); - Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + Endpoint::new(Default::default(), None, socket).unwrap() }; assert_eq!( addr, @@ -254,7 +255,7 @@ async fn ip_blocking() { .await .expect_err("server should have blocked this"); assert!( - matches!(e, crate::ConnectionError::ConnectionClosed(_)), + matches!(e, crate::quic::driver::ConnectionError::ConnectionClosed(_)), "wrong error" ); }, @@ -299,8 +300,11 @@ impl EndpointFactory { fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); let transport_config = Arc::new(transport_config); - let mut server_config = - crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + let mut server_config = crate::quic::driver::ServerConfig::with_single_cert( + vec![self.cert.cert.der().clone()], + key, + ) + .unwrap(); server_config.transport_config(transport_config.clone()); let mut roots = RootCertStore::empty(); @@ -309,7 +313,6 @@ impl EndpointFactory { self.endpoint_config.clone(), Some(server_config), UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - Arc::new(TokioRuntime), ) .unwrap(); let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); @@ -448,43 +451,8 @@ impl AsyncUdpSocket for FailingSocket { } } -/// Poll a connection driver before its registration returns, once the endpoint has failed. -#[derive(Debug)] -struct DriverLossRuntime { - failure: Arc, - stopped_on_first_poll: AtomicBool, -} - -impl crate::runtime::Runtime for DriverLossRuntime { - fn new_timer(&self, deadline: Instant) -> Pin> { - TokioRuntime.new_timer(deadline) - } - - fn spawn(&self, mut future: Pin + Send>>) { - if self.failure.armed.load(Ordering::SeqCst) { - let mut cx = Context::from_waker(Waker::noop()); - if future.as_mut().poll(&mut cx).is_ready() { - self.stopped_on_first_poll.store(true, Ordering::SeqCst); - return; - } - } - TokioRuntime.spawn(future); - } - - fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { - TokioRuntime.wrap_udp_socket(socket) - } - - fn now(&self) -> Instant { - TokioRuntime.now() - } -} - fn localhost_socket() -> Box { - TokioRuntime - .wrap_udp_socket( - UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - ) + wrap_udp_socket(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) .unwrap() } @@ -492,7 +460,6 @@ fn blocking_server_pair( server_alpn: &[u8], client_alpn: &[u8], server_socket: Box, - server_runtime: Arc, ) -> (Endpoint, Endpoint, Arc) { let factory = EndpointFactory::new(); let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); @@ -503,7 +470,7 @@ fn blocking_server_pair( .with_single_cert(vec![factory.cert.cert.der().clone()], key) .unwrap(); server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; - let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + let mut server_config = crate::quic::driver::ServerConfig::with_crypto(Arc::new( QuicServerConfig::try_from(server_crypto).unwrap(), )); let blocker = Arc::new(HandshakeBlocker::default()); @@ -516,14 +483,20 @@ fn blocking_server_pair( factory.endpoint_config.clone(), Some(server_config), server_socket, - server_runtime, ) .unwrap(); let mut roots = RootCertStore::empty(); roots.add(factory.cert.cert.der().clone()).unwrap(); let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); - let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + // One classical key share keeps the ClientHello in the first Initial, so a handshake failure + // surfaces from `accept` itself. aws-lc-rs's default post-quantum share spills it into a + // second packet, which only reaches the connection after it is registered. + let mut provider = default_provider(); + provider + .kx_groups + .retain(|group| group.name() == rustls::NamedGroup::X25519); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(provider.into()) .with_protocol_versions(&[&rustls::version::TLS13]) .unwrap() .with_root_certificates(roots) @@ -544,7 +517,7 @@ async fn wait_for_blocked_handshake(blocker: Arc) { async fn accept_with_blocked_start_session( server: Endpoint, -) -> Result { +) -> Result { let incoming = server.accept().await.unwrap(); // The test crypto provider intentionally blocks in `start_session`, so run // accept on the blocking pool rather than parking a Tokio worker thread. @@ -559,8 +532,12 @@ struct BlockedHandshake { server: Endpoint, blocker: Arc, release_guard: HandshakeReleaseGuard, - accept: tokio::task::JoinHandle>, - connect: tokio::task::JoinHandle>, + accept: tokio::task::JoinHandle< + Result, + >, + connect: tokio::task::JoinHandle< + Result, + >, } fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { @@ -572,22 +549,7 @@ fn blocked_handshake_with_socket( client_alpn: &[u8], server_socket: Box, ) -> BlockedHandshake { - blocked_handshake_with_runtime( - server_alpn, - client_alpn, - server_socket, - Arc::new(TokioRuntime), - ) -} - -fn blocked_handshake_with_runtime( - server_alpn: &[u8], - client_alpn: &[u8], - server_socket: Box, - server_runtime: Arc, -) -> BlockedHandshake { - let (client, server, blocker) = - blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let (client, server, blocker) = blocking_server_pair(server_alpn, client_alpn, server_socket); let server_addr = server.local_addr().unwrap(); let accept = tokio::spawn({ @@ -610,8 +572,10 @@ fn blocked_handshake_with_runtime( } async fn await_connection( - task: tokio::task::JoinHandle>, -) -> Result { + task: tokio::task::JoinHandle< + Result, + >, +) -> Result { timeout(Duration::from_secs(5), task) .await .unwrap() @@ -713,7 +677,9 @@ fn wait_idle_returns_when_the_driver_is_dropped() { let _ = waiter.join(); } -async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { +async fn wait_closed( + conn: &crate::quic::driver::Connection, +) -> crate::quic::driver::ConnectionError { timeout(Duration::from_secs(5), conn.closed()) .await .unwrap() @@ -914,7 +880,8 @@ fn run_echo(args: EchoArgs) { let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); let cert = CertificateDer::from(cert.cert); let mut server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); server_config.transport = transport_config.clone(); let server_sock = UdpSocket::bind(args.server_addr).unwrap(); @@ -922,13 +889,7 @@ fn run_echo(args: EchoArgs) { let server = { let _guard = runtime.enter(); let _guard = error_span!("server").entered(); - Endpoint::new( - Default::default(), - Some(server_config), - server_sock, - Arc::new(TokioRuntime), - ) - .unwrap() + Endpoint::new(Default::default(), Some(server_config), server_sock).unwrap() }; let mut roots = RootCertStore::empty(); @@ -1144,7 +1105,8 @@ async fn rebind_recv() { client.set_default_client_config(client_config); let server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); let server = { let _guard = tracing::error_span!("server").entered(); Endpoint::server( @@ -1254,8 +1216,8 @@ async fn split_accept_failure_wakes_idle_waiter() { let error = await_connection(accept).await.unwrap_err(); assert!(matches!( error, - crate::ConnectionError::TransportError(error) - if error.code == proto::TransportErrorCode::crypto(0x78) + crate::quic::driver::ConnectionError::TransportError(error) + if error.code == moq_quic::TransportErrorCode::crypto(0x78) )); // Repolling wait_idle alone would hide a missing notification now that the endpoint is // idle. Verify that the waiter registered before the failure was actually woken. @@ -1291,18 +1253,30 @@ async fn split_accept_close_during_pending_accept() { match await_connection(accept).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )); } - Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + Err(err) => assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )), } match await_connection(connect).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } Err(err) => { - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } } @@ -1364,51 +1338,6 @@ async fn split_accept_driver_lost_during_handshake() { wait_idle(&client).await; } -#[tokio::test] -async fn split_accept_driver_lost_before_connection_spawn() { - let _guard = subscribe(); - let failure = Arc::new(SocketFailure::default()); - let runtime = Arc::new(DriverLossRuntime { - failure: failure.clone(), - stopped_on_first_poll: AtomicBool::new(false), - }); - let socket = Box::new(FailingSocket { - inner: localhost_socket(), - failure: failure.clone(), - }); - let BlockedHandshake { - client, - server, - blocker, - release_guard: _release_guard, - accept, - connect, - } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); - - wait_for_blocked_handshake(blocker.clone()).await; - failure.trigger(); - assert!( - timeout(Duration::from_secs(5), server.accept()) - .await - .unwrap() - .is_none() - ); - - blocker.release(); - // The runtime polls the connection driver before registration returns. It must observe the - // closed event channel immediately, before transmitting any handshake packets. - assert!(await_connection(accept).await.is_err()); - assert!( - runtime.stopped_on_first_poll.load(Ordering::SeqCst), - "connection driver ran before its event channel was closed" - ); - wait_idle(&server).await; - - client.close(0u32.into(), b"done"); - assert!(await_connection(connect).await.is_err()); - wait_idle(&client).await; -} - #[tokio::test] async fn remote_address_monitoring() { let _guard = subscribe(); @@ -1782,8 +1711,8 @@ async fn dropped_endpoint_cleans_up() { let _guard = subscribe(); let mut endpoint_factory = EndpointFactory::new(); - let cid_generator = Arc::new(|| -> Box { - Box::::default() + let cid_generator = Arc::new(|| -> Box { + Box::::default() }); endpoint_factory .endpoint_config diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs index c3c3d3551d1..eda6e92d6c4 100644 --- a/rs/moq-tokio/src/quic/driver/work_limiter.rs +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -1,4 +1,4 @@ -use crate::{Duration, Instant}; +use crate::quic::driver::{Duration, Instant}; /// Limits the amount of time spent on a certain type of work in a cycle /// diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs index c3dcee477b7..e38cbd62e97 100644 --- a/rs/moq-tokio/src/quic/web/client.rs +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -1,28 +1,28 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; -use crate::proto::ConnectRequest; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use noq::crypto::rustls::QuicClientConfig; +use crate::quic::driver::crypto::rustls::QuicClientConfig; +use crate::quic::web::proto::ConnectRequest; use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; use tokio::net::lookup_host; use url::Host; -use crate::crypto; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::ALPN; -use crate::{ClientError, Session}; +use crate::quic::web::ALPN; +use crate::quic::web::crypto; +use crate::quic::web::{ClientError, Session}; /// Congestion control algorithm to use for the connection. /// /// Different algorithms make different tradeoffs between throughput and latency. pub enum CongestionControl { - /// Use the default congestion control algorithm (typically CUBIC). - Default, - /// Optimize for throughput (typically CUBIC). - Throughput, - /// Optimize for low latency (typically BBR). - LowLatency, + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -31,136 +31,124 @@ pub enum CongestionControl { /// This is optional; advanced users may use [Client::new] directly. #[derive(Clone)] pub struct ClientBuilder { - provider: crypto::Provider, - congestion_controller: - Option>, + provider: crypto::Provider, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ClientBuilder { - /// Create a Client builder, which can be used to establish multiple [Session]s. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - congestion_controller: None, - } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Accept any certificate from the server if it uses a known root CA. - pub fn with_system_roots(self) -> Result { - let mut roots = rustls::RootCertStore::empty(); - - let native = rustls_native_certs::load_native_certs(); - - // Log any errors that occurred while loading the native root certificates. - for err in native.errors { - tracing::warn!(?err, "failed to load root cert"); - } - - // Add the platform's native root certificates. - for cert in native.certs { - if let Err(err) = roots.add(cert) { - tracing::warn!(?err, "failed to add root cert"); - } - } - - let crypto = self - .builder() - .with_root_certificates(roots) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Supply certificates for accepted servers instead of using root CAs. - pub fn with_server_certificates( - self, - certs: Vec, - ) -> Result { - let hashes = certs.iter().map({ - let provider = self.provider.clone(); - move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() - }); - - self.with_server_certificate_hashes(hashes.collect()) - } - - /// Supply sha256 hashes for accepted certificates instead of using root CAs. - pub fn with_server_certificate_hashes( - self, - hashes: Vec>, - ) -> Result { - // Use a custom fingerprint verifier. - let fingerprints = Arc::new(ServerFingerprints { - provider: self.provider.clone(), - fingerprints: hashes, - }); - - // Configure the crypto client. - let crypto = self - .builder() - .dangerous() - .with_custom_certificate_verifier(fingerprints.clone()) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Access dangerous configuration options. - /// - /// This method returns a builder that provides access to potentially insecure - /// TLS configurations. These options are opt-in and require explicit acknowledgment - /// through the builder pattern, making the security implications clear at the call site. - pub fn dangerous(self) -> DangerousClientBuilder { - DangerousClientBuilder { inner: self } - } - - fn builder(&self) -> rustls::ConfigBuilder { - rustls::ClientConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13]) - .unwrap() - } - - fn build(self, mut crypto: rustls::ClientConfig) -> Result { - crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; - - let client_config = QuicClientConfig::try_from(crypto).unwrap(); - let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); - - let mut transport = noq::TransportConfig::default(); - if let Some(cc) = &self.congestion_controller { - transport.congestion_controller_factory(cc.clone()); - } - - client_config.transport_config(transport.into()); - - let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); - Ok(Client { - endpoint: client, - config: client_config, - }) - } + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self.builder().with_root_certificates(roots).with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates(self, certs: Vec) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes(self, hashes: Vec>) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = crate::quic::driver::ClientConfig::new(Arc::new(client_config)); + + let mut transport = crate::quic::driver::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = crate::quic::driver::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ClientBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -170,209 +158,182 @@ impl Default for ClientBuilder { /// These options should only be used when you understand the security implications, /// such as in local development or over a secure VPN connection. pub struct DangerousClientBuilder { - inner: ClientBuilder, + inner: ClientBuilder, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl DangerousClientBuilder { - /// Disable certificate verification entirely. - /// - /// This makes the connection vulnerable to man-in-the-middle attacks. - /// Only use this in secure environments, such as in local development or over a VPN connection. - /// - /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the - /// security sense, hence the explicit `dangerous()` builder requirement. - pub fn with_no_certificate_verification(self) -> Result { - let noop = NoCertificateVerification(self.inner.provider.clone()); - - let crypto = self - .inner - .builder() - .dangerous() - .with_custom_certificate_verifier(Arc::new(noop)) - .with_no_client_auth(); - - self.inner.build(crypto) - } + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } } /// A client for connecting to a WebTransport server. #[derive(Clone, Debug)] pub struct Client { - endpoint: noq::Endpoint, - config: noq::ClientConfig, + endpoint: crate::quic::driver::Endpoint, + config: crate::quic::driver::ClientConfig, } impl Client { - /// Manually create a client via a Noq endpoint and config. - /// - /// The ALPN MUST be set to [ALPN]. - pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { - Self { endpoint, config } - } - - /// Connect to the server. - pub async fn connect( - &self, - request: impl Into, - ) -> Result { - let request = request.into(); - - let port = request.url.port().unwrap_or(443); - - // TODO error on username:password in host - let (host, remote) = match request - .url - .host() - .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? - { - Host::Domain(domain) => { - let domain = domain.to_string(); - // Look up the DNS entry. - let mut remotes = match lookup_host((domain.clone(), port)).await { - Ok(remotes) => remotes, - Err(_) => return Err(ClientError::InvalidDnsName(domain)), - }; - - // Return the first entry. - let remote = match remotes.next() { - Some(remote) => remote, - None => return Err(ClientError::InvalidDnsName(domain)), - }; - - (domain, remote) - } - Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), - Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), - }; - - // Connect to the server using the addr we just resolved. - let conn = self - .endpoint - .connect_with(self.config.clone(), remote, &host)?; - let conn = conn.await?; - - // Connect with the connection we established. - Session::connect(conn, request).await - } + /// Manually create a client via a QUIC endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: crate::quic::driver::Endpoint, config: crate::quic::driver::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect(&self, request: impl Into) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self.endpoint.connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for Client { - fn default() -> Self { - ClientBuilder::new().with_system_roots().unwrap() - } + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } } #[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] #[derive(Debug)] struct ServerFingerprints { - provider: crypto::Provider, - fingerprints: Vec>, + provider: crypto::Provider, + fingerprints: Vec>, } impl ServerCertVerifier for ServerFingerprints { - fn verify_server_cert( - &self, - end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - let cert_hash = crypto::sha256(&self.provider, end_entity); - if self - .fingerprints - .iter() - .any(|fingerprint| fingerprint == cert_hash.as_ref()) - { - return Ok(rustls::client::danger::ServerCertVerified::assertion()); - } - - Err(rustls::Error::InvalidCertificate( - rustls::CertificateError::UnknownIssuer, - )) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.provider - .signature_verification_algorithms - .supported_schemes() - } + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider.signature_verification_algorithms.supported_schemes() + } } #[derive(Debug)] pub struct NoCertificateVerification(Arc); impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer<'_>, - _intermediates: &[CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.0.signature_verification_algorithms.supported_schemes() - } + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } } diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 4307355d3f9..471f5e416c7 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -6,153 +6,145 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum ConnectError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::ConnectError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("http error status: {0}")] - ErrorStatus(http::StatusCode), + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), - #[error("server returned protocol not in request: {0}")] - ProtocolMismatch(String), + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), } /// An HTTP/3 CONNECT request/response for establishing a WebTransport session. pub struct Connecting { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connecting { - pub async fn accept(conn: &noq::Connection) -> Result { - // Accept the stream that will be used to send the HTTP CONNECT request. - // If they try to send any other type of HTTP request, we will error out. - let (send, mut recv) = conn.accept_bi().await?; - - let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; - tracing::debug!(?request, "received CONNECT request"); - - // The request was successfully decoded, so we can send a response. - Ok(Self { - request, - send, - recv, - }) - } - - // Called by the server to send a response to the client and establish the session. - pub async fn respond( - mut self, - response: impl Into, - ) -> Result { - let response = response.into(); - - // Validate that our protocol was in the client's request. - if let Some(protocol) = &response.protocol { - if !self.request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - tracing::debug!(?response, "sending CONNECT response"); - response.write(&mut self.send).await?; - - Ok(Connected { - request: self.request, - response, - send: self.send, - recv: self.recv, - }) - } - - pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { - let mut connect = self.respond(status).await?; - connect.send.finish().ok(); - Ok(()) - } + pub async fn accept(conn: &crate::quic::driver::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { request, send, recv }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond(mut self, response: impl Into) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol + && !self.request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } } impl Deref for Connecting { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.request - } + fn deref(&self) -> &Self::Target { + &self.request + } } pub struct Connected { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // The response sent by the server. - pub response: ConnectResponse, + // The response sent by the server. + pub response: ConnectResponse, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connected { - /// Open a new WebTransport session on the given connection for the given URL. - /// - /// You may add any number of subprotocols allowing the server to select from. - /// If the list is empty the field will be omitted in the request header. - pub async fn open( - conn: &noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Create a new stream that will be used to send the CONNECT frame. - let (mut send, mut recv) = conn.open_bi().await?; - - tracing::debug!(?request, "sending CONNECT request"); - request.write(&mut send).await?; - - let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; - tracing::debug!(?response, "received CONNECT response"); - - // Throw an error if we didn't get a 200 OK. - if response.status != http::StatusCode::OK { - return Err(ConnectError::ErrorStatus(response.status)); - } - - // Validate that the server's protocol was in our request. - if let Some(protocol) = &response.protocol { - if !request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - Ok(Self { - request, - response, - send, - recv, - }) - } - - // The session ID is the stream ID of the CONNECT request. - pub fn session_id(&self) -> VarInt { - // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. - // We don't use the noq::VarInt because that would mean a noq dependency in - // web-transport-proto - let stream_id = noq::VarInt::from(self.send.id()); - VarInt::try_from(stream_id.into_inner()).unwrap() - } + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol + && !request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // Convert from the QUIC VarInt to the WebTransport VarInt, since web-transport-proto + // doesn't depend on moq-quic. + let stream_id = crate::quic::driver::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } } diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs index 1b8bb8ecf11..a1581975c97 100644 --- a/rs/moq-tokio/src/quic/web/crypto.rs +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -5,8 +5,8 @@ use std::sync::Arc; -use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::crypto::CryptoProvider; +use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::pki_types::CertificateDer; /// A shared reference to a crypto provider. @@ -22,25 +22,23 @@ pub type Provider = Arc; /// Panics if no provider is available. Either call `CryptoProvider::set_default()` /// or enable exactly one of the `ring` or `aws-lc-rs` features. pub fn default_provider() -> Provider { - // See - if let Some(provider) = CryptoProvider::get_default().cloned() { - return provider; - } + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } - #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] - { - return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); - } - #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] - { - return Arc::new(rustls::crypto::ring::default_provider()); - } - #[allow(unreachable_code)] - { - panic!( - "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." - ); - } + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!("CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature."); + } } /// Computes the SHA-256 hash of a certificate using the provided crypto provider. @@ -49,19 +47,19 @@ pub fn default_provider() -> Provider { /// /// Panics if the provider doesn't expose a SHA-256 hash algorithm. pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { - let hash_provider = provider.cipher_suites.iter().find_map(|suite| { - let hash_provider = suite.tls13()?.common.hash_provider; - if hash_provider.algorithm() == HashAlgorithm::SHA256 { - Some(hash_provider) - } else { - None - } - }); - if let Some(hash_provider) = hash_provider { - return hash_provider.hash(cert); - } + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } - panic!( - "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." - ); + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); } diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 654260fc4ce..5697bf684ec 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -2,70 +2,68 @@ use std::sync::{Arc, OnceLock}; use thiserror::Error; -use crate::{ConnectError, SettingsError}; +use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] pub enum ClientError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error: {0}")] - Connection(#[from] noq::ConnectionError), + #[error("connection error: {0}")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write: {0}")] - WriteError(#[from] noq::WriteError), + #[error("failed to write: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read: {0}")] - ReadError(#[from] noq::ReadError), + #[error("failed to read: {0}")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings: {0}")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect: {0}")] - HttpError(#[from] ConnectError), + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), - #[error("quic error: {0}")] - NoqError(#[from] noq::ConnectError), + #[error("quic error: {0}")] + ConnectError(#[from] crate::quic::driver::ConnectError), - #[error("invalid DNS name: {0}")] - InvalidDnsName(String), + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } -/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] pub enum SessionError { - #[error("connection error: {0}")] - ConnectionError(noq::ConnectionError), + #[error("connection error: {0}")] + ConnectionError(crate::quic::driver::ConnectionError), - #[error("webtransport error: {0}")] - WebTransportError(#[from] WebTransportError), + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), - #[error("send datagram error: {0}")] - SendDatagramError(#[from] noq::SendDatagramError), + #[error("send datagram error: {0}")] + SendDatagramError(#[from] crate::quic::driver::SendDatagramError), } -impl From for SessionError { - fn from(e: noq::ConnectionError) -> Self { - match &e { - noq::ConnectionError::ApplicationClosed(close) => { - match web_transport_proto::error_from_http3(close.error_code.into_inner()) { - Some(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - None => SessionError::ConnectionError(e), - } - } - _ => SessionError::ConnectionError(e), - } - } +impl From for SessionError { + fn from(e: crate::quic::driver::ConnectionError) -> Self { + match &e { + crate::quic::driver::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } } /// Why a session closed, shared by the session and its streams. The first close wins. @@ -74,192 +72,191 @@ impl From for SessionError { /// codes are encoded. #[derive(Debug)] pub(crate) struct CloseReason { - // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. - raw: bool, - reason: OnceLock, + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, } impl CloseReason { - pub(crate) fn new(raw: bool) -> Self { - Self { - raw, - reason: OnceLock::new(), - } - } - - /// The QUIC code to reset or stop a stream with. - pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { - if self.raw { - return code.into(); - } - noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() - } - - /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. - pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { - let code = code.into_inner(); - // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that - // range, so the mapped form cannot be mistaken for a raw code. - match web_transport_proto::error_from_http3(code) { - Some(code) => Some(code), - None if self.raw => u32::try_from(code).ok(), - None => None, - } - } - - /// Record the close reason, returning false if one was already recorded. - pub(crate) fn set(&self, err: SessionError) -> bool { - self.reason.set(err).is_ok() - } - - /// Replace an error caused by the connection closing with the recorded reason, or - /// decode the peer's raw QUIC close code when none was recorded. - pub(crate) fn map(&self, err: SessionError) -> SessionError { - let conn = match &err { - SessionError::ConnectionError(conn) - | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { - Some(conn) - } - SessionError::WebTransportError(WebTransportError::Closed(..)) => None, - _ => return err, - }; - - if let Some(reason) = self.reason.get() { - return reason.clone(); - } - - match conn { - Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { - match u32::try_from(close.error_code.into_inner()) { - Ok(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - Err(_) => err, - } - } - _ => err, - } - } + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> crate::quic::driver::VarInt { + if self.raw { + return code.into(); + } + crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: crate::quic::driver::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(crate::quic::driver::SendDatagramError::ConnectionLost(conn)) => Some(conn), + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(crate::quic::driver::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + Err(_) => err, + } + } + _ => err, + } + } } /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] pub enum WebTransportError { - #[error("closed: code={0} reason={1}")] - Closed(u32, String), + #[error("closed: code={0} reason={1}")] + Closed(u32, String), - #[error("unknown session")] - UnknownSession, + #[error("unknown session")] + UnknownSession, - #[error("read error: {0}")] - ReadError(#[from] noq::ReadExactError), + #[error("read error: {0}")] + ReadError(#[from] crate::quic::driver::ReadExactError), - #[error("write error: {0}")] - WriteError(#[from] noq::WriteError), + #[error("write error: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), } -/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +/// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] pub enum WriteError { - #[error("STOP_SENDING: {0}")] - Stopped(u32), + #[error("STOP_SENDING: {0}")] + Stopped(u32), - #[error("invalid STOP_SENDING: {0}")] - InvalidStopped(noq::VarInt), + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(crate::quic::driver::VarInt), - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("stream closed")] - ClosedStream, + #[error("stream closed")] + ClosedStream, } -impl From for WriteError { - fn from(e: noq::WriteError) -> Self { - match e { - noq::WriteError::Stopped(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - } - } - noq::WriteError::ClosedStream => WriteError::ClosedStream, - noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), - noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for WriteError { + fn from(e: crate::quic::driver::WriteError) -> Self { + match e { + crate::quic::driver::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + crate::quic::driver::WriteError::ClosedStream => WriteError::ClosedStream, + crate::quic::driver::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + crate::quic::driver::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } } -/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +/// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] pub enum ReadError { - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("RESET_STREAM: {0}")] - Reset(u32), + #[error("RESET_STREAM: {0}")] + Reset(u32), - #[error("invalid RESET_STREAM: {0}")] - InvalidReset(noq::VarInt), + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(crate::quic::driver::VarInt), - #[error("stream already closed")] - ClosedStream, + #[error("stream already closed")] + ClosedStream, } -impl From for ReadError { - fn from(value: noq::ReadError) -> Self { - match value { - noq::ReadError::Reset(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - } - } - noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), - noq::ReadError::ClosedStream => ReadError::ClosedStream, - noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for ReadError { + fn from(value: crate::quic::driver::ReadError) -> Self { + match value { + crate::quic::driver::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + crate::quic::driver::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + crate::quic::driver::ReadError::ClosedStream => ReadError::ClosedStream, + crate::quic::driver::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + crate::quic::driver::ReadError::IllegalOrderedRead => { + unreachable!("WebTransport streams are only read in order") + } + } + } } -/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] pub enum ReadExactError { - #[error("finished early")] - FinishedEarly(usize), + #[error("finished early")] + FinishedEarly(usize), - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadExactError { - fn from(e: noq::ReadExactError) -> Self { - match e { - noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), - noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), - } - } +impl From for ReadExactError { + fn from(e: crate::quic::driver::ReadExactError) -> Self { + match e { + crate::quic::driver::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + crate::quic::driver::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } } -/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] pub enum ReadToEndError { - #[error("too long")] - TooLong, + #[error("too long")] + TooLong, - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadToEndError { - fn from(e: noq::ReadToEndError) -> Self { - match e { - noq::ReadToEndError::TooLong => ReadToEndError::TooLong, - noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), - } - } +impl From for ReadToEndError { + fn from(e: crate::quic::driver::ReadToEndError) -> Self { + match e { + crate::quic::driver::ReadToEndError::TooLong => ReadToEndError::TooLong, + crate::quic::driver::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } } /// An error indicating the stream was already closed. @@ -267,39 +264,39 @@ impl From for ReadToEndError { #[error("stream closed")] pub struct ClosedStream; -impl From for ClosedStream { - fn from(_: noq::ClosedStream) -> Self { - ClosedStream - } +impl From for ClosedStream { + fn from(_: crate::quic::driver::ClosedStream) -> Self { + ClosedStream + } } /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] pub enum ServerError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error")] - Connection(#[from] noq::ConnectionError), + #[error("connection error")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write")] - WriteError(#[from] noq::WriteError), + #[error("failed to write")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read")] - ReadError(#[from] noq::ReadError), + #[error("failed to read")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect")] - ConnectError(#[from] ConnectError), + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), - #[error("io error: {0}")] - IoError(Arc), + #[error("io error: {0}")] + IoError(Arc), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } // #[derive(Clone, Error, Debug)] @@ -317,89 +314,89 @@ pub enum ServerError { // SessionError(#[from] SessionError), // } -// impl From for SendDatagramError { -// fn from(value: noq::SendDatagramError) -> Self { +// impl From for SendDatagramError { +// fn from(value: crate::quic::driver::SendDatagramError) -> Self { // match value { -// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, -// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, -// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, -// noq::SendDatagramError::ConnectionLost(e) => +// crate::quic::driver::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// crate::quic::driver::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// crate::quic::driver::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// crate::quic::driver::SendDatagramError::ConnectionLost(e) => // SendDatagramError::SessionError(e.into()), } // } // } impl web_transport_trait::Error for SessionError { - fn session_error(&self) -> Option<(u32, String)> { - if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { - return Some((*code, reason.to_string())); - } + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } - None - } + None + } } impl web_transport_trait::Error for WriteError { - fn session_error(&self) -> Option<(u32, String)> { - if let WriteError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - WriteError::Stopped(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } } impl web_transport_trait::Error for ReadError { - fn session_error(&self) -> Option<(u32, String)> { - if let ReadError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - ReadError::Reset(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } } #[cfg(test)] mod tests { - use super::*; - - /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its - /// range is not a WebTransport code. - #[test] - fn http3_stream_codes_stay_mapped() { - let close = CloseReason::new(false); - let mapped = close.encode_stream_code(5); - assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); - assert_eq!(close.decode_stream_code(mapped), Some(5)); - assert_eq!(close.decode_stream_code(5u32.into()), None); - } - - /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. - #[test] - fn raw_stream_codes_pass_through() { - let close = CloseReason::new(true); - assert_eq!(close.encode_stream_code(5).into_inner(), 5); - assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); - assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); - - let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); - assert_eq!(close.decode_stream_code(legacy), Some(5)); - - // Past u32 and outside the HTTP/3 range: neither form. - let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); - assert_eq!(close.decode_stream_code(invalid), None); - } + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = crate::quic::driver::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } } diff --git a/rs/moq-tokio/src/quic/web/fan.rs b/rs/moq-tokio/src/quic/web/fan.rs new file mode 100644 index 00000000000..0962877fcdc --- /dev/null +++ b/rs/moq-tokio/src/quic/web/fan.rs @@ -0,0 +1,140 @@ +use std::sync::{Arc, Mutex}; +use std::task::{Wake, Waker}; + +use kio::{Waiter, WaiterList}; + +/// A shared [`WaiterList`] that a [`Waker`] can wake. +/// +/// The accept futures below take one `Waker` and keep it. Handing them a caller's waker +/// would make the latest caller own the wakeup for everyone, so they are polled with +/// [`Fan::waker`] instead, which outlives every caller and wakes the whole list. +#[derive(Clone, Default)] +pub(crate) struct Fan(Arc); + +#[derive(Default)] +struct Inner(Mutex); + +#[derive(Default)] +struct State { + list: WaiterList, + /// Outstanding [`Hold`]s; a wake while any is alive is owed instead of delivered. + held: usize, + owed: bool, +} + +impl Fan { + /// Park `waiter` until the next wake. + pub(crate) fn register(&self, waiter: &Waiter) { + waiter.register(&mut self.0.0.lock().unwrap().list); + } + + /// A waker that wakes every parked waiter. + pub(crate) fn waker(&self) -> Waker { + Waker::from(self.0.clone()) + } + + /// Wake every parked waiter, or owe the wake until the last [`Hold`] drops. + pub(crate) fn wake(&self) { + self.0.notify(); + } + + /// Defer wakes until the guard drops, so none is delivered under the caller's lock. + pub(crate) fn hold(&self) -> Hold { + self.0.0.lock().unwrap().held += 1; + Hold(self.0.clone()) + } +} + +impl Inner { + fn notify(&self) { + let mut list = { + let mut state = self.0.lock().unwrap(); + if state.held > 0 { + state.owed = true; + return; + } + state.list.take() + }; + // Outside the lock: a waker may resume its task inline. + list.wake(); + } +} + +impl Wake for Inner { + fn wake(self: Arc) { + self.notify(); + } + + fn wake_by_ref(self: &Arc) { + self.notify(); + } +} + +/// Defers a [`Fan`]'s wakes while alive, delivering any owed one on drop. +pub(crate) struct Hold(Arc); + +impl Drop for Hold { + fn drop(&mut self) { + let owed = { + let mut state = self.0.0.lock().unwrap(); + state.held -= 1; + state.held == 0 && std::mem::take(&mut state.owed) + }; + if owed { + self.0.notify(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hold_defers_the_wake_until_dropped() { + let fan = Fan::default(); + let (waker, count) = counting(); + let waiter = Waiter::new(waker); + fan.register(&waiter); + + let hold = fan.hold(); + fan.waker().wake_by_ref(); + assert_eq!(count.load(), 0, "held back"); + + drop(hold); + assert_eq!(count.load(), 1, "delivered on drop"); + } + + #[test] + fn waker_wakes_every_waiter() { + let fan = Fan::default(); + let (a, count_a) = counting(); + let (b, count_b) = counting(); + // The list holds waiters weakly, so they have to outlive the wake. + let (a, b) = (Waiter::new(a), Waiter::new(b)); + fan.register(&a); + fan.register(&b); + + fan.wake(); + assert_eq!((count_a.load(), count_b.load()), (1, 1)); + } + + struct Count(std::sync::atomic::AtomicUsize); + + impl Count { + fn load(&self) -> usize { + self.0.load(std::sync::atomic::Ordering::SeqCst) + } + } + + impl Wake for Count { + fn wake(self: Arc) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + fn counting() -> (Waker, Arc) { + let count = Arc::new(Count(Default::default())); + (Waker::from(count.clone()), count) + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs index e5f37064927..47bf299eaee 100644 --- a/rs/moq-tokio/src/quic/web/mod.rs +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -3,7 +3,7 @@ //! //! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. //! This library hides that detail and tries to expose only the QUIC API, delegating as much as -//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! possible to the underlying implementation. See [`crate::quic::driver`] for more documentation. //! //! QUIC provides two primary APIs: //! @@ -43,12 +43,13 @@ pub use session::*; // Internal mod connect; +mod fan; mod settings; use connect::*; use settings::*; -// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +// Required to access proto::ConnectError wrapped in ClientError pub use connect::ConnectError; /// The HTTP/3 ALPN is required when negotiating a QUIC connection. @@ -57,15 +58,8 @@ pub const ALPN: &str = "h3"; /// Export our simple crypto provider. pub mod crypto; -/// Re-export the underlying QUIC implementation. -pub use noq; - /// Re-export the http crate because it's in the public API. pub use http; -/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of -/// [`SessionAccept`]'s poll methods holds its place in the accept queue. -pub use kio; - /// Re-export the WebTransport protocol implementation. pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 4c3d1881722..1e46bac784b 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -1,159 +1,131 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::Bytes; -use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; +use crate::quic::web::{CloseReason, ReadError, ReadExactError, ReadToEndError, SessionError}; -/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +/// A stream that can be used to receive bytes. See [`crate::quic::driver::RecvStream`]. #[derive(Debug)] pub struct RecvStream { - inner: noq::RecvStream, - close: Arc, + inner: crate::quic::driver::RecvStream, + close: Arc, } impl RecvStream { - pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { - Self { - inner: stream, - close, - } - } - - /// Decode the peer's reset code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::ReadError) -> ReadError { - match e { - noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - }, - e => match ReadError::from(e) { - ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Tell the other end to stop sending data with the given error code. See - /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error - /// space with HTTP/3. A raw QUIC session sends the code as is. - pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { - let code = self.close.encode_stream_code(code); - self.inner.stop(code) - } - - // Unfortunately, we have to wrap ReadError for a bunch of functions. - - /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - self.inner.read(buf).await.map_err(|e| self.map_error(e)) - } - - /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. - pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { - self.inner.read_exact(buf).await.map_err(|e| match e { - noq::ReadExactError::ReadError(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - self.inner - .read_chunk(max_length) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. - pub async fn read_many_chunks( - &mut self, - bufs: &mut [Bytes], - ) -> Result, ReadError> { - self.inner - .read_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. - pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { - self.inner - .read_to_end(size_limit) - .await - .map_err(|e| match e { - noq::ReadToEndError::Read(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Block until the stream has been reset and return the error code. See - /// [`noq::RecvStream::received_reset`]. - /// - /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. - pub async fn received_reset(&mut self) -> Result, SessionError> { - match self.inner.received_reset().await { - Ok(None) => Ok(None), - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), - Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.inner.id() - } - - /// Returns the number of bytes read from this stream. - /// - /// This is the offset of the next byte to be read, i.e. the length of the contiguous - /// prefix of the stream consumed by the application. - pub fn bytes_read(&self) -> Result { - self.inner.bytes_read().map_err(|_| ClosedStream) - } - - // We purposely don't expose the 0RTT because it's not valid with WebTransport + pub(crate) fn new(stream: crate::quic::driver::RecvStream, close: Arc) -> Self { + Self { inner: stream, close } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::ReadError) -> ReadError { + match e { + crate::quic::driver::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`crate::quic::driver::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), crate::quic::driver::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`crate::quic::driver::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`crate::quic::driver::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + crate::quic::driver::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`crate::quic::driver::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length, true) + .await + .map(|chunk| chunk.map(|chunk| chunk.bytes)) + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`crate::quic::driver::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner.read_to_end(size_limit).await.map_err(|e| match e { + crate::quic::driver::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`crate::quic::driver::RecvStream::received_reset`]. + /// + /// Unlike the driver, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(crate::quic::driver::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.inner.id() + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport } impl tokio::io::AsyncRead for RecvStream { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut tokio::io::ReadBuf, - ) -> Poll> { - Pin::new(&mut self.inner).poll_read(cx, buf) - } + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } } impl web_transport_trait::RecvStream for RecvStream { - type Error = ReadError; + type Error = ReadError; - fn stop(&mut self, code: u32) { - Self::stop(self, code).ok(); - } + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } - async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { - self.read(dst).await - } + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } - async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { - self.read_chunk(max).await - } + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } - async fn closed(&mut self) -> Result<(), Self::Error> { - self.received_reset().await?; - Ok(()) - } + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } } diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 39af0d1fc7e..7f877185274 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -1,197 +1,168 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::{Buf, Bytes}; -use crate::{CloseReason, ClosedStream, SessionError, WriteError}; +use crate::quic::web::{CloseReason, ClosedStream, SessionError, WriteError}; -/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// A stream that can be used to send bytes. See [`crate::quic::driver::SendStream`]. /// /// This wrapper is mainly needed for error codes, which is unfortunate. /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. #[derive(Debug)] pub struct SendStream { - stream: noq::SendStream, - close: Arc, + stream: crate::quic::driver::SendStream, + close: Arc, } impl SendStream { - pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { - Self { stream, close } - } - - /// Decode the peer's stop code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::WriteError) -> WriteError { - match e { - noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - }, - e => match WriteError::from(e) { - WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. - /// This is a u32 with WebTransport because we share the error space with HTTP/3. - /// A raw QUIC session sends the code as is. - pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { - let code = self.close.encode_stream_code(code); - self.stream.reset(code).map_err(Into::into) - } - - /// Wait until the stream has been stopped and return the error code. See - /// [`noq::SendStream::stopped`]. - /// - /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. - /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not - /// supported. - pub async fn stopped(&self) -> Result, SessionError> { - match self.stream.stopped().await { - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Ok(None) => Ok(None), - Err(noq::StoppedError::ConnectionLost(conn_err)) => { - Err(self.close.map(conn_err.into())) - } - Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - // Unfortunately, we have to wrap WriteError for a bunch of functions. - - /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. - pub async fn write(&mut self, buf: &[u8]) -> Result { - self.stream.write(buf).await.map_err(|e| self.map_error(e)) - } - - /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. - pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { - self.stream - .write_all(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write chunks of data to the stream, returning the number of bytes written. - /// - /// See [`noq::SendStream::write_many_chunks`]. - pub async fn write_many_chunks( - &mut self, - bufs: &mut &mut [Bytes], - ) -> Result { - self.stream - .write_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. - pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { - self.stream - .write_chunk(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. - pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { - self.stream - .write_all_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Mark the stream as finished, such that no more data can be written. See - /// [`noq::SendStream::finish`]. - /// - /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. - /// If you cancel futures by dropping them you'll get incomplete writes. - pub fn finish(&mut self) -> Result<(), ClosedStream> { - self.stream.finish().map_err(Into::into) - } - - pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { - self.stream.set_priority(order).map_err(Into::into) - } - - pub fn priority(&self) -> Result { - self.stream.priority().map_err(Into::into) - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.stream.id() - } + pub(crate) fn new(stream: crate::quic::driver::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::WriteError) -> WriteError { + match e { + crate::quic::driver::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`crate::quic::driver::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`crate::quic::driver::SendStream::stopped`]. + /// + /// Unlike the driver, this returns None if the code is not a valid WebTransport error code. + /// Also unlike the driver, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(crate::quic::driver::StoppedError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`crate::quic::driver::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`crate::quic::driver::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream.write_all(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`crate::quic::driver::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream.write_chunk(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`crate::quic::driver::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream.write_all_chunks(bufs).await.map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`crate::quic::driver::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in QUIC libraries. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.stream.id() + } } impl tokio::io::AsyncWrite for SendStream { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - // We have to use this syntax because noq added its own poll_write method. - tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_shutdown(cx) - } + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + // We have to use this syntax because the driver has its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } } impl web_transport_trait::SendStream for SendStream { - type Error = WriteError; - - fn set_priority(&mut self, order: i32) { - Self::set_priority(self, order).ok(); - } - - fn reset(&mut self, code: u32) { - Self::reset(self, code).ok(); - } - - fn finish(&mut self) -> Result<(), Self::Error> { - Self::finish(self).map_err(|_| WriteError::ClosedStream) - } - - async fn write(&mut self, buf: &[u8]) -> Result { - Self::write(self, buf).await - } - - async fn write_buf(&mut self, buf: &mut B) -> Result { - // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. - let size = buf.chunk().len(); - let chunk = buf.copy_to_bytes(size); - self.write_chunk(chunk).await?; - Ok(size) - } - - async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { - self.write_chunk(chunk).await - } - - async fn closed(&mut self) -> Result<(), Self::Error> { - // NOTE: This used to require &mut in an older version of Noq. - match self.stopped().await? { - Some(code) => Err(WriteError::Stopped(code)), - None => Ok(()), - } - } + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as the driver will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } } diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs index 8af3e81c983..381ec11934e 100644 --- a/rs/moq-tokio/src/quic/web/server.rs +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -1,15 +1,15 @@ #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use std::sync::Arc; -use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use rustls::pki_types::{CertificateDer, PrivateKeyDer}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::{crypto, CongestionControl}; -use crate::{ - proto::{ConnectRequest, ConnectResponse}, - Connecting, ServerError, Session, Settings, +use crate::quic::web::{CongestionControl, crypto}; +use crate::quic::web::{ + Connecting, ServerError, Session, Settings, + proto::{ConnectRequest, ConnectResponse}, }; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -17,184 +17,178 @@ use crate::{ /// /// This is optional; advanced users may use [Server::new] directly. pub struct ServerBuilder { - provider: crypto::Provider, - addr: std::net::SocketAddr, - congestion_controller: - Option>, + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ServerBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ServerBuilder { - /// Create a server builder with sane defaults. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - addr: "[::]:443".parse().unwrap(), - congestion_controller: None, - } - } - - /// Listen on the specified address. - pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { - Self { addr, ..self } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Supply a certificate used for TLS. - // TODO support multiple certs based on...? - pub fn with_certificate( - self, - chain: Vec>, - key: PrivateKeyDer<'static>, - ) -> Result { - // Standard Noq setup - let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13])? - .with_no_client_auth() - .with_single_cert(chain, key)?; - - config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important - - let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); - let config = noq::ServerConfig::with_crypto(Arc::new(config)); - - let server = - noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; - - Ok(Server::new(server)) - } + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard QUIC endpoint setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::quic::web::ALPN.as_bytes().to_vec()]; // this one is important + + let config: crate::quic::driver::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = crate::quic::driver::ServerConfig::with_crypto(Arc::new(config)); + + let server = + crate::quic::driver::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } } /// A WebTransport server that accepts new sessions. pub struct Server { - endpoint: noq::Endpoint, - accept: FuturesUnordered>>, + endpoint: crate::quic::driver::Endpoint, + accept: FuturesUnordered>>, } impl core::ops::Deref for Server { - type Target = noq::Endpoint; + type Target = crate::quic::driver::Endpoint; - fn deref(&self) -> &Self::Target { - &self.endpoint - } + fn deref(&self) -> &Self::Target { + &self.endpoint + } } impl Server { - /// Manually create a new server with a manually constructed Endpoint. - /// - /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. - pub fn new(endpoint: noq::Endpoint) -> Self { - Self { - endpoint, - accept: Default::default(), - } - } - - /// Accept a new WebTransport session Request from a client. - pub async fn accept(&mut self) -> Option { - loop { - tokio::select! { - res = self.endpoint.accept() => { - let conn = res?; - self.accept.push(Box::pin(async move { - let conn = conn.await?; - Request::accept(conn).await - })); - } - Some(res) = self.accept.next() => { - if let Ok(session) = res { - return Some(session) - } - } - } - } - } + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::quic::web::ALPN` for WebTransport to work. + pub fn new(endpoint: crate::quic::driver::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } } /// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to /// accept or reject the session based on the URL. pub struct Request { - conn: noq::Connection, - settings: Settings, - connect: Connecting, + conn: crate::quic::driver::Connection, + settings: Settings, + connect: Connecting, } impl Request { - /// Accept a new WebTransport session from a client. - pub async fn accept(conn: noq::Connection) -> Result { - // Perform the H3 handshake by sending/reciving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Accept the CONNECT request but don't send a response yet. - let connect = Connecting::accept(&conn).await?; - - // Return the resulting request with a reference to the settings/connect streams. - Ok(Self { - conn, - settings, - connect, - }) - } - - pub async fn ok(self) -> Result { - self.respond(ConnectResponse::OK).await - } - - /// Reply to the session with the given response, usually 200 OK. - /// - /// [ConnectResponse::with_protocol] can be used to select a subprotocol. - pub async fn respond( - self, - response: impl Into, - ) -> Result { - let response = response.into(); - let connect = self.connect.respond(response).await?; - Ok(Session::new(self.conn, self.settings, connect)) - } - - /// Reject the session with the given status code. - pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { - self.connect.reject(status).await?; - Ok(()) - } - - /// Returns the underlying QUIC connection. - pub fn conn(&self) -> &noq::Connection { - &self.conn - } - - /// Returns the CONNECT request that was sent by the client. - /// - /// DEPRECATED: You can access this via the Deref impl. - pub fn connect(&self) -> &ConnectRequest { - &self.connect - } + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: crate::quic::driver::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond(self, response: impl Into) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &crate::quic::driver::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } } impl core::ops::Deref for Request { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.connect - } + fn deref(&self) -> &Self::Target { + &self.connect + } } diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index 82a9149d847..ad0fc8c2d69 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1,639 +1,609 @@ use std::{ - fmt, - future::Future, - io::Cursor, - ops::Deref, - pin::Pin, - sync::{Arc, Mutex, OnceLock}, - task::{Context, Poll, Waker}, - time::Duration, + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, }; use bytes::{Bytes, BytesMut}; use futures::stream::{FuturesUnordered, Stream, StreamExt}; -use kio::{Fan, Waiter}; +use kio::Waiter; -use crate::{ - proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, - ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, - WebTransportError, +use crate::quic::web::fan::Fan; +use crate::quic::web::{ + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, WebTransportError, + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, }; /// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake /// has not completed, or it is not valid UTF-8. /// -/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// The driver only exposes the ALPN via a downcast of the boxed handshake data, so the result /// is owned rather than borrowed from the connection. #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -fn negotiated_alpn(conn: &noq::Connection) -> Option { - let data = conn.handshake_data()?; - let data = data.downcast::().ok()?; +fn negotiated_alpn(conn: &crate::quic::driver::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data + .downcast::() + .ok()?; - String::from_utf8(data.protocol?).ok() + String::from_utf8(data.protocol?).ok() } -/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// `crate::quic::driver::crypto::rustls` only exists once a TLS backend is enabled, so there is no /// handshake data type to name here. /// /// This crate's own builders are gated the same way, so normally there is no connection -/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// to read an ALPN from either. A caller that enables a provider on `moq-quic` directly and /// brings its own endpoint can still establish one, and [`Session::protocol`] reports /// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to /// get the ALPN. #[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] -fn negotiated_alpn(_conn: &noq::Connection) -> Option { - None +fn negotiated_alpn(_conn: &crate::quic::driver::Connection) -> Option { + None } /// An established WebTransport session, acting like a full QUIC connection. See -/// [`noq::Connection`]. +/// [`crate::quic::driver::Connection`]. /// /// It is important to remember that WebTransport is layered on top of QUIC: /// 1. Each stream starts with a few bytes identifying the stream type and session ID. /// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. /// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. /// -/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// Deref is used to expose non-overloaded methods on [`crate::quic::driver::Connection`]. /// These should be safe to use with WebTransport, but file a PR if you find one that isn't. #[derive(Clone)] pub struct Session { - conn: noq::Connection, - - // The session ID, as determined by the stream ID of the connect request. - session_id: Option, - - // The accept logic is stateful, so use an Arc to share it. - accept: Option>>, - - // Cache the headers in front of each stream we open. - header_uni: Vec, - header_bi: Vec, - header_datagram: Vec, - - // Keep a reference to the settings and connect stream to avoid closing them until dropped. - #[allow(dead_code)] - settings: Option>, - - // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. - // Wrapped in Arc>> so close() can take it exactly once. - connect_send: Arc>>, - - // Why the session closed: set once by a local close() or the background task - // when a remote CloseWebTransportSession capsule is received. - close: Arc, - - // The request sent by the client, or None for a raw QUIC session. - request: Option, - - // The response sent by the server, or None for a raw QUIC session. - response: Option, - - // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only - // exposes it behind an allocating downcast, so `protocol()` has nothing in the - // connection to borrow from and needs somewhere to keep the result. - // - // Resolved on first read rather than at construction: `raw()` accepts any - // connection, including one whose handshake has not completed and so has no ALPN - // yet. Only a successful read is latched, so an early call cannot pin `None` for - // the life of the session. Shared across clones. - // - // Unused by HTTP/3 sessions, which read the subprotocol out of the response. - alpn: Arc>, + conn: crate::quic::driver::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. The driver only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, } impl Session { - pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { - // The session ID is the stream ID of the CONNECT request. - let session_id = connect.session_id(); - - // Cache the tiny header we write in front of each stream we open. - let mut header_uni = Vec::new(); - StreamUni::WEBTRANSPORT.encode(&mut header_uni); - session_id.encode(&mut header_uni); - - let mut header_bi = Vec::new(); - Frame::WEBTRANSPORT.encode(&mut header_bi); - session_id.encode(&mut header_bi); - - let mut header_datagram = Vec::new(); - session_id.encode(&mut header_datagram); - - let close = Arc::new(CloseReason::new(false)); - - // Accept logic is stateful, so use an Arc to share it. - let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); - - let this = Self { - conn, - accept: Some(Arc::new(Mutex::new(accept))), - session_id: Some(session_id), - header_uni, - header_bi, - header_datagram, - settings: Some(Arc::new(settings)), - connect_send: Arc::new(Mutex::new(Some(connect.send))), - close: close.clone(), - request: Some(connect.request.clone()), - response: Some(connect.response.clone()), - alpn: Default::default(), - }; - - // Run a background task to read capsules from the CONNECT recv stream. - let conn2 = this.conn.clone(); - tokio::spawn(Self::run_recv(conn2, connect.recv, close)); - - this - } - - // Read capsules from the CONNECT recv stream until it's closed, - // then record the close error and tear down the connection. - async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { - let close_info = Self::read_capsules(recv).await; - let code = close_info.as_ref().map_or(0, |(c, _)| *c); - - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Try to record the remote close error. If close() already set - // the error, it owns the connection teardown, so we bail out. - match close_info { - Some((code, reason)) => { - let err = WebTransportError::Closed(code, reason.clone()); - if !close.set(err.into()) { - return; - } - conn.close(http3_code, reason.as_bytes()); - } - None => { - let err = noq::ConnectionError::LocallyClosed.into(); - if !close.set(err) { - return; - } - conn.close(http3_code, b""); - } - }; - } - - // Keep reading capsules from the CONNECT recv stream until it's closed. - // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, - // or None if the stream closed without a capsule. - async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { - let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); - loop { - match reader.read().await { - Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { - code, - reason, - })) => return Some((code, reason)), - Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} - Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { - tracing::warn!(%typ, size = payload.len(), "unknown capsule"); - } - Ok(None) => return None, - Err(e) => { - tracing::warn!(?e, "failed to read capsule"); - return None; - } - } - } - } - - /// Connect using an established QUIC connection if you want to create the connection yourself. - /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. - pub async fn connect( - conn: noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Perform the H3 handshake by sending/receiving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Send the HTTP/3 CONNECT request. - let connect = Connected::open(&conn, request).await?; - - // Return the resulting session with a reference to the control/connect streams. - // If either stream is closed, then the session will be closed, so we need to keep them - // around. - let session = Session::new(conn, settings, connect); - - Ok(session) - } - - /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. - pub async fn accept_uni(&self) -> Result { - if let Some(accept) = &self.accept { - // `kio::wait` owns the waiter, so dropping this future — a `timeout` that - // expires, say — also drops its registration in `SessionAccept`. - kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let recv = self - .conn - .accept_uni() - .await - .map_err(|e| self.map_error(e))?; - Ok(RecvStream::new(recv, self.close.clone())) - } - } - - /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. - pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - if let Some(accept) = &self.accept { - kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - } - - /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. - pub async fn open_uni(&self) -> Result { - let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_uni) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(SendStream::new(send, self.close.clone())) - } - - /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. - pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_bi) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - - /// Asynchronously receives an application datagram from the remote peer. - /// - /// This method is used to receive an application datagram sent by the remote - /// peer over the connection. - /// It waits for a datagram to become available and returns the received bytes. - pub async fn read_datagram(&self) -> Result { - let mut datagram = self - .conn - .read_datagram() - .await - .map_err(|e| self.map_error(e))?; - - let mut cursor = Cursor::new(&datagram); - - if let Some(session_id) = self.session_id { - // We have to check and strip the session ID from the datagram. - let actual_id = - VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; - if actual_id != session_id { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // Return the datagram without the session ID. - let datagram = datagram.split_off(cursor.position() as usize); - - Ok(datagram) - } - - /// Sends an application datagram to the remote peer. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram(buf.into()) - } else { - self.conn.send_datagram(data) - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Sends an application datagram to the remote peer, waiting for buffer space if needed. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram_wait(buf.into()).await - } else { - self.conn.send_datagram_wait(data).await - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Computes the maximum size of datagrams that may be passed to - /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. - pub fn max_datagram_size(&self) -> usize { - self.conn - .max_datagram_size() - .unwrap_or(0) - .saturating_sub(self.header_datagram.len()) - } - - /// Returns the available buffer space for sending datagrams. - /// - /// This value is adjusted to account for the WebTransport session header. - pub fn datagram_send_buffer_space(&self) -> usize { - self.conn - .datagram_send_buffer_space() - .saturating_sub(self.header_datagram.len()) - } - - /// Close the session with an error code and reason. - /// - /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` - /// capsule is written on the CONNECT stream before the QUIC connection is closed. - /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. - /// - /// The capsule write and connection close happen asynchronously in a spawned task - /// that keeps the session alive until then, so the caller may drop it right away. - /// `await` [`Session::closed()`] to learn when the connection is gone. Session - /// operations will fail once the QUIC connection is closed. - pub fn close(&self, code: u32, reason: &[u8]) { - // First close wins: a no-op if the connection already closed, the background - // task recorded a remote close, or close() was already called. - if let Some(err) = self.conn.close_reason() { - self.close.set(self.map_error(err)); - return; - } - let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); - if !self.close.set(err) { - return; - } - - if self.session_id.is_some() { - // Take the send stream for the capsule write. - let send = self.connect_send.lock().unwrap().take(); - - if let Some(send) = send { - let reason = String::from_utf8_lossy(reason).into_owned(); - let capsule = - web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .unwrap_or(Duration::from_millis(100)); - let timeout = (rtt * 3).max(Duration::from_millis(100)); - - // The task owns a clone of the whole session, not just the connection: - // dropping the caller's last handle would otherwise end the HTTP/3 - // control and QPACK streams before the capsule is delivered, which a - // browser treats as a connection error, losing the code and reason. - let session = self.clone(); - tokio::spawn(async move { - session - .close_with_capsule(send, capsule, code, timeout) - .await; - }); - } - } else { - // Raw QUIC mode: no capsule needed. - self.conn.close(code.into(), reason); - } - } - - /// Deliver the close capsule and FIN, then give the peer its teardown grace. - async fn close_with_capsule( - &self, - mut send: noq::SendStream, - capsule: web_transport_proto::Capsule, - code: u32, - timeout: std::time::Duration, - ) { - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Encode the capsule, then wrap it in an HTTP/3 DATA frame. - // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT - // stream (RFC 9297 Section 3.2). - let mut capsule_bytes = Vec::new(); - capsule.encode(&mut capsule_bytes); - - let mut frame = Vec::new(); - Frame::DATA.encode(&mut frame); - let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { - tracing::warn!("capsule too large to encode as DATA frame"); - self.conn.close(http3_code, b""); - return; - }; - len.encode(&mut frame); - frame.extend_from_slice(&capsule_bytes); - - // Write the DATA frame to the CONNECT send stream. - if let Err(e) = send.write_all(&frame).await { - tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); - self.conn.close(http3_code, b""); - return; - } - - // FIN the send stream so the peer knows no more capsules are coming. - if let Err(e) = send.finish() { - tracing::warn!(?e, "failed to finish CONNECT send stream"); - self.conn.close(http3_code, b""); - return; - } - - // A queued write is not delivery: tearing down QUIC before the DATA and FIN - // are acknowledged can discard the capsule during loss recovery. Transport - // failure (including endpoint shutdown) also completes this wait. - match send.stopped().await { - Ok(None) => {} - Ok(Some(code)) => { - tracing::debug!(?code, "peer stopped the CONNECT stream during close"); - self.conn.close(http3_code, b""); - return; - } - Err(e) => { - tracing::debug!( - ?e, - "connection ended before the close capsule was acknowledged" - ); - return; - } - } - - // Wait for the peer to close the CONNECT stream after receiving the capsule. - if tokio::time::timeout(timeout, self.conn.closed()) - .await - .is_err() - { - tracing::debug!("timeout waiting for peer to close; force-closing connection"); - self.conn.close(http3_code, b""); - } - } - - /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. - /// - /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be - /// [`WebTransportError::Closed`] with the code and reason from the capsule. - /// - /// Unlike [`noq::Connection::closed`], this does **not** return early when - /// [`close()`](Self::close) has been called. It waits for the underlying QUIC - /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has - /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. - pub async fn closed(&self) -> SessionError { - self.map_error(self.conn.closed().await) - } - - /// Return why the session was closed, or None if it's not closed. See - /// [`noq::Connection::close_reason`]. - pub fn close_reason(&self) -> Option { - self.conn.close_reason().map(|e| self.map_error(e)) - } - - /// Report an error caused by the connection closing as the session's close reason. - fn map_error(&self, e: impl Into) -> SessionError { - self.close.map(e.into()) - } - - async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { - match send.write_all(buf).await { - Ok(_) => Ok(()), - Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), - Err(err) => Err(WebTransportError::WriteError(err).into()), - } - } - - /// Create a new session from a raw QUIC connection. - /// - /// This is used to pretend like a QUIC connection is a WebTransport session, - /// making it easier to support WebTransport and raw QUIC simultaneously. - /// - /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both - /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC - /// handshake, read from `conn` on demand — a connection that is still handshaking - /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. - pub fn raw(conn: noq::Connection) -> Self { - Self { - conn, - session_id: None, - header_uni: Default::default(), - header_bi: Default::default(), - header_datagram: Default::default(), - accept: None, - settings: None, - connect_send: Arc::new(Mutex::new(None)), - close: Arc::new(CloseReason::new(true)), - request: None, - response: None, - alpn: Default::default(), - } - } - - /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn request(&self) -> Option<&ConnectRequest> { - self.request.as_ref() - } - - /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn response(&self) -> Option<&ConnectResponse> { - self.response.as_ref() - } - - /// Returns the application protocol negotiated for this session. - /// - /// For an HTTP/3 session this is the subprotocol the server selected via - /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. - /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw - /// connection is still handshaking. - pub fn protocol(&self) -> Option<&str> { - if let Some(response) = &self.response { - return response.protocol.as_deref(); - } - - if let Some(alpn) = self.alpn.get() { - return Some(alpn); - } - - // Latch only a successful read, so a call made while the connection is still - // handshaking cannot pin `None` for the life of the session. - let alpn = negotiated_alpn(&self.conn)?; - Some(self.alpn.get_or_init(|| alpn)) - } - - /// Return connection-level statistics. - pub fn stats(&self) -> SessionStats { - let path = self.conn.path_stats(noq::PathId::ZERO); - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .or_else(|| path.map(|p| p.rtt)); - SessionStats { - stats: self.conn.stats(), - path, - rtt, - } - } + pub(crate) fn new(conn: crate::quic::driver::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv( + conn: crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + close: Arc, + ) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = crate::quic::driver::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { code, reason })) => { + return Some((code, reason)); + } + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`crate::quic::driver::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self.conn.accept_uni().await.map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`crate::quic::driver::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`crate::quic::driver::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`crate::quic::driver::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self.conn.read_datagram().await.map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(crate::quic::driver::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let timeout = (self.conn.rtt() * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session.close_with_capsule(send, capsule, code, timeout).await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: crate::quic::driver::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!(?e, "connection ended before the close capsule was acknowledged"); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()).await.is_err() { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`crate::quic::driver::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`crate::quic::driver::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`crate::quic::driver::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut crate::quic::driver::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(crate::quic::driver::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: crate::quic::driver::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + SessionStats(self.conn.stats()) + } } impl Deref for Session { - type Target = noq::Connection; + type Target = crate::quic::driver::Connection; - fn deref(&self) -> &Self::Target { - &self.conn - } + fn deref(&self) -> &Self::Target { + &self.conn + } } impl fmt::Debug for Session { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.conn.fmt(f) - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } } impl PartialEq for Session { - fn eq(&self, other: &Self) -> bool { - self.conn.stable_id() == other.conn.stable_id() - } + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } } impl Eq for Session {} @@ -645,687 +615,656 @@ impl Eq for Session {} // inline, and the first thing a resumed accepter does is take this same lock. An // arrival or a failure is exactly what the others are parked waiting to retry // after, so `Ready` is the signal. -fn poll_accept_uni_shared( - accept: &Mutex, - waiter: &Waiter, -) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.uni_waiters.clone(); - - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_uni(waiter); - - (result, waiters, hold) - }; - - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); - - if result.is_ready() { - waiters.wake(); - } - - result +fn poll_accept_uni_shared(accept: &Mutex, waiter: &Waiter) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result } fn poll_accept_bi_shared( - accept: &Mutex, - waiter: &Waiter, + accept: &Mutex, + waiter: &Waiter, ) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.bi_waiters.clone(); + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_bi(waiter); + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); - (result, waiters, hold) - }; + (result, waiters, hold) + }; - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); - if result.is_ready() { - waiters.wake(); - } + if result.is_ready() { + waiters.wake(); + } - result + result } // Type aliases just so clippy doesn't complain about the complexity. -type AcceptUni = dyn Stream> + Send; -type AcceptBi = - dyn Stream> + Send; -type PendingUni = dyn Future> + Send; -type PendingBi = - dyn Future, SessionError>> + Send; +type AcceptUni = + dyn Stream> + Send; +type AcceptBi = dyn Stream< + Item = Result< + (crate::quic::driver::SendStream, crate::quic::driver::RecvStream), + crate::quic::driver::ConnectionError, + >, + > + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = dyn Future, SessionError>> + + Send; // Logic just for accepting streams, which is annoying because of the stream header. pub struct SessionAccept { - session_id: VarInt, - - // Shared close reason for propagation to accepted streams. - close: Arc, - - // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them. Again, this is just so they don't get closed until we drop the session. - qpack_encoder: Option, - qpack_decoder: Option, - - accept_uni: Pin>, - accept_bi: Pin>, - - // Keep track of work being done to read/write the WebTransport stream header. - pending_uni: FuturesUnordered>>, - pending_bi: FuturesUnordered>>, - - // Waiters from concurrent callers of accept_bi / accept_uni. - // Every clone of the session polls this one struct, so an arrival has to be fanned - // out: each caller registers here and all of them are woken when a stream lands — - // by the caller that saw it, once it has released the lock on this struct. - bi_waiters: Fan, - uni_waiters: Fan, - - // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with - // the same waker every time. That waker outlives every caller, so an accepter that - // drops its future cannot take the wakeup path with it, and the layer below holds - // one registration rather than one per caller. - bi_waker: Waker, - uni_waker: Waker, + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, } impl SessionAccept { - pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { - // Create a stream that just outputs new streams, so it's easy to call from poll. - let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { - Some((conn.accept_uni().await, conn)) - })); - - let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { - Some((conn.accept_bi().await, conn)) - })); - - let bi_waiters = Fan::new(); - let uni_waiters = Fan::new(); - let bi_waker = bi_waiters.waker(); - let uni_waker = uni_waiters.waker(); - - Self { - session_id, - close, - - qpack_decoder: None, - qpack_encoder: None, - - accept_uni, - accept_bi, - - pending_uni: FuturesUnordered::new(), - pending_bi: FuturesUnordered::new(), - - bi_waiters, - uni_waiters, - bi_waker, - uni_waker, - } - } - - /// Poll for the next unidirectional WebTransport stream. - /// - /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops - /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive - /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this - /// with [`kio::wait`], which holds the waiter inside the future it builds. - /// - /// A `Ready` here means every *other* parked accepter should be woken so it can - /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes - /// them once the lock on this struct is released. - // - // Poll-based because we accept and decode streams in parallel. In async land this - // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is - // runtime-agnostic. - pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { - // Register before polling, not on the way out: the shared waker can fire from the - // layer below at any point here, and a wake that lands before the caller is on - // the list would be lost. - self.uni_waiters.register(waiter); - - let waker = self.uni_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let recv = match res { - Ok(recv) => recv, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_uni(recv, self.session_id); - self.pending_uni.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "unidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - // Decide if we keep looping based on the type. - match typ { - StreamUni::WEBTRANSPORT => { - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok(recv)); - } - StreamUni::QPACK_DECODER => { - self.qpack_decoder = Some(recv); - } - StreamUni::QPACK_ENCODER => { - self.qpack_encoder = Some(recv); - } - _ => { - // ignore unknown streams - tracing::debug!(?typ, "ignoring unknown unidirectional stream"); - } - } - } - } - - // Reads the stream header, returning the stream type. - async fn decode_uni( - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result<(StreamUni, noq::RecvStream), SessionError> { - // Read the VarInt at the start of the stream. - let typ = StreamUni(read_varint(&mut recv).await?); - - if typ == StreamUni::WEBTRANSPORT { - // Read the session_id and validate it - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them, so return everything. - Ok((typ, recv)) - } - - /// Poll for the next bidirectional WebTransport stream. - /// - /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` - /// registration is weak and owned by the caller, and a `Ready` is what the other - /// parked accepters need to be woken for. - pub fn poll_accept_bi( - &mut self, - waiter: &Waiter, - ) -> Poll> { - // Register before polling; see `poll_accept_uni`. - self.bi_waiters.register(waiter); - - let waker = self.bi_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let (send, recv) = match res { - Ok(pair) => pair, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_bi(send, recv, self.session_id); - self.pending_bi.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let res = match self.pending_bi.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "bidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - if let Some((send, recv)) = res { - // Wrap the streams in our own types for correct error codes. - let send = SendStream::new(send, self.close.clone()); - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok((send, recv))); - } - - // Keep looping if it's a stream we want to ignore. - } - } - - // Reads the stream header, returning Some if it's a WebTransport stream. - async fn decode_bi( - send: noq::SendStream, - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result, SessionError> { - let typ = read_varint(&mut recv).await?; - if Frame(typ) != Frame::WEBTRANSPORT { - tracing::debug!(?typ, "ignoring unknown bidirectional stream"); - return Ok(None); - } - - // Read the session ID and validate it. - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - - Ok(Some((send, recv))) - } + pub(crate) fn new(conn: crate::quic::driver::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::default(); + let uni_waiters = Fan::default(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, crate::quic::driver::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: crate::quic::driver::SendStream, + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } } // Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any // failure as a truncation, which would hide a stream the peer reset before its header // arrived. -async fn read_varint(recv: &mut noq::RecvStream) -> Result { - let mut buf = [0u8; 8]; - recv.read_exact(&mut buf[..1]).await?; +async fn read_varint(recv: &mut crate::quic::driver::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; - // The first two bits encode the length. - let size = 1 << (buf[0] >> 6); - recv.read_exact(&mut buf[1..size]).await?; + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; - Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) } // Without reliable reset, a stream the peer resets early loses its header with it, which // moq does routinely. Anything else is a misbehaving peer. fn log_header_error(err: SessionError, direction: &'static str) { - match err { - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError( - noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), - ), - )) => tracing::debug!(?err, direction, "stream closed before its header"), - _ => tracing::warn!(?err, direction, "failed to decode stream header"), - } + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError( + crate::quic::driver::ReadError::Reset(_) | crate::quic::driver::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } } -pub struct SessionStats { - stats: noq::ConnectionStats, - path: Option, - rtt: Option, -} +pub struct SessionStats(crate::quic::driver::ConnectionStats); impl web_transport_trait::Stats for SessionStats { - fn bytes_sent(&self) -> Option { - Some(self.stats.udp_tx.bytes) - } - - fn bytes_received(&self) -> Option { - Some(self.stats.udp_rx.bytes) - } - - fn bytes_lost(&self) -> Option { - self.path.map(|path| path.lost_bytes) - } - - fn packets_sent(&self) -> Option { - Some(self.stats.udp_tx.datagrams) - } - - fn packets_received(&self) -> Option { - Some(self.stats.udp_rx.datagrams) - } - - fn packets_lost(&self) -> Option { - self.path.map(|path| path.lost_packets) - } - - fn rtt(&self) -> Option { - self.rtt - } - - fn estimated_send_rate(&self) -> Option { - let path = self.path?; - let rtt = self.rtt?; - let rtt_secs = rtt.as_secs_f64(); - if path.cwnd > 0 && rtt_secs > 0.0 { - Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) - } else { - None - } - } + fn bytes_sent(&self) -> Option { + Some(self.0.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.0.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + Some(self.0.path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.0.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.0.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + Some(self.0.path.lost_packets) + } + + fn rtt(&self) -> Option { + Some(self.0.path.rtt) + } + + /// The congestion controller's bandwidth estimate, in bits per second. + fn estimated_send_rate(&self) -> Option { + self.0.path.bandwidth_estimate.map(|bytes| bytes.saturating_mul(8)) + } } impl web_transport_trait::Session for Session { - type SendStream = SendStream; - type RecvStream = RecvStream; - type Error = SessionError; - - async fn accept_uni(&self) -> Result { - Self::accept_uni(self).await - } - - async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::accept_bi(self).await - } - - async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::open_bi(self).await - } - - async fn open_uni(&self) -> Result { - Self::open_uni(self).await - } - - fn close(&self, code: u32, reason: &str) { - Self::close(self, code, reason.as_bytes()); - } - - async fn closed(&self) -> Self::Error { - Self::closed(self).await - } - - fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { - Self::send_datagram(self, data) - } - - async fn recv_datagram(&self) -> Result { - Self::read_datagram(self).await - } - - fn max_datagram_size(&self) -> usize { - Self::max_datagram_size(self) - } - - fn protocol(&self) -> Option<&str> { - Self::protocol(self) - } - - #[allow(refining_impl_trait)] - fn stats(&self) -> SessionStats { - Self::stats(self) - } + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } } #[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] mod tests { - use std::{ - io::{self, IoSliceMut}, - net::{Ipv4Addr, SocketAddr}, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - }; - - use rcgen::{CertifiedKey, KeyPair}; - use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; - - use super::*; - - const SESSION: VarInt = VarInt::from_u32(0); - - #[derive(Debug, Default)] - struct Gate { - blocked: AtomicBool, - attempted: tokio::sync::Notify, - wakers: Mutex>, - } - - #[derive(Debug)] - struct Socket { - inner: Box, - gate: Arc, - } - - #[derive(Debug)] - struct Sender { - inner: Pin>, - gate: Arc, - } - - impl noq::UdpSender for Sender { - fn poll_send( - self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, - cx: &mut Context<'_>, - ) -> Poll> { - let this = self.get_mut(); - let mut wakers = this.gate.wakers.lock().unwrap(); - if this.gate.blocked.load(Ordering::SeqCst) { - if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { - wakers.push(cx.waker().clone()); - } - this.gate.attempted.notify_one(); - return Poll::Pending; - } - drop(wakers); - this.inner.as_mut().poll_send(transmit, cx) - } - } - - impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { - Box::pin(Sender { - inner: self.inner.create_sender(), - gate: self.gate.clone(), - }) - } - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], - ) -> Poll> { - self.inner.poll_recv(cx, bufs, meta) - } - fn local_addr(&self) -> io::Result { - self.inner.local_addr() - } - } - - /// A connected client and server QUIC connection, plus the endpoints that drive them. - async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { - let (client, server, endpoints, _) = gated_pair().await; - (client, server, endpoints) - } - - async fn gated_pair() -> ( - noq::Connection, - noq::Connection, - [noq::Endpoint; 2], - Arc, - ) { - #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - - let CertifiedKey { cert, signing_key } = - rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); - let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( - &signing_key, - ))); - - let server_config = - noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); - let runtime: Arc = Arc::new(noq::TokioRuntime); - let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - let gate = Arc::new(Gate::default()); - let socket = Socket { - inner: runtime.wrap_udp_socket(udp).unwrap(), - gate: gate.clone(), - }; - let server = noq::Endpoint::new_with_abstract_socket( - Default::default(), - Some(server_config), - Box::new(socket), - runtime, - ) - .unwrap(); - - let mut roots = rustls::RootCertStore::empty(); - roots.add(cert.der().clone()).unwrap(); - let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); - let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); - - let connecting = client - .connect_with(client_config, server.local_addr().unwrap(), "localhost") - .unwrap(); - let accept = async { server.accept().await.unwrap().await.unwrap() }; - let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); - - (client_conn, server_conn, [client, server], gate) - } - - /// A queued capsule must survive a stalled socket beyond the teardown grace. - #[tokio::test] - async fn close_waits_for_capsule_ack() { - close_with_stalled_socket(false).await; - } - - /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. - #[tokio::test] - async fn close_ack_wait_ends_with_connection() { - close_with_stalled_socket(true).await; - } - - async fn close_with_stalled_socket(cancel: bool) { - let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; - let (client, server) = tokio::join!( - Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), - async { - crate::Request::accept(server_conn) - .await - .unwrap() - .ok() - .await - }, - ); - let client = client.unwrap(); - let server = server.unwrap(); - let conn = server.conn.clone(); - let settings = Arc::downgrade(server.settings.as_ref().unwrap()); - tokio::time::pause(); - gate.blocked.store(true, Ordering::SeqCst); - server.close(2, b"unauthorized"); - drop(server); - gate.attempted.notified().await; - tokio::time::advance(Duration::from_secs(1)).await; - tokio::task::yield_now().await; - assert!( - conn.close_reason().is_none(), - "closed before the capsule was acknowledged" - ); - - if cancel { - conn.close(0u32.into(), b"drain deadline"); - conn.closed().await; - tokio::task::yield_now().await; - assert_eq!( - settings.strong_count(), - 0, - "close task retained the HTTP/3 streams after cancellation" - ); - return; - } - - gate.blocked.store(false, Ordering::SeqCst); - for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { - waker.wake(); - } - tokio::time::resume(); - let err = client.closed().await; - assert!( - matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), - "{err:?}" - ); - conn.closed().await; - } - - fn assert_reset(err: SessionError) { - assert!( - matches!( - &err, - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) - )) if code.into_inner() == 7 - ), - "{err:?}" - ); - } - - /// A uni stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn uni_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - // The stream type alone, so the reset also cuts off the session ID. - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54]).await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A bi stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn bi_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - let (mut send, _recv) = client.open_bi().await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let (send, recv) = server.accept_bi().await.unwrap(); - let err = SessionAccept::decode_bi(send, recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A stream that names another session is still reported as `UnknownSession`. - #[tokio::test] - async fn uni_other_session() { - let (client, server, _endpoints) = pair().await; - - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); - send.finish().unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert!( - matches!( - err, - SessionError::WebTransportError(WebTransportError::UnknownSession) - ), - "{err:?}" - ); - } + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl crate::quic::driver::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &moq_sock::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl crate::quic::driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [moq_sock::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + ) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der(&signing_key))); + + let server_config = crate::quic::driver::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: crate::quic::driver::wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = crate::quic::driver::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = crate::quic::driver::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = crate::quic::driver::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { crate::quic::web::Request::accept(server_conn).await.unwrap().ok().await }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError(crate::quic::driver::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::UnknownSession)), + "{err:?}" + ); + } } diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index 08ea9a53a1f..fdee95f23c7 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -4,67 +4,67 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum SettingsError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::SettingsError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), - #[error("WebTransport is not supported")] - WebTransportUnsupported, + #[error("WebTransport is not supported")] + WebTransportUnsupported, - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), } pub struct Settings { - // A reference to the send/recv stream, so we don't close it until dropped. - #[allow(dead_code)] - send: noq::SendStream, + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: crate::quic::driver::SendStream, - #[allow(dead_code)] - recv: noq::RecvStream, + #[allow(dead_code)] + recv: crate::quic::driver::RecvStream, } impl Settings { - // Establish the H3 connection. - pub async fn connect(conn: &noq::Connection) -> Result { - let recv = Self::accept(conn); - let send = Self::open(conn); + // Establish the H3 connection. + pub async fn connect(conn: &crate::quic::driver::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); - // Run both tasks concurrently until one errors or they both complete. - let (send, recv) = try_join!(send, recv)?; - Ok(Self { send, recv }) - } + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } - async fn accept(conn: &noq::Connection) -> Result { - let mut recv = conn.accept_uni().await?; - let settings = web_transport_proto::Settings::read(&mut recv).await?; + async fn accept(conn: &crate::quic::driver::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; - tracing::debug!(?settings, "received SETTINGS frame"); + tracing::debug!(?settings, "received SETTINGS frame"); - if settings.supports_webtransport() == 0 { - return Err(SettingsError::WebTransportUnsupported); - } + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } - Ok(recv) - } + Ok(recv) + } - async fn open(conn: &noq::Connection) -> Result { - let mut settings = web_transport_proto::Settings::default(); - settings.enable_webtransport(1); + async fn open(conn: &crate::quic::driver::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); - tracing::debug!(?settings, "sending SETTINGS frame"); + tracing::debug!(?settings, "sending SETTINGS frame"); - let mut send = conn.open_uni().await?; - settings.write(&mut send).await?; + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; - Ok(send) - } + Ok(send) + } } diff --git a/rs/moq-tokio/src/resolve.rs b/rs/moq-tokio/src/resolve.rs index 3f58d119faf..2ce9408e11c 100644 --- a/rs/moq-tokio/src/resolve.rs +++ b/rs/moq-tokio/src/resolve.rs @@ -289,7 +289,7 @@ impl Candidates { /// are handed out anyway so the dial surfaces the OS error instead of a /// confusing "no DNS entries". See /// for the Windows failure this family matching originally fixed. - #[cfg(any(feature = "noq", test))] + #[cfg(any(feature = "quic", test))] pub(crate) fn with_local(mut self, local: SocketAddr, dual_stack: bool) -> Self { self.local = Some((local, dual_stack)); self diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 3c57e4b287a..754d40ff572 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -16,7 +16,7 @@ use url::Url; // Only the transports that finish their handshake in a spawned future need `.boxed()`; // the stream listeners hand back an already-built `Request`. -#[cfg(any(feature = "noq", feature = "iroh", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "iroh", feature = "websocket"))] use futures::FutureExt; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; @@ -86,7 +86,7 @@ impl crate::listen::Config { } /// Default bind address used when [`crate::listen::Config::bind`] is not set. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) const DEFAULT_BIND: net::SocketAddr = net::SocketAddr::V6(net::SocketAddrV6::new(net::Ipv6Addr::UNSPECIFIED, 443, 0, 0)); @@ -109,7 +109,7 @@ pub(crate) enum Parts { /// The QUIC listener only, as one member of a `SO_REUSEPORT` group. Folding /// the member in here is what stops a group member and a stream-only server /// from being asked for at once. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] Member(crate::listen::Socket), } @@ -130,7 +130,7 @@ impl Parts { /// This server's socket in a complete group, when it is a member of one. /// Consuming, because only one backend may own the serving handle. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] fn member(self) -> Option { match self { Self::Member(member) => Some(member), @@ -197,7 +197,7 @@ impl Config { } /// Copy the settings one QUIC worker owns. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn worker(&self) -> Self { Self { listen: self.listen.clone(), @@ -232,8 +232,8 @@ pub struct Server { streams: StreamListeners, #[cfg(feature = "iroh")] iroh: Option, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "websocket")] websocket: Option, } @@ -250,10 +250,10 @@ pub struct Server { /// only caller. #[derive(Clone, Debug)] #[allow(dead_code)] -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) struct SocketRetainer { - #[cfg(feature = "noq")] - noq: Option>, + #[cfg(feature = "quic")] + quic: Option>, } impl Server { @@ -314,9 +314,9 @@ impl Server { // `parts.quic()` and not `build_quic`: a caller that asked for streams only // is not asking for a backend, while leaving everything else configured is // still the error it always was. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] if config.bind.is_some() && parts.quic() { - return Err(Error::NoBackend("--listen requires the noq feature")); + return Err(Error::NoBackend("--listen requires the quic feature")); } // Only the QUIC backend verifies client certificates; the qmux listeners @@ -327,8 +327,8 @@ impl Server { return Err(Error::MtlsUnsupported); } if parts.quic() && !config.tls.root.is_empty() { - #[cfg(not(feature = "noq"))] - return Err(Error::NoBackend("--listen-tls-root requires the noq feature")); + #[cfg(not(feature = "quic"))] + return Err(Error::NoBackend("--listen-tls-root requires the quic feature")); } #[cfg(feature = "_certs")] @@ -337,7 +337,7 @@ impl Server { return Err(crate::tls::Error::ConflictingServe.into()); } #[allow(unused_mut)] - let mut needed = cfg!(feature = "noq") && build_quic; + let mut needed = cfg!(feature = "quic") && build_quic; #[cfg(feature = "tcp")] { needed |= build_streams && config.tcp.bind.is_some() && config.tcp.tls == Some(true); @@ -362,13 +362,13 @@ impl Server { // The member is a serving handle released by a complete reuseport group, // so the backend may take it only once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let member = parts.member(); - #[cfg(feature = "noq")] - let noq = build_quic + #[cfg(feature = "quic")] + let quic = build_quic .then(|| { - crate::noq::NoqServer::new(crate::noq::Options { + crate::quic::Server::new(crate::quic::Options { config: config.clone(), quic: quic.clone(), member, @@ -437,8 +437,8 @@ impl Server { streams, #[cfg(feature = "iroh")] iroh, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "websocket")] websocket, }) @@ -510,9 +510,9 @@ impl Server { /// /// Empty when no TLS-bearing backend is configured (e.g. a stream-only server). pub fn certificates(&self) -> crate::tls::Certificates { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return noq.certificates(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return quic.certificates(); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`): no certificates. crate::tls::Certificates::empty() @@ -522,17 +522,17 @@ impl Server { /// group after this server is gone. /// /// Crate-private: only the worker group retains sockets this way. A worker - /// member serves the same noq backend as the tokio listener. - #[cfg(feature = "noq")] + /// member serves the same QUIC backend as the tokio listener. + #[cfg(feature = "quic")] pub(crate) fn retain(&self) -> SocketRetainer { SocketRetainer { - #[cfg(feature = "noq")] - noq: self.noq.as_ref().and_then(|server| server.retain()), + #[cfg(feature = "quic")] + quic: self.quic.as_ref().and_then(|server| server.retain()), } } #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -624,7 +624,7 @@ impl Server { /// The body of [`Listener::accept`]. Stream accept loops start here if /// [`Server::bind`] left them stopped; [`Server::listen`] already started them. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -641,22 +641,22 @@ impl Server { } loop { // The QUIC endpoint address, reported as a QUIC session's local side. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let local = self.local_addr().ok(); // tokio::select! does not support cfg directives on arms, so we need to create the futures here. // `Some(None)` once a configured endpoint stops, which only losing its // driver does while we accept. - #[cfg(feature = "noq")] - let noq_accept = async { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_mut() { - return Some(noq.accept().await); + #[cfg(feature = "quic")] + let quic_accept = async { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_mut() { + return Some(quic.accept().await); } None }; - #[cfg(not(feature = "noq"))] - let noq_accept = async { None::<()> }; + #[cfg(not(feature = "quic"))] + let quic_accept = async { None::<()> }; #[cfg(feature = "iroh")] let iroh_accept = async { @@ -714,8 +714,8 @@ impl Server { #[cfg(not(any(feature = "tcp", all(feature = "uds", unix))))] let _ = request; } - Some(_conn) = noq_accept => { - #[cfg(feature = "noq")] + Some(_conn) = quic_accept => { + #[cfg(feature = "quic")] { // The process would keep its other listeners and silently drop // QUIC, so stop accepting at all and let the owner fail loudly. @@ -734,13 +734,13 @@ impl Server { // Accept the transport (capturing url + mTLS identity) and exchange the // MoQ SETUP up front, so path/role are known before the caller authorizes // (like the stream bindings). - let Accepted { session, url, identity, authority, mut link } = within(deadline, super::noq::accept(_conn, alpns)).await?; + let Accepted { session, url, identity, authority, mut link } = within(deadline, crate::quic::accept(_conn, alpns)).await?; link.local = local; let binding = match url { Some(_) => moq_net::transport::Binding::Uri, None => moq_net::transport::Binding::InBand }; let (request, deadline) = accept_request(&setup.server, session, binding, deadline).await?; // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; - Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline, active }) + Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Quic(Box::new(request)), deadline, active }) }.boxed()); } } @@ -809,9 +809,9 @@ impl Server { /// Errors with [`Error::NoBackend`] on a stream-only server, which has no /// QUIC listener. pub fn local_addr(&self) -> crate::Result { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return Ok(noq.local_addr()?); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return Ok(quic.local_addr()?); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`). Err(Error::NoBackend("no QUIC listener configured")) @@ -828,9 +828,9 @@ impl Server { #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] self.streams.shutdown().await; - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.take() { - noq.shutdown().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.take() { + quic.shutdown().await; } #[cfg(feature = "iroh")] if let Some(iroh) = self.iroh.take() { @@ -844,9 +844,9 @@ impl Server { /// Close the QUIC connections without waiting for the socket, as a drop must. fn close(&mut self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - noq.close(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + quic.close(); } } } @@ -943,7 +943,7 @@ async fn serve_session(request: Request) -> crate::Result<()> { /// The MoQ server an accepted connection handshakes against, and how long it has. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -957,7 +957,7 @@ struct Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -972,7 +972,7 @@ impl Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1035,7 +1035,7 @@ struct Deadline { impl Deadline { #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1078,7 +1078,7 @@ impl Deadline { /// Nothing is established yet that could carry a close code, so an expired /// handshake is dropped, which closes its connection. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1526,8 +1526,8 @@ async fn stream_request( type PendingRequest = moq_net::server::Handshake; pub(crate) enum RequestKind { - #[cfg(feature = "noq")] - Noq(Box>>), + #[cfg(feature = "quic")] + Quic(Box>>), #[cfg(feature = "iroh")] Iroh(Box>>), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1537,7 +1537,7 @@ pub(crate) enum RequestKind { /// The transport-level facts a backend captures while accepting a connection, before the /// MoQ SETUP. Grouped so the shared accept loop builds a [`Request`] from named fields /// rather than a wide tuple. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) struct Accepted { pub session: S, pub url: Option, @@ -1606,8 +1606,8 @@ pub enum Reject { macro_rules! request_ref { ($self:expr, $r:ident => $body:expr) => { match &$self.kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1620,8 +1620,8 @@ macro_rules! request_ref { macro_rules! request_into { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1634,8 +1634,8 @@ macro_rules! request_into { macro_rules! request_map { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => RequestKind::Noq(Box::new($body)), + #[cfg(feature = "quic")] + RequestKind::Quic($r) => RequestKind::Quic(Box::new($body)), #[cfg(feature = "iroh")] RequestKind::Iroh($r) => RequestKind::Iroh(Box::new($body)), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1729,7 +1729,7 @@ impl Request { /// transport carries no host (iroh, stream bindings). /// /// Reported as offered: rustls sends no SNI for an IP-literal dial (RFC 6066), - /// so noq reports `None`. + /// so the QUIC backend reports `None`. /// /// Not the moq-net IETF SETUP `Authority` parameter. Client-asserted and not authenticated, /// so authorize on the token or [`Self::peer_identity`] rather than on this value. @@ -1824,7 +1824,7 @@ impl Request { /// The client certificate chain the peer presented, if any, validated /// against a configured [`crate::tls::Listen::root`] during the handshake. /// - /// Captured at the transport handshake (before the SETUP). Noq supports mTLS; + /// Captured at the transport handshake (before the SETUP). The QUIC backend supports mTLS; /// other transports always return `None`. Use it to grant /// elevated access or to close the session once the certificate expires (see /// [`crate::tls::PeerIdentity::expiry`]). @@ -1859,7 +1859,7 @@ mod tests { assert!(matches!(died, Some(Err(err)) if err.is_panic())); } - #[cfg(all(not(feature = "noq"), feature = "_certs"))] + #[cfg(all(not(feature = "quic"), feature = "_certs"))] #[test] fn no_quic_backend_needs_no_served_identity() { Server::new(Config::default()).expect("an absent QUIC backend must not load a TLS identity"); @@ -2654,7 +2654,7 @@ mod tests { /// Closing a listener must release its UDP socket before returning, even while /// an accepted session still holds its QUIC connection, so a restart can rebind /// the port at once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn close_releases_quic_socket() { let config = crate::listen::Config { @@ -2709,7 +2709,7 @@ mod tests { } /// An explicit QUIC bind cannot be honored without a QUIC backend. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn quic_bind_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2729,7 +2729,7 @@ mod tests { /// A client CA on the default QUIC listener names the missing backend, not a /// stream-only server the caller never configured. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn client_ca_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2752,7 +2752,7 @@ mod tests { /// A QUIC-only server reports nothing. It multiplexes over one UDP socket and /// never calls `accept`, so a zero counter would be a watch that cannot fire. - #[cfg(all(feature = "noq", not(feature = "tcp")))] + #[cfg(all(feature = "quic", not(feature = "tcp")))] #[test] fn accept_health_is_empty_without_a_stream_listener() { let server = crate::listen::Config::default() @@ -2773,7 +2773,7 @@ mod tests { /// Building the endpoint needs a runtime, and `certificates()` must stay /// readable without one (no guard escapes to the caller). - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn certificates_expose_generated_fingerprints() { let mut config = crate::listen::Config { diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 1c19e4310f5..2ff15e67784 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -376,7 +376,7 @@ mod tests { listen.tcp.tls = Some(true); listen.tls.generate = vec!["localhost".into()]; // Client-certificate policy is refused without a QUIC listener to read it. - if !cfg!(feature = "noq") { + if !cfg!(feature = "quic") { continue; } listen.bind = Some("127.0.0.1:0".parse().unwrap()); diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index 79f56ecef66..a28dcfcd7cc 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -24,7 +24,7 @@ use rustls::pki_types::PrivatePkcs8KeyDer; /// Errors loading or generating TLS certificates and keys. /// -/// Shared by the client TLS config and noq server. +/// Shared by the client TLS config and QUIC server. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -791,7 +791,7 @@ pub(crate) enum Verification { Fingerprints(Vec<[u8; 32]>), /// Standard CA verification. When `system` is set the platform/default trust - /// store is trusted too; noq uses the OS platform verifier. + /// store is trusted too; the QUIC backend uses the OS platform verifier. /// `custom` are extra PEM roots trusted in addition. Roots { custom: CustomRoots, system: bool }, } @@ -955,7 +955,7 @@ impl Connect { /// plaintext fetch, and there is nothing to bootstrap when verification is /// disabled. With CA roots (the default), `http://` is the deliberate /// per-connection way to pin a self-signed relay, so it is allowed. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn allows_http_bootstrap(&self) -> bool { self.fingerprint.is_empty() && !self.insecure.unwrap_or_default() } @@ -1634,7 +1634,7 @@ impl PeerIdentity { /// Wrap the type-erased identity from the QUIC connection. /// Returns `None` if the peer presented no certificate or the identity is /// not a certificate chain. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn from_any(identity: Option>) -> Option { let chain = identity?.downcast::>>().ok()?; Some(Self { chain: *chain }) @@ -2092,7 +2092,7 @@ impl rustls::client::danger::ServerCertVerifier for FingerprintVerifier { } #[cfg(test)] -#[cfg(all(feature = "noq", feature = "aws-lc-rs"))] +#[cfg(all(feature = "quic", feature = "aws-lc-rs"))] mod tests { #[tokio::test] async fn resolved_config_keeps_reload_owner_alive() { @@ -2266,7 +2266,7 @@ mod tests { params.self_signed(&key).unwrap().into() } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_expiry_reads_not_after() { // notAfter at a whole second so the round-trip is exact. @@ -2277,7 +2277,7 @@ mod tests { params.not_after = not_after; let cert: CertificateDer<'static> = params.self_signed(&key).unwrap().into(); - // noq hand back the chain as a boxed Vec. + // The QUIC backend hands back the chain as a boxed Vec. let identity: Box = Box::new(vec![cert]); let parsed = PeerIdentity::from_any(Some(identity)).expect("chain parsed"); let expiry = parsed.expiry().expect("expiry parsed"); @@ -2287,7 +2287,7 @@ mod tests { ); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_none_without_chain() { assert!(PeerIdentity::from_any(None).is_none()); @@ -2390,7 +2390,7 @@ mod tests { /// keeps its task, an OS directory watch, and the certificate keys alive for the /// rest of the process. The `Arc` is the observable half: while the task lives it /// holds one, so a `Weak` that still upgrades is a watcher that never stopped. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[tokio::test] async fn dropping_the_reload_guard_stops_the_watcher() { let key = rcgen::KeyPair::generate().unwrap(); @@ -2525,7 +2525,7 @@ mod tests { /// A resumed handshake skips certificate verification, so a listener with a /// live peer set must disable resumption for removals to take effect. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn removed_peers_cannot_resume() { let server_identity = Identity::generate(["localhost"]).unwrap(); @@ -2559,7 +2559,7 @@ mod tests { } /// A plain-TLS embedder receives a working config from the shared identity. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn served_config_handshakes() { let server = Listen { @@ -2685,7 +2685,7 @@ mod tests { (file, path) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn signed_certificates() -> ( String, CertificateDer<'static>, @@ -2722,7 +2722,7 @@ mod tests { ) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn handshake_kinds( client: Arc, server: Arc, @@ -2754,7 +2754,7 @@ mod tests { panic!("TLS handshake did not settle"); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_roots_disable_session_resumption() { use std::io::Write; @@ -2805,7 +2805,7 @@ mod tests { ); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_client_roots_disable_server_resumption() { let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates(); @@ -2884,7 +2884,7 @@ mod tests { assert!(handshake_kinds(reloadable_client, reloadable).is_err()); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn custom_roots_reload_for_new_client_and_server_handshakes() { let (ca_a, server_a, _, client_a, _) = signed_certificates(); @@ -3296,7 +3296,7 @@ mod legacy_tests { /// The accept side, where a dropped `--server-tls-root` takes the mTLS client /// CAs with it and leaves the listener accepting unauthenticated peers. /// moq-cli's `export hls` flattens this type directly, so it is a live path. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn the_server_builders_refuse_a_released_spelling() { let tls = parse(&["--server-tls-root", "/tmp/ca.pem"]).listen; diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index a87ec2f1c12..5ab97f52a9e 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -1,7 +1,7 @@ //! Adapts the async transport interface to the poll one moq-net requires. //! //! moq-net accepts its own `moq_net::transport::poll` traits. -//! A backend that offers only the async interface (qmux, iroh, noq) is wrapped +//! A backend that offers only the async interface (qmux, iroh, quic) is wrapped //! in [`Session`] here, at the edge where async already lives. The wrapping //! costs one allocation per operation and one copy per write, and its //! closed-watch emulation is weaker than a native implementation (see diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 609de2be9f0..90b2e67fd1b 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -184,7 +184,7 @@ fn won_key(url: &Url) -> Result<(String, u16)> { } /// Forget that WebSocket won for `url`, giving QUIC its head start again. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) fn forget(url: &Url) { if let Ok(key) = won_key(url) { WEBSOCKET_WON.lock().unwrap().remove(&key); @@ -192,13 +192,13 @@ pub(crate) fn forget(url: &Url) { } /// Record that WebSocket won for `url`, as a fallback dial that lands does. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn remember(url: &Url) { WEBSOCKET_WON.lock().unwrap().insert(won_key(url).unwrap()); } /// Whether the next dial to `url` skips QUIC's head start. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn won(url: &Url) -> bool { won_key(url).is_ok_and(|key| WEBSOCKET_WON.lock().unwrap().contains(&key)) } @@ -339,7 +339,7 @@ pub struct Resolved { /// The fallback arm of the QUIC-vs-WebSocket race, so only compiled when there is a /// QUIC dial to race against. A WebSocket-only build calls [`connect`] directly. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) async fn race_handle( config: &Config, tls: &rustls::ClientConfig, diff --git a/rs/moq-tokio/src/worker/group.rs b/rs/moq-tokio/src/worker/group.rs index 282a1db1163..c98642d7528 100644 --- a/rs/moq-tokio/src/worker/group.rs +++ b/rs/moq-tokio/src/worker/group.rs @@ -94,7 +94,7 @@ impl Workers { // hands out one member per slot in the order the kernel numbers them by. let mut forming = moq_sock::shard::Group::acquire(requested, config.count).map_err(|err| match err { moq_sock::shard::Error::Count { count, max } => Error::WorkerCount { count, max }, - moq_sock::shard::Error::Resolve(err) => crate::noq::Error::BindSocket(err).into(), + moq_sock::shard::Error::Resolve(err) => crate::quic::Error::BindSocket(err).into(), // The port lock is the only other way to lose the address, and it is // held by exactly one thing: another group of this UID. _ => Error::WorkerOverlap { addr: requested }, @@ -102,16 +102,16 @@ impl Workers { let count = forming.count(); let mut claims = Vec::with_capacity(count as usize); while let Some(member) = forming.member() { - claims.push(member.bind().map_err(crate::noq::Error::BindSocket)?); + claims.push(member.bind().map_err(crate::quic::Error::BindSocket)?); } - let mut group = forming.complete(claims).map_err(crate::noq::Error::BindSocket)?; + let mut group = forming.complete(claims).map_err(crate::quic::Error::BindSocket)?; let shared = Arc::new(Shared::default()); let mut workers = Vec::with_capacity(count as usize); let mut certificates = None; - while let Some(member) = group.member().map_err(crate::noq::Error::BindSocket)? { + while let Some(member) = group.member().map_err(crate::quic::Error::BindSocket)? { let index = member.shard().index(); // `max(1)` because an empty core list means pinning is off, not that // there are no workers. diff --git a/rs/moq-tokio/src/worker/mod.rs b/rs/moq-tokio/src/worker/mod.rs index 403835a4c0f..1cb9991806a 100644 --- a/rs/moq-tokio/src/worker/mod.rs +++ b/rs/moq-tokio/src/worker/mod.rs @@ -20,10 +20,10 @@ // Everything but the knobs: a group binds and serves QUIC, so it needs a backend // to bind with. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod group; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub use group::{Group, Member, Spawner, Workers}; /// How many QUIC workers to run, and whether to pin them. diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 60973441975..161ebce324c 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -1,4 +1,4 @@ -//! Integration tests for noq QUIC and the iroh transport. +//! Integration tests for QUIC and the iroh transport. use std::time::Duration; @@ -10,7 +10,7 @@ fn timed() -> moq_net::track::Info { const TIMEOUT: Duration = Duration::from_secs(10); /// Inputs for [`connect_test`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct ConnectTest<'a> { /// URL scheme to dial (`moqt` for raw QUIC, `https` for WebTransport). scheme: &'a str, @@ -39,7 +39,7 @@ struct ConnectTest<'a> { /// /// Dials `localhost`, so the client sends an SNI. Use [`no_sni_test`] to cover /// the SNI-less path. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn backend_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -60,7 +60,7 @@ async fn backend_test(scheme: &str) { /// Raw QUIC (`moqt`/`moql`) has no request URI, so the whole request target has to /// ride the SETUP; WebTransport carries it in the CONNECT URL instead. Either way the /// server reports the same route and query through [`moq_tokio::server::Request`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn path_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -80,7 +80,7 @@ async fn path_test(scheme: &str) { /// in the server name). Raw QUIC has no in-band request URL, so this exercises /// the accept path with an empty server name, which must still establish rather /// than reject. Binds the loopback IP directly to avoid dual-stack flakiness. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn no_sni_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -98,7 +98,7 @@ async fn no_sni_test(scheme: &str) { /// Publish a broadcast on the server bound to `bind`, subscribe on a client that /// dials `authority`, and verify the data arrives over the requested scheme. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn connect_test(config: ConnectTest<'_>) { let ConnectTest { scheme, @@ -225,7 +225,7 @@ async fn connect_test(config: ConnectTest<'_>) { /// Write a self-signed PEM cert + key for `name` to `dir`, prefixed by `stem`. /// /// Returns the two paths, in the order the `cert` and `key` lists want them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::path::PathBuf, std::path::PathBuf) { use std::io::Write; @@ -253,7 +253,7 @@ fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::pat /// when the server picked the certificate the SNI asked for. Without SNI /// selection every client would get the first configured certificate, so the /// `alt.localhost` pin below would never match. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn sni_test() { let dir = tempfile::tempdir().expect("tempdir"); let (first_cert, first_key) = write_self_signed(dir.path(), "first", "localhost"); @@ -310,7 +310,7 @@ async fn sni_test() { } /// A generated certificate joins the file-backed ones rather than replacing them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn cert_sources_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -335,7 +335,7 @@ async fn cert_sources_test() { /// Rotate the certificate files under a running listener and assert the served /// set follows, without the listener being rebuilt. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn reload_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -383,7 +383,7 @@ async fn reload_test() { /// Generate a CA, a server cert + key, and a client cert + key (all PEM, the /// leaf certs signed by the CA) written to a tempdir. Returns the dir plus the /// five paths so the caller can wire them into the TLS configs. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { use rcgen::{ BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, KeyUsagePurpose, @@ -436,7 +436,7 @@ fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { } /// Filesystem paths to the PEM material produced by [`generate_mtls_certs`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct MtlsPaths { ca: std::path::PathBuf, server_cert: std::path::PathBuf, @@ -447,7 +447,7 @@ struct MtlsPaths { /// Connect with a client certificate signed by a CA the server trusts, and /// assert the server observes the validated peer certificate via mTLS. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn mtls_test(scheme: &str, reject: bool) { let (_dir, paths) = generate_mtls_certs(); @@ -736,14 +736,14 @@ async fn iroh_connect_test(version: Option<&str>) { .expect("server task failed"); } -// ── Noq backend ───────────────────────────────────────────────────── +// ── QUIC backend ──────────────────────────────────────────────────── /// A client that closes before its runtime stops tells the server at once, instead of /// leaving it to the idle timeout, which is what a process exiting on a signal does. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_reaches_server() { +async fn quic_client_close_reaches_server() { // An idle timeout inside TIMEOUT would hide a lost close. let mut quic = moq_tokio::quic::Config::default(); quic.idle_timeout = Some(TIMEOUT * 3); @@ -802,10 +802,10 @@ async fn noq_client_close_reaches_server() { /// A client that finishes its track and closes before its runtime stops still delivers /// the queued group and the track's finish, instead of the close discarding them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_finished_track() { +async fn quic_client_close_drains_finished_track() { // Small, since the debug build logging every packet is slow and the drain has // one second. Queued right before the close, it is still unacknowledged then. let payload: Vec = (0..1024).map(|i| i as u8).collect(); @@ -916,10 +916,10 @@ async fn noq_client_close_drains_finished_track() { /// A GOAWAY leaves the old session serving its subscriptions after the replacement /// connects, and a close then drains that predecessor too rather than dropping it. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_migrated_predecessor() { +async fn quic_client_close_drains_migrated_predecessor() { // Several congestion windows, so it is still in flight when the close starts. let payload: Vec = (0..64 * 1024).map(|i| i as u8).collect(); @@ -1064,10 +1064,10 @@ async fn noq_client_close_drains_migrated_predecessor() { /// A close cuts a predecessor that cannot drain off at its handover deadline, /// rather than holding it for the full one second close window. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_keeps_predecessor_handover() { +async fn quic_client_close_keeps_predecessor_handover() { let quic = moq_tokio::quic::Config::default(); let mut server_config = moq_tokio::listen::Config::default(); server_config.bind = Some("127.0.0.1:0".parse().unwrap()); @@ -1173,73 +1173,73 @@ async fn noq_client_close_keeps_predecessor_handover() { ); } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic() { +async fn quic_raw_quic() { backend_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_no_sni() { +async fn quic_raw_quic_no_sni() { no_sni_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_path() { +async fn quic_raw_quic_path() { path_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_moql_path() { +async fn quic_raw_quic_moql_path() { path_test("moql").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport_path() { +async fn quic_webtransport_path() { path_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport() { +async fn quic_webtransport() { backend_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_mtls() { +async fn quic_mtls() { mtls_test("https", false).await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_sni_certificate() { +async fn quic_sni_certificate() { sni_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_sources() { +async fn quic_cert_sources() { cert_sources_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_reload() { +async fn quic_cert_reload() { reload_test().await; } @@ -1249,8 +1249,8 @@ async fn noq_cert_reload() { /// files it left behind. /// /// Both ends write into one directory, so this covers the client and server paths -/// at once. Noq writes one file per connection. -#[cfg(all(feature = "qlog", feature = "noq"))] +/// at once. Each endpoint writes one file. +#[cfg(all(feature = "qlog", feature = "quic"))] async fn qlog_test(scheme: &str) -> Vec { let dir = tempfile::tempdir().expect("failed to create tempdir"); @@ -1276,7 +1276,7 @@ async fn qlog_test(scheme: &str) -> Vec { .collect(); for trace in &traces { - // Noq writes JSON-SEQ (RFC 7464): each record is a 0x1e separator then + // qlog streams JSON-SEQ (RFC 7464): each record is a 0x1e separator then // JSON, the first being the qlog header. Checking the bytes rather than just the // length catches a writer that was buffered and never flushed. let raw = std::fs::read(trace).expect("failed to read trace"); @@ -1303,9 +1303,9 @@ async fn qlog_test(scheme: &str) -> Vec { traces } -#[cfg(all(feature = "qlog", feature = "noq"))] +#[cfg(all(feature = "qlog", feature = "quic"))] #[tokio::test] -async fn noq_qlog() { +async fn quic_qlog() { let traces = qlog_test("moqt").await; assert!(!traces.is_empty(), "expected at least one trace"); } @@ -1334,7 +1334,7 @@ async fn connect_once( /// The size is the point: a frame that fits inside one window would pass whether or /// not the setting ever reached the backend. /// -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn window_test(scheme: &str) { let mut quic = moq_tokio::quic::Config::default(); quic.receive_window = Some(64 * 1024); @@ -1357,9 +1357,9 @@ async fn window_test(scheme: &str) { .await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] -async fn noq_windows() { +async fn quic_windows() { window_test("moqt").await; } diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 7cb70653e90..4ee4044de54 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -2481,7 +2481,7 @@ async fn quic_driver_task_inherits_connection_span() { .expect("group closed prematurely"); assert_eq!(&frame.payload[..], b"hello"); - // Other tasks (e.g. noq's own internal per-connection IO loop) legitimately + // Other tasks (e.g. the QUIC driver's per-connection IO loop) legitimately // carry the "conn" span too, since they're spawned while it's entered. So it's // not enough for the marker to appear *somewhere* in the logs; it must be on // the exact line moq-net emits from inside the driver task the runtime diff --git a/rs/moq-tokio/tests/close_code.rs b/rs/moq-tokio/tests/close_code.rs index 9d5f287e97e..c3149c1f774 100644 --- a/rs/moq-tokio/tests/close_code.rs +++ b/rs/moq-tokio/tests/close_code.rs @@ -3,7 +3,7 @@ //! Each case runs several fresh connections, since the losses this guards against were //! races between the peer's close and whatever the transport reported next. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-tokio/tests/stream_code.rs b/rs/moq-tokio/tests/stream_code.rs index 922fe36488a..508a56db8d1 100644 --- a/rs/moq-tokio/tests/stream_code.rs +++ b/rs/moq-tokio/tests/stream_code.rs @@ -6,11 +6,11 @@ //! subscription it never made with STOP_SENDING, and the client reads that code off the //! wire. moq-transport codes take the same raw session. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::{sync::Arc, time::Duration}; -use web_transport_moq::noq; +use moq_tokio::quic::driver; /// moq-lite sends CANCEL (0x1) for a group it has no subscription for. const CANCEL: u32 = 0x1; @@ -68,7 +68,7 @@ async fn serve() -> Server { /// Dial the server over raw QUIC with `alpn`, send an empty SETUP and a GROUP for an /// unknown subscription, and return the code the server stopped the GROUP with. -async fn refused_code(server: &Server, alpn: &str) -> Option { +async fn refused_code(server: &Server, alpn: &str) -> Option { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let mut roots = rustls::RootCertStore::empty(); @@ -77,12 +77,12 @@ async fn refused_code(server: &Server, alpn: &str) -> Option { .with_root_certificates(roots) .with_no_client_auth(); tls.alpn_protocols = vec![alpn.as_bytes().to_vec()]; - let quic = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + let quic = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); - let endpoint = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); + let endpoint = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); let conn = endpoint .connect_with( - noq::ClientConfig::new(Arc::new(quic)), + driver::ClientConfig::new(Arc::new(quic)), ([127, 0, 0, 1], server.port).into(), "localhost", ) diff --git a/rs/moq-tokio/tests/worker.rs b/rs/moq-tokio/tests/worker.rs index c31196628b9..bce5b6607e2 100644 --- a/rs/moq-tokio/tests/worker.rs +++ b/rs/moq-tokio/tests/worker.rs @@ -3,7 +3,7 @@ //! Linux-only, because the mode is: every worker binds the listen address with //! `SO_REUSEPORT`, and no other platform load-balances a unicast UDP port across //! the group. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] use std::net::UdpSocket; diff --git a/rs/moq-transcode/Cargo.toml b/rs/moq-transcode/Cargo.toml index 1f5a9009828..39ef2b9d429 100644 --- a/rs/moq-transcode/Cargo.toml +++ b/rs/moq-transcode/Cargo.toml @@ -44,7 +44,7 @@ tracing = { workspace = true } [dev-dependencies] anyhow = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } # test-util unlocks tokio::time::pause for the deterministic live-feed tests. tokio = { workspace = true, features = ["full", "test-util"] } url = { workspace = true } diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index e373fe2e652..a0e1f2513e1 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -13,13 +13,13 @@ keywords = ["quic", "io-uring", "networking", "linux"] categories = ["asynchronous", "network-programming", "os::linux-apis"] [features] -default = ["noq"] -# moq-noq-proto, with TLS through rustls. -noq = ["dep:moq-noq-proto", "dep:rustls", "dep:rustls-platform-verifier"] +default = ["quic"] +# QUIC and WebTransport on moq-quic, with TLS through rustls. +quic = ["dep:moq-quic", "dep:moq-clock", "dep:rustls", "dep:rustls-platform-verifier"] # Capture qlog traces (`quic::Transport::qlog`). Off by default: it compiles the -# qlog crate and its serde_json encoder into noq, which a production build has +# qlog crate and its serde_json encoder into moq-quic, which a production build has # no use for. -qlog = ["moq-noq-proto?/qlog"] +qlog = ["moq-quic?/qlog"] [target.'cfg(target_os = "linux")'.dependencies] bytes = "1" @@ -27,9 +27,10 @@ http = "1" io-uring = { workspace = true } kio = { workspace = true } libc = { workspace = true } +# The QUIC driver hands moq-quic its clock's instants. +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } -# default-features off so the crypto provider is ours to pick. -moq-noq-proto = { workspace = true, features = ["bloom", "tracing-log", "rustls-aws-lc-rs"], optional = true } +moq-quic = { workspace = true, features = ["bloom", "rustls-aws-lc-rs"], optional = true } moq-sock = { workspace = true } rand = "0.10" rustc-hash = { workspace = true } @@ -56,7 +57,7 @@ web-transport-trait = { workspace = true } [target.'cfg(target_os = "linux")'.dev-dependencies] io-uring = { workspace = true } libc = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } nix = { workspace = true } rcgen = { version = "0.14", default-features = false, features = ["pem", "aws_lc_rs"] } rustls = "0.23" @@ -65,7 +66,6 @@ rustls = "0.23" serde_json = { workspace = true } tempfile = { workspace = true } tracing = "0.1" -web-transport-moq = { workspace = true } [[bench]] name = "udp_tokio" @@ -76,10 +76,9 @@ name = "udp_uring" harness = false [[bench]] -name = "echo_noq" -path = "benches/echo_noq.rs" +name = "echo_quic" harness = false -required-features = ["noq"] +required-features = ["quic"] [[bench]] name = "session_lite" diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index af817d5e938..db5a5a8aeca 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -71,35 +71,35 @@ the tokio stack. ## Backends -The `quic` module uses the sans-IO [moq-noq-proto](https://github.com/moq-dev/noq) -stack with rustls. The `noq` feature is enabled by default and remains optional -so the worker, timers, and UDP socket can be built without QUIC. +The `quic` module uses the sans-IO [moq-quic](../moq-quic) stack with rustls. +The `quic` feature is enabled by default and remains optional so the worker, +timers, and UDP socket can be built without QUIC. | Feature | Stack | TLS | |---|---|---| -| `noq` (default) | [moq-noq-proto](https://github.com/moq-dev/noq) | rustls | +| `quic` (default) | [moq-quic](../moq-quic) | rustls | Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features noq +cargo test -p moq-uring --features quic ``` `moq-relay` enables it with `--features io-uring`. -The `qlog` feature turns on noq's capture support. It writes one file per -connection. +The `qlog` feature turns on moq-quic's capture support. It writes one file per +endpoint on the accept side and per dial on the connect side. ## Validation -`tests/echo.rs` runs an echo against a tokio noq peer over the worker: -handshake, half a megabyte each way, and timers driven by noq's timeout. +`tests/echo.rs` runs an echo against a tokio QUIC peer over the worker: +handshake, half a megabyte each way, and timers driven by moq-quic's timeout. `tests/session.rs` runs full moq-lite sessions through `quic::Endpoint` (including two clients demuxed on one server socket), and `tests/endpoint.rs` covers the endpoint mechanics (dial+accept on one socket, version negotiation, the dial-only refusal), `tests/workers.rs` runs a steered two-worker reuseport group serving one port across threads, and -`tests/web.rs` is WebTransport interop against `web-transport-moq`: stream and +`tests/web.rs` is WebTransport interop against moq-tokio's: stream and datagram echo through the H3 framing, close codes through the capsule, and a full moq-lite session over WebTransport. All of them skip (loudly) below the kernel floor, which @@ -109,8 +109,8 @@ includes GitHub-hosted CI runners. `udp_tokio` and `udp_uring` are the disposable syscall-level matrices from the first spike (recv batching x GRO x GSO, epoll vs io\_uring); see git history -for their methodology. `echo_noq` is the ablation matrix over the real -worker: the same noq echo with receive batching, GRO, and GSO toggled one +for their methodology. `echo_quic` is the ablation matrix over the real +worker: the same QUIC echo with receive batching, GRO, and GSO toggled one at a time. `worker_turns` sweeps socket counts and datagrams per burst independently to measure drive-loop batching without QUIC. Its smoke cases run in the Rust nightly benchmark group. diff --git a/rs/moq-uring/benches/echo_noq.rs b/rs/moq-uring/benches/echo_quic.rs similarity index 93% rename from rs/moq-uring/benches/echo_noq.rs rename to rs/moq-uring/benches/echo_quic.rs index e9c0866f1c3..a7fbc1008dd 100644 --- a/rs/moq-uring/benches/echo_noq.rs +++ b/rs/moq-uring/benches/echo_quic.rs @@ -1,4 +1,4 @@ -//! Echo a tokio noq peer through the io_uring worker while ablating UDP features. +//! Echo a tokio QUIC peer through the io_uring worker while ablating UDP features. use criterion::{criterion_group, criterion_main}; @@ -50,7 +50,7 @@ mod linux { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -72,11 +72,11 @@ mod linux { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); @@ -154,7 +154,7 @@ mod linux { ("all-off", none), ]; - let mut group = c.benchmark_group("echo_noq"); + let mut group = c.benchmark_group("echo_quic"); group.throughput(Throughput::Bytes(PAYLOAD as u64)); for (name, udp_config) in ablations { diff --git a/rs/moq-uring/benches/session_lite.rs b/rs/moq-uring/benches/session_lite.rs index cc629a98b6b..1b4bd616e04 100644 --- a/rs/moq-uring/benches/session_lite.rs +++ b/rs/moq-uring/benches/session_lite.rs @@ -1,7 +1,7 @@ //! The PR-4 ablation matrix at session level: a full moq-lite session pair on //! one worker, publisher to subscriber over loopback QUIC, toggling receive //! batching, GRO, and GSO exactly like the raw echo. Each iteration delivers -//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_noq`'s unit), so +//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_quic`'s unit), so //! the two matrices are directly comparable: the difference is the moq-net //! machine and container framing on top of the same wire path. //! @@ -9,11 +9,11 @@ use criterion::{criterion_group, criterion_main}; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] #[path = "../tests/support.rs"] mod support; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] mod linux { use std::net::UdpSocket; use std::time::Instant; @@ -235,10 +235,10 @@ mod linux { } } -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] use linux::benchmark; -#[cfg(not(all(target_os = "linux", feature = "noq")))] +#[cfg(not(all(target_os = "linux", feature = "quic")))] fn benchmark(_: &mut criterion::Criterion) {} criterion_group!(benches, benchmark); diff --git a/rs/moq-uring/src/lib.rs b/rs/moq-uring/src/lib.rs index da3c2be544c..09f75226f62 100644 --- a/rs/moq-uring/src/lib.rs +++ b/rs/moq-uring/src/lib.rs @@ -21,7 +21,7 @@ //! member of a steered reuseport group ([`udp::Bound`]) issues connection ids //! that steer back to it. [`Handle::run`] supplies time and schedules driver //! wakeups; callers run the returned drivers with [`Handle::spawn`]. The stack -//! underneath is enabled by the `noq` feature; a build without it leaves the +//! underneath is enabled by the `quic` feature; a build without it leaves the //! module out. //! //! [`metrics::Metrics`] is how the worker's own health leaves its thread: @@ -39,7 +39,7 @@ mod error; pub mod metrics; mod park; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub mod quic; mod shared; mod timer; diff --git a/rs/moq-uring/src/quic/noq/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs similarity index 87% rename from rs/moq-uring/src/quic/noq/connection.rs rename to rs/moq-uring/src/quic/driver/connection.rs index 9def53c5ee1..fb70a5a1126 100644 --- a/rs/moq-uring/src/quic/noq/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -7,7 +7,7 @@ use std::task::{Context, Poll}; use std::time::Instant; use bytes::Bytes; -use moq_noq_proto::{ConnectionHandle, Dir, StreamId, VarInt}; +use moq_quic::{ConnectionHandle, Dir, StreamId, VarInt}; use rustc_hash::FxHashMap; use super::super::{Error, SEGMENT}; @@ -28,7 +28,7 @@ const TRAIN_SEGMENTS: usize = 63; /// operation can mutate the connection, drop that borrow, and then register a /// waiter without ever holding both. pub(crate) struct Inner { - pub(crate) conn: RefCell, + pub(crate) conn: RefCell, pub(crate) state: RefCell, /// The worker driving this connection, which anything layered on it /// (the WebTransport handshake, say) runs on too. @@ -44,7 +44,7 @@ pub(crate) struct State { established: bool, establish_waiters: kio::WaiterList, - /// Whoever is waiting for a peer-initiated stream. noq-proto hands the + /// Whoever is waiting for a peer-initiated stream. moq-quic hands the /// ids out in order, so the queue is its own. accept_bi_waiters: kio::WaiterList, accept_uni_waiters: kio::WaiterList, @@ -55,7 +55,7 @@ pub(crate) struct State { /// Per-stream read/write parking, keyed by stream id. /// /// FxHash rather than SipHash on all four of these: they sit on the - /// driver's per-event path, and noq-proto assigns the ids. + /// driver's per-event path, and moq-quic assigns the ids. readable: FxHashMap, writable: FxHashMap, /// Send streams waiting for their end: a FIN the peer acknowledged, or a @@ -83,7 +83,7 @@ pub(crate) struct State { dead: bool, /// The close this side asked for, until the driver has put it on the wire. - /// noq raises no event for it, so this is what the terminal error is + /// moq-quic raises no event for it, so this is what the terminal error is /// built from. local_close: Option<(u64, String)>, } @@ -191,13 +191,13 @@ impl Inner { /// Start tracking send stream `id`, which a handle now owns. /// - /// Seeded from noq rather than empty: a peer can open a bidirectional + /// Seeded from moq-quic rather than empty: a peer can open a bidirectional /// stream and stop it before the application ever accepts it, and that - /// event arrives with no handle to record it against. noq still knows, + /// event arrives with no handle to record it against. moq-quic still knows, /// so a `poll_closed` on the fresh handle reports the stop instead of /// waiting for an event that has already been and gone. pub(crate) fn track(&self, id: StreamId) { - // Err means noq has no send half for the id, which is nothing to + // Err means moq-quic has no send half for the id, which is nothing to // report either way. let stopped = self.conn.borrow_mut().send_stream(id).stopped().ok().flatten(); self.state @@ -242,11 +242,11 @@ impl Inner { /// space, which needs the whole varint range. pub(crate) fn close_code(&self, code: u64, reason: &str) { self.conn.borrow_mut().close( - Instant::now(), + moq_clock::Instant::now(), VarInt::from_u64(code).unwrap_or(VarInt::MAX), Bytes::copy_from_slice(reason.as_bytes()), ); - // noq raises no event for a close the application asked for, so the + // moq-quic raises no event for a close the application asked for, so the // terminal error is ours to publish. Not here though: the driver // publishes it once it has staged the CONNECTION_CLOSE, so a caller // that stops driving the worker the moment `poll_closed` resolves has @@ -355,7 +355,7 @@ impl Clone for Connection { /// Build a connection's shared state and its driver. /// /// The driver future does timers, event sweeps, and egress; ingress arrives -/// from the endpoint's demux task, which feeds noq-proto directly and +/// from the endpoint's demux task, which feeds moq-quic directly and /// [kicks](Inner::kick) the driver. The caller spawns the future and reclaims /// the connection's bookkeeping once it resolves. pub(crate) fn launch( @@ -363,7 +363,7 @@ pub(crate) fn launch( socket: Rc, endpoint: Weak, key: ConnectionHandle, - conn: moq_noq_proto::Connection, + conn: moq_quic::Connection, ) -> (Shared, impl Future + use<>) { let shared = Rc::new(Inner { conn: RefCell::new(conn), @@ -443,7 +443,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu let handshake = conn .crypto_session() .handshake_data() - .and_then(|data| data.downcast::().ok()); + .and_then(|data| data.downcast::().ok()); let (alpn, server_name) = match handshake { Some(data) => ( data.protocol.map(|proto| String::from_utf8_lossy(&proto).into_owned()), @@ -451,11 +451,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu ), None => (None, None), }; - // Multipath is never negotiated here, so the first path is the only one. - let remote = conn - .network_path(moq_noq_proto::PathId::ZERO) - .expect("an established connection has its first path") - .remote(); + let remote = conn.remote_address(); (alpn, server_name, remote) }; @@ -563,7 +559,7 @@ impl web_transport_trait::poll::Session for Connection { Poll::Ready(Ok(())) } // The send queue is full; a flush frees space. - Err(moq_noq_proto::SendDatagramError::Blocked(_)) => { + Err(moq_quic::SendDatagramError::Blocked(_)) => { let mut state = self.shared.state.borrow_mut(); waiter.register(&mut state.datagram_send_waiters); Poll::Pending @@ -608,20 +604,16 @@ impl web_transport_trait::poll::Session for Connection { } fn stats(&self) -> impl web_transport_trait::Stats { - let (stats, path) = { - let mut conn = self.shared.conn.borrow_mut(); - let stats = conn.stats(); - let path = conn.path_stats(moq_noq_proto::PathId::ZERO).unwrap_or_default(); - (stats, path) - }; + let stats = self.shared.conn.borrow().stats(); Stats { bytes_sent: stats.udp_tx.bytes, bytes_received: stats.udp_rx.bytes, - bytes_lost: stats.lost_bytes, + bytes_lost: stats.path.lost_bytes, packets_sent: stats.udp_tx.datagrams, packets_received: stats.udp_rx.datagrams, - packets_lost: stats.lost_packets, - rtt: path.rtt, + packets_lost: stats.path.lost_packets, + rtt: stats.path.rtt, + bandwidth: stats.path.bandwidth_estimate, } } } @@ -632,7 +624,7 @@ impl std::fmt::Debug for Connection { } } -/// A snapshot of noq-proto's counters in [`web_transport_trait::Stats`] +/// A snapshot of moq-quic's counters in [`web_transport_trait::Stats`] /// shape. struct Stats { bytes_sent: u64, @@ -642,6 +634,8 @@ struct Stats { packets_received: u64, packets_lost: u64, rtt: std::time::Duration, + /// The congestion controller's bandwidth estimate in bytes per second. + bandwidth: Option, } impl web_transport_trait::Stats for Stats { @@ -673,24 +667,22 @@ impl web_transport_trait::Stats for Stats { Some(self.rtt) } - /// Nothing, because noq-proto exposes no delivery or pacing rate. + /// The congestion controller's bandwidth estimate, in bits per second. /// - /// The congestion window over the RTT is not a stand-in for one: BBR - /// deliberately holds about twice the bandwidth-delay product (nearly - /// three times it while starting up), so that number is a multiple of - /// what the path will carry, and moq-net feeds this straight into its - /// bandwidth allocator and PROBE. A missing sample is a state the model - /// already handles (`stats.estimated_send_rate` is an `Option`, and no - /// bandwidth producer is created without one); an inflated one is a rate - /// an encoder will chase. + /// BBR estimates the delivery rate; CUBIC offers none. The congestion + /// window over the RTT is not a stand-in: BBR deliberately holds about + /// twice the bandwidth-delay product, so that number is a multiple of what + /// the path will carry, and moq-net feeds this straight into its bandwidth + /// allocator and PROBE. A missing sample is a state the model already + /// handles; an inflated one is a rate an encoder will chase. fn estimated_send_rate(&self) -> Option { - None + self.bandwidth.map(|bytes| bytes.saturating_mul(8)) } } /// The per-connection task: endpoint events, application events, timers, and /// packets out. Packets in come from the endpoint's demux task, which feeds -/// noq-proto directly and kicks this driver. +/// moq-quic directly and kicks this driver. struct Driver { shared: Shared, socket: Rc, @@ -700,11 +692,11 @@ struct Driver { endpoint: Weak, key: ConnectionHandle, deadline: crate::Timer, - /// Egress staging: noq-proto writes into a `Vec`, so a train is built + /// Egress staging: moq-quic writes into a `Vec`, so a train is built /// here and copied into the socket's registered buffer. scratch: Vec, /// A CONNECTION_CLOSE has been handed to the socket. Not implied by a - /// flush that came back empty: noq paces the close like any other packet, + /// flush that came back empty: moq-quic paces the close like any other packet, /// so it can sit behind a pacing timer after the application asked for it. close_staged: bool, } @@ -732,9 +724,9 @@ impl Driver { } // A closed connection still owes the peer its CONNECTION_CLOSE // (and a retransmit for each packet that arrives after), so the - // driver runs until noq says the drain is over. + // driver runs until moq-quic says the drain is over. if self.shared.conn.borrow().is_drained() { - // A close noq never managed to send (the server's + // A close moq-quic never managed to send (the server's // anti-amplification limit can withhold it) ends here too. self.publish_close(); return Poll::Ready(()); @@ -752,11 +744,12 @@ impl Driver { // polling before the set would leave the firing to wake nobody // (fatal on a dial nobody answers, where no ingress ever re-polls // us). - self.deadline.set(self.shared.conn.borrow_mut().poll_timeout()); + self.deadline + .set(self.shared.conn.borrow_mut().poll_timeout().map(super::to_std)); if self.deadline.poll(waiter).is_pending() { return Poll::Pending; } - self.shared.conn.borrow_mut().handle_timeout(Instant::now()); + self.shared.conn.borrow_mut().handle_timeout(moq_clock::Instant::now()); } } @@ -791,25 +784,24 @@ impl Driver { let mut state = self.shared.state.borrow_mut(); match event { - moq_noq_proto::Event::Connected => { + moq_quic::Event::Connected => { state.established = true; state.establish_waiters.wake(); } - moq_noq_proto::Event::ConnectionLost { reason } => state.fail(reason.into()), - moq_noq_proto::Event::DatagramReceived => state.datagram_recv_waiters.wake(), - moq_noq_proto::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), - moq_noq_proto::Event::HandshakeDataReady => {} - moq_noq_proto::Event::Stream(event) => sweep_stream(&mut state, event), - moq_noq_proto::Event::HandshakeConfirmed - | moq_noq_proto::Event::Path(_) - | moq_noq_proto::Event::NatTraversal(_) => {} + moq_quic::Event::ConnectionLost { reason } => state.fail(reason.into()), + moq_quic::Event::DatagramReceived => state.datagram_recv_waiters.wake(), + moq_quic::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), + moq_quic::Event::HandshakeDataReady => {} + moq_quic::Event::Stream(event) => sweep_stream(&mut state, event), + // Non-exhaustive upstream; nothing else needs a wakeup. + _ => {} } } } /// Publish the terminal error for a close this side asked for. /// - /// noq raises no event for it, so the driver is what reports it, and + /// moq-quic raises no event for it, so the driver is what reports it, and /// only once the CONNECTION_CLOSE is staged (or the drain is over), since /// an application is free to stop driving the worker the moment /// `poll_closed` resolves. Staged is not delivered: the send is @@ -832,7 +824,7 @@ impl Driver { // Backpressure, or nothing left to stage. Poll::Pending => return Poll::Pending, } - // Requeue behind the other ready tasks. If noq is drained, the next + // Requeue behind the other ready tasks. If moq-quic is drained, the next // poll costs one empty acquire and then parks normally. waiter.waker().wake_by_ref(); Poll::Pending @@ -858,27 +850,26 @@ impl Driver { tx.len() )))); } - let segments = std::num::NonZeroUsize::new(segments).expect("segments was checked above"); self.scratch.clear(); let (transmit, closing) = { let mut conn = self.shared.conn.borrow_mut(); // Nothing to send, or paced; the buffer returns to the pool on drop. - let Some(transmit) = conn.poll_transmit(Instant::now(), segments, &mut self.scratch) else { + let Some(transmit) = conn.poll_transmit(moq_clock::Instant::now(), segments, &mut self.scratch) else { return Poll::Pending; }; - // Once closed, noq transmits nothing but the CONNECTION_CLOSE. + // Once closed, moq-quic transmits nothing but the CONNECTION_CLOSE. (transmit, conn.is_closed()) }; tx[..transmit.size].copy_from_slice(&self.scratch[..transmit.size]); // A lone datagram is its own segment size, and the socket's GSO - // stride has to match what noq actually packed. + // stride has to match what moq-quic actually packed. let transmit = udp::Transmit { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { return Poll::Ready(Err(Error::Io(err.to_string()))); @@ -913,25 +904,25 @@ fn end(state: &mut State, id: StreamId, end: End) { } /// Apply one stream event to the parking tables. -fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { +fn sweep_stream(state: &mut State, event: moq_quic::StreamEvent) { // Waking removes the entry: a still-interested poller re-registers on its // next poll, so the maps only hold streams somebody is parked on. match event { - moq_noq_proto::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), - moq_noq_proto::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), - moq_noq_proto::StreamEvent::Available { .. } => state.open_waiters.wake(), - moq_noq_proto::StreamEvent::Readable { id } => { + moq_quic::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), + moq_quic::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), + moq_quic::StreamEvent::Available { .. } => state.open_waiters.wake(), + moq_quic::StreamEvent::Readable { id } => { if let Some(mut waiters) = state.readable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Writable { id } => { + moq_quic::StreamEvent::Writable { id } => { if let Some(mut waiters) = state.writable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Finished { id } => end(state, id, End::Delivered), - moq_noq_proto::StreamEvent::Stopped { id, error_code } => { + moq_quic::StreamEvent::Finished { id } => end(state, id, End::Delivered), + moq_quic::StreamEvent::Stopped { id, error_code } => { end(state, id, End::Stopped(error_code.into_inner())); // A writer blocked on capacity has to learn it will never come. if let Some(mut waiters) = state.writable.remove(&id) { @@ -943,7 +934,7 @@ fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { #[cfg(test)] mod tests { - use moq_noq_proto::Side; + use moq_quic::Side; use super::*; diff --git a/rs/moq-uring/src/quic/noq/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs similarity index 92% rename from rs/moq-uring/src/quic/noq/endpoint.rs rename to rs/moq-uring/src/quic/driver/endpoint.rs index 2a7ff9d8e5d..2246efed3ed 100644 --- a/rs/moq-uring/src/quic/noq/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -1,6 +1,6 @@ -//! The noq endpoint: one socket, many noq-proto connections. +//! The endpoint: one socket, many moq-quic connections. //! -//! noq-proto's own [`moq_noq_proto::Endpoint`] is the routing table: it parses +//! moq-quic's own [`moq_quic::Endpoint`] is the routing table: it parses //! each datagram, hands it to the connection its destination id names, mints //! and retires ids as peers consume them, and answers an unsupported version //! itself. What is left for us is the socket, the accept backlog, and a driver @@ -17,7 +17,7 @@ use std::task::Poll; use std::time::Instant; use bytes::BytesMut; -use moq_noq_proto::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; +use moq_quic::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; use rustc_hash::FxHashMap; use super::super::{Error, endpoint::Config}; @@ -42,11 +42,11 @@ pub(crate) struct Inner { socket: Rc, local: SocketAddr, /// The routing table, and the server configuration it accepts with. - endpoint: RefCell, + endpoint: RefCell, accepting: RefCell>, /// Every live connection, looked up per received datagram. /// - /// FxHash rather than SipHash: noq-proto hands out the handle, and it + /// FxHash rather than SipHash: moq-quic hands out the handle, and it /// owns connection-id routing (and its hasher choices) itself, so nothing /// a peer picks reaches this map. conns: RefCell>, @@ -91,7 +91,7 @@ impl Endpoint { let server = match &config.server { Some(server) => { let mut server = super::server_config(server)?; - // noq buffers half-open handshakes itself, so it enforces + // moq-quic buffers half-open handshakes itself, so it enforces // the same bound the accept queue does. server.max_incoming(config.backlog); Some(Arc::new(server)) @@ -101,7 +101,7 @@ impl Endpoint { let accepting = server.is_some().then(|| Accepting { queue: VecDeque::new() }); // MTU discovery is off (the GSO pool sends fixed SEGMENT datagrams), // so the endpoint has no reason to allow it either. - let endpoint = moq_noq_proto::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); + let endpoint = moq_quic::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); let inner = Rc::new(Inner { activity: config.activity, @@ -165,7 +165,7 @@ impl Endpoint { .inner .endpoint .borrow_mut() - .connect(Instant::now(), client, config.peer, &config.server_name) + .connect(moq_clock::Instant::now(), client, config.peer, &config.server_name) .map_err(|err| Error::Quic(err.to_string()))?; let shared = self.inner.launch(key, conn); // Flush the Initial flight; the demux task takes it from here. @@ -246,7 +246,7 @@ impl Inner { /// Route every datagram in one receive, then kick the connections fed. fn demux(self: &Rc, packet: &mut udp::Packet) { let from = packet.from(); - let ecn = packet.ecn().map(super::ecn_to_noq); + let ecn = packet.ecn().map(super::ecn_to_quic); // Where the endpoint writes its own answers (version negotiation, // retry, a refusal), reused across the whole receive. let mut buf = Vec::new(); @@ -261,8 +261,9 @@ impl Inner { // arrived on, so the four-tuple stays half known, exactly as it // does on a platform without `IP_PKTINFO`. let event = self.endpoint.borrow_mut().handle( - Instant::now(), - from.into(), + moq_clock::Instant::now(), + from, + None, ecn, BytesMut::from(&segment[..]), &mut buf, @@ -339,9 +340,14 @@ impl Inner { return None; }; buf.clear(); - let now = Instant::now(); - let deadline = self.handshake.timeout.and_then(|timeout| now.checked_add(timeout)); - let accepted = self.endpoint.borrow_mut().accept(incoming, now, buf, None); + let deadline = self + .handshake + .timeout + .and_then(|timeout| Instant::now().checked_add(timeout)); + let accepted = self + .endpoint + .borrow_mut() + .accept(incoming, moq_clock::Instant::now(), buf, None); let (key, conn) = match accepted { Ok(accepted) => accepted, Err(err) => { @@ -402,7 +408,7 @@ impl Inner { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { tracing::debug!(%err, "failed to send an endpoint response"); @@ -410,7 +416,7 @@ impl Inner { } /// Register `conn`, spawn its driver, and arrange its teardown. - fn launch(self: &Rc, key: ConnectionHandle, conn: moq_noq_proto::Connection) -> connection::Shared { + fn launch(self: &Rc, key: ConnectionHandle, conn: moq_quic::Connection) -> connection::Shared { let (shared, driver) = connection::launch(&self.owner, self.socket.clone(), Rc::downgrade(self), key, conn); self.conns.borrow_mut().insert(key, shared.clone()); @@ -427,8 +433,8 @@ impl Inner { pub(crate) fn on_connection_event( &self, key: ConnectionHandle, - event: moq_noq_proto::EndpointEvent, - ) -> Option { + event: moq_quic::EndpointEvent, + ) -> Option { self.endpoint.borrow_mut().handle_event(key, event) } diff --git a/rs/moq-uring/src/quic/noq/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs similarity index 79% rename from rs/moq-uring/src/quic/noq/mod.rs rename to rs/moq-uring/src/quic/driver/mod.rs index 00d457bcdb4..b1dc8fc86de 100644 --- a/rs/moq-uring/src/quic/noq/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -1,13 +1,13 @@ -//! The rustls-backed noq-proto stack on the worker's UDP path. +//! The rustls-backed moq-quic stack on the worker's UDP path. //! //! TLS goes through rustls, which is what the rest of this workspace speaks, //! so a build selecting this backend links one TLS stack instead of two. //! -//! [`moq_noq_proto::Endpoint`] holds -//! the connection-id routing table, mints and retires ids, answers unsupported -//! versions, and buffers half-open handshakes. So [`Endpoint`] here is mostly -//! the socket plumbing around it, and the parts that are ours (the accept -//! backlog, shard steering, and the driver task per connection. +//! [`moq_quic::Endpoint`] holds the connection-id routing table, mints and +//! retires ids, answers unsupported versions, and buffers half-open +//! handshakes. So [`Endpoint`] here is mostly the socket plumbing around it, +//! plus the parts that are ours: the accept backlog, shard steering, and the +//! driver task per connection. mod connection; mod endpoint; @@ -21,7 +21,7 @@ pub(crate) use connection::{End, Shared}; use std::sync::Arc; -use moq_noq_proto::crypto::rustls::{QuicClientConfig, QuicServerConfig}; +use moq_quic::crypto::rustls::{QuicClientConfig, QuicServerConfig}; use rustls::pki_types::pem::PemObject; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; @@ -37,25 +37,34 @@ const CONNECTION_WINDOW: u32 = 16 * 1024 * 1024; /// How many datagrams to buffer in each direction. const DATAGRAM_WINDOW: usize = 64 * SEGMENT; -fn ecn_to_noq(ecn: udp::Ecn) -> moq_noq_proto::EcnCodepoint { +fn ecn_to_quic(ecn: udp::Ecn) -> moq_quic::EcnCodepoint { match ecn { - udp::Ecn::Ect0 => moq_noq_proto::EcnCodepoint::Ect0, - udp::Ecn::Ect1 => moq_noq_proto::EcnCodepoint::Ect1, - udp::Ecn::Ce => moq_noq_proto::EcnCodepoint::Ce, + udp::Ecn::Ect0 => moq_quic::EcnCodepoint::Ect0, + udp::Ecn::Ect1 => moq_quic::EcnCodepoint::Ect1, + udp::Ecn::Ce => moq_quic::EcnCodepoint::Ce, } } -fn ecn_from_noq(ecn: moq_noq_proto::EcnCodepoint) -> udp::Ecn { +fn ecn_from_quic(ecn: moq_quic::EcnCodepoint) -> udp::Ecn { match ecn { - moq_noq_proto::EcnCodepoint::Ect0 => udp::Ecn::Ect0, - moq_noq_proto::EcnCodepoint::Ect1 => udp::Ecn::Ect1, - moq_noq_proto::EcnCodepoint::Ce => udp::Ecn::Ce, + moq_quic::EcnCodepoint::Ect0 => udp::Ecn::Ect0, + moq_quic::EcnCodepoint::Ect1 => udp::Ecn::Ect1, + moq_quic::EcnCodepoint::Ce => udp::Ecn::Ce, } } -impl From for Error { - fn from(err: moq_noq_proto::ConnectionError) -> Self { - use moq_noq_proto::ConnectionError; +/// Map a moq-quic deadline onto the worker's timers. +/// +/// moq-quic counts on moq-clock's instants while the worker's timers still take +/// std's, until the worker becomes a moq-clock backend. A deadline already past +/// maps to now, which fires at once either way. +fn to_std(at: moq_clock::Instant) -> std::time::Instant { + std::time::Instant::now() + at.saturating_duration_since(moq_clock::Instant::now()) +} + +impl From for Error { + fn from(err: moq_quic::ConnectionError) -> Self { + use moq_quic::ConnectionError; match err { ConnectionError::ApplicationClosed(close) => Self::App { code: close.error_code.into_inner(), @@ -91,10 +100,8 @@ fn provider() -> Arc { /// The endpoint-wide configuration: how connection ids are minted, and the /// largest datagram we tell peers we can receive. -pub(crate) fn endpoint_config( - shard: Option, -) -> Result, Error> { - let mut config = moq_noq_proto::EndpointConfig::default(); +pub(crate) fn endpoint_config(shard: Option) -> Result, Error> { + let mut config = moq_quic::EndpointConfig::default(); config.cid_generator(Arc::new(move || Box::new(Cids { shard }))); config .max_udp_payload_size(SEGMENT as u16) @@ -104,7 +111,7 @@ pub(crate) fn endpoint_config( /// Mints the endpoint's connection ids, steering prefix included. /// -/// noq-proto asks its generator for every id it issues, dials and rotations +/// moq-quic asks its generator for every id it issues, dials and rotations /// alike, so this is the one place the reuseport group's byte has to be /// stamped. #[derive(Debug)] @@ -112,9 +119,9 @@ struct Cids { shard: Option, } -impl moq_noq_proto::ConnectionIdGenerator for Cids { - fn generate_cid(&mut self) -> moq_noq_proto::ConnectionId { - moq_noq_proto::ConnectionId::new(&super::endpoint::cid(self.shard)) +impl moq_quic::ConnectionIdGenerator for Cids { + fn generate_cid(&mut self) -> moq_quic::ConnectionId { + moq_quic::ConnectionId::new(&super::endpoint::cid(self.shard)) } fn cid_len(&self) -> usize { @@ -127,7 +134,7 @@ impl moq_noq_proto::ConnectionIdGenerator for Cids { } /// Dial as `config` says. -pub(crate) fn client_config(config: &client::Config) -> Result { +pub(crate) fn client_config(config: &client::Config) -> Result { let provider = provider(); let builder = rustls::ClientConfig::builder_with_provider(provider.clone()) .with_protocol_versions(&[&rustls::version::TLS13]) @@ -170,21 +177,21 @@ pub(crate) fn client_config(config: &client::Config) -> Result Result { +pub(crate) fn server_config(config: &server::Config) -> Result { let crypto = QuicServerConfig::try_from(config.tls.clone()).map_err(|err| Error::Tls(err.to_string()))?; - let mut server = moq_noq_proto::ServerConfig::with_crypto(Arc::new(crypto)); + let mut server = moq_quic::ServerConfig::with_crypto(Arc::new(crypto)); let transport = transport_config(&config.transport)?; #[cfg(feature = "qlog")] - let transport = with_qlog(transport, &config.transport); + let transport = with_qlog(transport, &config.transport, qlog::Side::Server); server.transport_config(Arc::new(transport)); Ok(server) } @@ -233,53 +240,36 @@ pub(crate) fn identity_server_config( /// Attach the configured qlog sink, if any. /// -/// Noq asks a factory per connection, so each gets a file of its own. +/// moq-quic holds one sink per transport config, which is per endpoint on the +/// accept side and per dial on the connect side; its events carry the +/// connection's qlog `group_id` either way. #[cfg(feature = "qlog")] -fn with_qlog(mut transport: moq_noq_proto::TransportConfig, config: &Transport) -> moq_noq_proto::TransportConfig { +fn with_qlog( + mut transport: moq_quic::TransportConfig, + config: &Transport, + side: qlog::Side, +) -> moq_quic::TransportConfig { let Some(sink) = config.qlog.clone() else { return transport; }; - transport.qlog_factory(Arc::new(Traces { sink })); + let mut qlog = moq_quic::QlogConfig::default(); + qlog.writer(sink.trace(side)); + transport.qlog_stream(qlog.into_stream()); transport } -/// Opens one trace per connection, which is what noq-proto's factory hook is -/// for. -#[cfg(feature = "qlog")] -#[derive(Debug)] -struct Traces { - sink: qlog::Sink, -} - -#[cfg(feature = "qlog")] -impl moq_noq_proto::QlogFactory for Traces { - fn for_connection( - &self, - side: moq_noq_proto::Side, - _remote: std::net::SocketAddr, - initial_dst_cid: moq_noq_proto::ConnectionId, - _now: std::time::Instant, - ) -> Option { - let side = match side { - moq_noq_proto::Side::Client => qlog::Side::Client, - moq_noq_proto::Side::Server => qlog::Side::Server, - }; - Some(moq_noq_proto::QlogConfig::new(self.sink.trace(&initial_dst_cid, side))) - } -} - /// The per-connection knobs both roles share. -fn transport_config(config: &Transport) -> Result { - use moq_noq_proto::VarInt; +fn transport_config(config: &Transport) -> Result { + use moq_quic::VarInt; - let idle = moq_noq_proto::IdleTimeout::try_from(config.idle_timeout) + let idle = moq_quic::IdleTimeout::try_from(config.idle_timeout) .map_err(|_| Error::Quic(format!("idle timeout out of range: {:?}", config.idle_timeout)))?; let streams = VarInt::from_u64(config.max_streams) .map_err(|_| Error::Quic(format!("stream limit out of range: {}", config.max_streams)))?; - let mut transport = moq_noq_proto::TransportConfig::default(); + let mut transport = moq_quic::TransportConfig::default(); transport.max_idle_timeout(Some(idle)); transport.keep_alive_interval(config.keep_alive); transport.max_concurrent_bidi_streams(streams); @@ -301,22 +291,22 @@ fn transport_config(config: &Transport) -> Result Arc::new(moq_noq_proto::congestion::CubicConfig::default()) - as Arc, - Congestion::Delay => Arc::new(moq_noq_proto::congestion::Bbr3Config::default()), + Congestion::Loss => Arc::new(moq_quic::congestion::CubicConfig::default()) + as Arc, + Congestion::Delay => Arc::new(moq_quic::congestion::Bbr3Config::default()), }); Ok(transport) } /// A receive window on the wire is a QUIC varint. One that does not fit cannot /// be applied, so it is an error rather than a smaller window than was asked. -fn varint_window(name: &str, value: u64) -> Result { - moq_noq_proto::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) +fn varint_window(name: &str, value: u64) -> Result { + moq_quic::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) } /// ALPN protocols on the wire, which is a length-prefixed list of byte @@ -410,10 +400,10 @@ impl rustls::client::danger::ServerCertVerifier for NoVerify { mod tests { use super::{CONNECTION_WINDOW, STREAM_WINDOW, Transport, transport_config}; - /// noq keeps the windows private, but its `Debug` prints them. The leading + /// moq-quic keeps the windows private, but its `Debug` prints them. The leading /// space keeps `receive_window` from matching `stream_receive_window`, and /// the trailing comma keeps a shorter value from matching a longer one. - fn has_window(config: &moq_noq_proto::TransportConfig, name: &str, value: u64) -> bool { + fn has_window(config: &moq_quic::TransportConfig, name: &str, value: u64) -> bool { format!("{config:?}").contains(&format!(" {name}: {value},")) } diff --git a/rs/moq-uring/src/quic/noq/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs similarity index 90% rename from rs/moq-uring/src/quic/noq/stream.rs rename to rs/moq-uring/src/quic/driver/stream.rs index bb9d90110e3..e415f57f308 100644 --- a/rs/moq-uring/src/quic/noq/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -1,6 +1,6 @@ -//! Stream handles: thin, direct calls into the shared noq-proto connection. +//! Stream handles: thin, direct calls into the shared moq-quic connection. //! -//! Single-threaded sans-IO means a write goes straight into noq's send +//! Single-threaded sans-IO means a write goes straight into moq-quic's send //! queue (no staging copy) and a read comes straight out of its reassembly //! buffer; the handles just kick the driver so egress reaches the wire and //! park on the per-stream waiter lists the driver wakes. @@ -8,7 +8,7 @@ use std::task::{Context, Poll}; use bytes::{Buf, Bytes, BytesMut}; -use moq_noq_proto::{StreamId, VarInt}; +use moq_quic::{StreamId, VarInt}; use super::super::Error; use super::{End, Shared}; @@ -51,7 +51,7 @@ impl SendStream { self.fin || self.reset } - /// Queue as much of `buf` as noq will take right now, without parking. + /// Queue as much of `buf` as moq-quic will take right now, without parking. /// Best-effort, for the close path where nobody is left to poll. pub(crate) fn try_write(&mut self, buf: &[u8]) -> usize { if self.fin || self.reset { @@ -105,14 +105,14 @@ impl web_transport_trait::poll::SendStream for SendStream { self.shared.kick(); Poll::Ready(Ok(n)) } - // No capacity right now; the driver wakes us when noq reports + // No capacity right now; the driver wakes us when moq-quic reports // the stream writable. - Err(moq_noq_proto::WriteError::Blocked) => { + Err(moq_quic::WriteError::Blocked) => { self.shared.park_writable(self.id, waiter); Poll::Pending } - Err(moq_noq_proto::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), - Err(moq_noq_proto::WriteError::ClosedStream) => { + Err(moq_quic::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), + Err(moq_quic::WriteError::ClosedStream) => { Poll::Ready(Err(Error::Quic("stream already finished".to_string()))) } } @@ -120,7 +120,7 @@ impl web_transport_trait::poll::SendStream for SendStream { fn set_priority(&mut self, order: i32) { // The trait (like W3C sendOrder) sends HIGHER values first, and so - // does noq. + // does moq-quic. let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); } @@ -133,12 +133,12 @@ impl web_transport_trait::poll::SendStream for SendStream { // A STOP_SENDING beat us here. Carry the code like `poll_write` // does, or `moq_net::Error::from_transport` cannot decode a // routine cancellation. - Err(moq_noq_proto::FinishError::Stopped(code)) => { + Err(moq_quic::FinishError::Stopped(code)) => { self.reset = true; return Err(Error::Stop(code.into_inner())); } // Already finished or reset, so the FIN it wanted is out. - Err(moq_noq_proto::FinishError::ClosedStream) => {} + Err(moq_quic::FinishError::ClosedStream) => {} } self.fin = true; self.shared.kick(); @@ -154,7 +154,7 @@ impl web_transport_trait::poll::SendStream for SendStream { if self.reset { return Poll::Ready(Ok(())); } - // noq reports a send stream's end as an event, which the driver + // moq-quic reports a send stream's end as an event, which the driver // records: an acknowledged FIN, or the peer's STOP_SENDING. match self.shared.ended(self.id) { Some(End::Stopped(code)) => Poll::Ready(Err(Error::Stop(code))), @@ -199,7 +199,7 @@ const READ_AHEAD: usize = 64 * 1024; /// How much to ask for per read-ahead chunk. const READ_CHUNK: usize = 8 * 1024; -/// One read out of noq's reassembly buffer. +/// One read out of moq-quic's reassembly buffer. enum Read { /// Bytes, at most as many as were asked for. Chunk(Bytes), @@ -216,12 +216,12 @@ pub struct RecvStream { shared: Shared, id: StreamId, park: kio::Park, - /// Every byte up to the FIN was read out of noq; reads report the end + /// Every byte up to the FIN was read out of moq-quic; reads report the end /// once `backlog` is drained too. finished: bool, /// We stopped the stream; no more reads matter. stopped: bool, - /// Bytes `poll_closed` read ahead, handed to `poll_read` before noq's. + /// Bytes `poll_closed` read ahead, handed to `poll_read` before moq-quic's. backlog: BytesMut, } @@ -261,7 +261,7 @@ impl RecvStream { self.shared.kick(); } - /// Take up to `max` bytes out of noq's reassembly buffer. + /// Take up to `max` bytes out of moq-quic's reassembly buffer. /// /// Reading is what returns the peer's flow control credit, so a read that /// owes it a frame kicks the driver. @@ -279,8 +279,8 @@ impl RecvStream { let read = match chunks.next(max) { Ok(Some(chunk)) => Read::Chunk(chunk.bytes), Ok(None) => Read::Finished, - Err(moq_noq_proto::ReadError::Blocked) => Read::Blocked, - Err(moq_noq_proto::ReadError::Reset(code)) => Read::Reset(code.into_inner()), + Err(moq_quic::ReadError::Blocked) => Read::Blocked, + Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), }; let transmit = chunks.finalize().should_transmit(); drop(conn); @@ -351,7 +351,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { if self.finished || self.stopped { return Poll::Ready(Ok(())); } - // The FIN sits behind whatever the peer sent before it, and noq only + // The FIN sits behind whatever the peer sent before it, and moq-quic only // reports the stream finished once that is read out. Waiting on // readability alone would park behind bytes nobody is reading, so read // ahead into the backlog `poll_read` serves first: this watch resolves diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index a21d8214256..a312ab95849 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -25,7 +25,7 @@ use super::server; -pub use super::noq::Endpoint; +pub use super::driver::Endpoint; /// Every connection id this endpoint issues is this long: long enough to be /// unguessable per socket, and fixed because a short header does not encode diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index 9ed2b811168..f6e84c8959f 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -18,7 +18,7 @@ //! [`web`] layer's HTTP/3 CONNECT handshake on top of the same adapter, with //! [`web::Session`] as the one transport type covering both. //! -//! Noq is the sans-IO QUIC stack underneath the worker. +//! moq-quic is the sans-IO QUIC stack underneath the worker. pub mod client; pub mod endpoint; @@ -28,10 +28,10 @@ pub mod qlog; pub mod server; pub mod web; -mod noq; +mod driver; +pub use driver::{Connection, RecvStream, SendStream}; pub use endpoint::Endpoint; -pub use noq::{Connection, RecvStream, SendStream}; /// The QUIC payload size every full datagram in a GSO train uses, and the /// stride GRO coalesces with. @@ -77,7 +77,7 @@ impl Identity { client_auth: server::ClientAuth, ) -> Result, Error> { let alpn = alpn.iter().map(|proto| proto.as_ref().as_bytes().to_vec()).collect(); - noq::identity_server_config(self, alpn, &client_auth) + driver::identity_server_config(self, alpn, &client_auth) } /// The PEM certificate chain being presented, for a caller that publishes @@ -135,8 +135,9 @@ pub struct Transport { pub keep_alive: Option, /// Where to write qlog traces, or `None` (the default) to write none. /// - /// Noq takes one sink per configuration and writes one file per connection, - /// named from that connection's Initial destination connection ID. + /// moq-quic takes one sink per transport config, so this writes one file + /// per endpoint on the accept side and per dial on the connect side, each + /// event tagged with the qlog `group_id` of the connection it belongs to. /// /// Only compiled with the `qlog` feature, so a build without it cannot ask /// for traces the backends would not produce. @@ -162,9 +163,9 @@ impl Default for Transport { /// The congestion control family a connection runs. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. /// -/// The default is [`Loss`](Self::Loss), which is what noq runs +/// The default is [`Loss`](Self::Loss), which is what moq-quic runs /// unasked. An application carrying live media wants [`Delay`](Self::Delay) /// and should say so; the relay does. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/rs/moq-uring/src/quic/qlog.rs b/rs/moq-uring/src/quic/qlog.rs index 0fe2d8250ce..e3e412763cf 100644 --- a/rs/moq-uring/src/quic/qlog.rs +++ b/rs/moq-uring/src/quic/qlog.rs @@ -8,7 +8,7 @@ //! the worker would stall every connection sharing its core for as long as //! the filesystem takes. //! -//! One file per connection, in the qlog JSON-SEQ format the tokio listener already +//! One file per trace, in the qlog JSON-SEQ format the tokio listener already //! writes, so an existing workflow reads both without knowing which runtime //! produced them. @@ -117,23 +117,14 @@ impl Sink { }) } - /// A writer for one connection's trace, named from its connection id. - pub(crate) fn trace(&self, cid: &[u8], side: Side) -> Box { - self.open(cid, side) - } - - fn open(&self, cid: &[u8], side: Side) -> Box { - use std::fmt::Write as _; - + /// A writer for one trace, which every connection sharing a transport + /// config writes into, each event tagged with its connection's qlog + /// `group_id`. + pub(crate) fn trace(&self, side: Side) -> Box { let inner = self.inner.clone(); + // The trace slot keeps each file unique within the sink. let file = inner.next.fetch_add(1, Ordering::Relaxed); - // The trace slot stays unique within the sink even when a peer reuses - // the same Initial destination connection id. - let cid = cid.iter().fold(String::with_capacity(cid.len() * 2), |mut id, byte| { - let _ = write!(id, "{byte:02x}"); - id - }); - let id = format!("connection{file}-{cid}"); + let id = format!("trace{file}"); let path = inner.dir.join(format!( "moq-{}-{}-{}-{id}-{}.qlog", inner.started, @@ -369,8 +360,8 @@ mod tests { let first = Sink::directory(dir.path()).expect("first sink"); let second = Sink::directory(dir.path()).expect("second sink"); - let mut first_trace = first.trace(b"first", Side::Server); - let mut second_trace = second.trace(b"second", Side::Server); + let mut first_trace = first.trace(Side::Server); + let mut second_trace = second.trace(Side::Server); first_trace.write_all(b"first").expect("write first trace"); second_trace.write_all(b"second").expect("write second trace"); drop(first_trace); @@ -387,14 +378,14 @@ mod tests { } #[test] - fn reused_connection_ids_use_distinct_files() { + fn traces_use_distinct_files() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let mut first = sink.trace(b"same", Side::Server); + let mut first = sink.trace(Side::Server); first.write_all(b"first").expect("write first trace"); drop(first); - let mut second = sink.trace(b"same", Side::Server); + let mut second = sink.trace(Side::Server); second.write_all(b"second").expect("write second trace"); drop(second); drop(sink); @@ -411,7 +402,7 @@ mod tests { fn an_open_trace_accounts_for_staging_and_close() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let trace = sink.trace(b"accounting", Side::Server); + let trace = sink.trace(Side::Server); let accounted = CHUNK + std::mem::size_of::(); for _ in 0..10_000 { @@ -433,7 +424,7 @@ mod tests { let held = MEMORY_MAX - CHUNK + 1; assert!(sink.inner.reserve(held)); - let mut trace = sink.trace(b"dropped", Side::Server); + let mut trace = sink.trace(Side::Server); trace.write_all(b"dropped").expect("write dropped trace"); drop(trace); sink.inner.release(held); diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 915724d1950..192592f9b21 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -1012,7 +1012,7 @@ impl std::fmt::Debug for RecvStream { /// Unmap a raw-QUIC stream code that arrived in the HTTP/3 code space. /// -/// Older `web-transport-moq` raw sessions map RESET_STREAM and STOP_SENDING +/// Older moq-tokio raw sessions map RESET_STREAM and STOP_SENDING /// codes as if they were WebTransport. No `u32` application code reaches that /// range, so a code in it can only be the mapped form. Anything else is the /// peer's own code and passes through. diff --git a/rs/moq-uring/tests/echo.rs b/rs/moq-uring/tests/echo.rs index f50825b136a..2f2fc623d9b 100644 --- a/rs/moq-uring/tests/echo.rs +++ b/rs/moq-uring/tests/echo.rs @@ -1,6 +1,6 @@ -//! A tokio noq peer echoing through the io_uring WebTransport server. +//! A tokio QUIC peer echoing through the io_uring WebTransport server. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -48,7 +48,7 @@ async fn write_finish(send: &mut quic::web::SendStream, mut buf: &[u8]) { } #[test] -fn echo_noq_peer() { +fn echo_tokio_peer() { let Some(mut worker) = worker() else { return }; let handle = worker.handle(); let certs = support::certs().expect("certificates"); @@ -56,7 +56,7 @@ fn echo_noq_peer() { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -78,11 +78,11 @@ fn echo_noq_peer() { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); diff --git a/rs/moq-uring/tests/ecn.rs b/rs/moq-uring/tests/ecn.rs index 5f2c3117969..1f89c4e7a77 100644 --- a/rs/moq-uring/tests/ecn.rs +++ b/rs/moq-uring/tests/ecn.rs @@ -2,7 +2,7 @@ //! the peer's `Packet::ecn` reads back, over IPv4 and IPv6, as a lone //! datagram and as a GSO train, with CE surviving too. Without the TOS and //! traffic-class control messages the peer sees no mark, its ACKs carry no -//! ECN counts, and noq disables ECN on the first one. +//! ECN counts, and moq-quic disables ECN on the first one. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. diff --git a/rs/moq-uring/tests/endpoint.rs b/rs/moq-uring/tests/endpoint.rs index 899bb2cd9bd..a6e41b28e61 100644 --- a/rs/moq-uring/tests/endpoint.rs +++ b/rs/moq-uring/tests/endpoint.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -200,7 +200,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { // Junk first: it must not be fatal, and whatever answer it draws // must be smaller than it was, so the socket is never an - // amplifier. Noq answers a packet naming no connection with a + // amplifier. moq-quic answers a packet naming no connection with a // stateless reset, which is what tells a // peer holding stale state to give up. let junk = [0u8; 64]; @@ -276,7 +276,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { assert!( versions .iter() - .any(|version| moq_noq_proto::DEFAULT_SUPPORTED_VERSIONS.contains(version)), + .any(|version| moq_quic::DEFAULT_SUPPORTED_VERSIONS.contains(version)), "the supported version is offered: {versions:?}" ); drop((endpoint, dial_only)); @@ -382,7 +382,7 @@ fn an_unanswered_dial_times_out() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The dial ends at noq's handshake idle timeout (10s), not the idle timeout. + // The dial ends at moq-quic's handshake idle timeout (10s), not the idle timeout. let dial = dial_config(hole.local_addr().expect("addr")); let result = worker @@ -419,7 +419,7 @@ fn the_backlog_bounds_pending_handshakes() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The server never answers, so the dial ends at noq's handshake idle + // The server never answers, so the dial ends at moq-quic's handshake idle // timeout (10s). let dial = dial_config(server); diff --git a/rs/moq-uring/tests/identity.rs b/rs/moq-uring/tests/identity.rs index 4a1673e2bcc..18bf14da98b 100644 --- a/rs/moq-uring/tests/identity.rs +++ b/rs/moq-uring/tests/identity.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/qlog.rs b/rs/moq-uring/tests/qlog.rs index 9a1ff28ef6b..c732b74bb55 100644 --- a/rs/moq-uring/tests/qlog.rs +++ b/rs/moq-uring/tests/qlog.rs @@ -2,14 +2,12 @@ //! tooling can read. //! //! The point of the test is the two halves the relay depends on: the sink -//! actually produces a file for the connection (per connection on noq and -//! per connection on noq, and -//! every record in it is JSON, so a reader is not handed a truncated or +//! actually produces a file for each end of the connection, and every record in it is JSON, so a reader is not handed a truncated or //! interleaved trace. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "qlog", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "qlog", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/session.rs b/rs/moq-uring/tests/session.rs index 22fef4fb5d2..5571c8e587e 100644 --- a/rs/moq-uring/tests/session.rs +++ b/rs/moq-uring/tests/session.rs @@ -8,7 +8,7 @@ //! Origin drivers run through Tokio's explicit-time adapter while session //! drivers use the worker's clock and timer. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -308,7 +308,7 @@ fn two_lite_sessions_share_the_server_socket() { } /// The client verifies for real, trusting only the certificate the server -/// presents: nothing handshakes unless the configured roots reach noq. +/// presents: nothing handshakes unless the configured roots reach moq-quic. #[test] fn configured_roots_verify_the_server() { let Some(mut worker) = worker() else { return }; diff --git a/rs/moq-uring/tests/teardown.rs b/rs/moq-uring/tests/teardown.rs index ad1383a5e9c..26a8993b958 100644 --- a/rs/moq-uring/tests/teardown.rs +++ b/rs/moq-uring/tests/teardown.rs @@ -10,7 +10,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index 0cfef78d1c2..2fbf2157ba4 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -1,11 +1,11 @@ -//! WebTransport interop: the reference tokio stack (`web-transport-moq`, +//! WebTransport interop: the reference tokio stack (`moq_tokio::quic::web`, //! what browsers interop with) dials the uring server. One test hand-drives //! streams and datagrams through the H3 framing; the other runs a whole //! moq-lite session over it, which is exactly the browser-to-relay path. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -53,9 +53,9 @@ fn h3_endpoint(handle: &moq_uring::Handle, certs: &support::Certs) -> quic::Endp } /// The tokio-side client, in its own runtime on its own thread. -fn noq_client( +fn tokio_client( url: String, - body: impl FnOnce(web_transport_moq::Session) -> ClientFuture + Send + 'static, + body: impl FnOnce(moq_tokio::quic::web::Session) -> ClientFuture + Send + 'static, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { // May already be installed by a sibling test; either way one exists. @@ -65,12 +65,12 @@ fn noq_client( .build() .expect("tokio runtime"); rt.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = - web_transport_moq::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")).with_protocol(PROTO); + let request = moq_tokio::quic::web::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")) + .with_protocol(PROTO); let session = client.connect(request).await.expect("connect"); body(session).await; }); @@ -79,8 +79,8 @@ fn noq_client( type ClientFuture = std::pin::Pin + Send>>; -/// Read a noq-side stream to its end. -async fn read_all(recv: &mut web_transport_moq::RecvStream) -> Vec { +/// Read a tokio-side stream to its end. +async fn read_all(recv: &mut moq_tokio::quic::web::RecvStream) -> Vec { let mut out = Vec::new(); let mut buf = [0u8; 4096]; while let Some(n) = recv.read(&mut buf).await.expect("read") { @@ -126,7 +126,7 @@ fn webtransport_echo_end_to_end() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/echo?token=abc"), |session| { + let client = tokio_client(format!("https://{addr}/echo?token=abc"), |session| { Box::pin(async move { // Bidirectional echo. let (mut send, mut recv) = session.open_bi().await.expect("open_bi"); @@ -239,7 +239,7 @@ fn lite_session_over_webtransport() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/"), |session| { + let client = tokio_client(format!("https://{addr}/"), |session| { Box::pin(async move { assert_eq!(session.protocol(), Some(PROTO), "negotiated subprotocol"); let (sub_origin, sub_driver) = origin::Producer::new(origin::Config::default()); @@ -339,7 +339,7 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future std::thread::JoinHandle { +fn tokio_client_err(url: String) -> std::thread::JoinHandle { std::thread::spawn(move || { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let rt = tokio::runtime::Builder::new_current_thread() @@ -347,12 +347,12 @@ fn noq_client_err(url: String) -> std::thread::JoinHandle break err, } }; - assert!(matches!(err, web_transport_moq::WriteError::Stopped(0)), "got {err:?}"); + assert!( + matches!(err, moq_tokio::quic::web::WriteError::Stopped(0)), + "got {err:?}" + ); session.close(CLOSE_CODE, CLOSE_REASON.as_bytes()); session.closed().await; diff --git a/rs/moq-uring/tests/workers.rs b/rs/moq-uring/tests/workers.rs index d832d4c0a49..e427de24b4d 100644 --- a/rs/moq-uring/tests/workers.rs +++ b/rs/moq-uring/tests/workers.rs @@ -9,7 +9,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index 8c6e634b144..3724bfd7945 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -128,7 +128,7 @@ def test_media_keeps_dependency_contracts_and_vpx_tests(self): CARGO_STUB = r"""#!/bin/bash printf '%s\n' "$*" >> "$FEATURE_CALLS" -if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features noq' ]]; then +if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features quic' ]]; then echo 'a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' >&2 exit 1 fi diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh index 8727b4a233a..555ef55272a 100755 --- a/sh/rs/tokio-features.sh +++ b/sh/rs/tokio-features.sh @@ -19,8 +19,8 @@ cargo clippy --locked -p moq-tokio --all-features -- -D warnings # Only a feature-owner edit needs every isolated permutation on a PR. Shared # dependencies still compile at both extremes; Nightly runs every permutation. if ((all)); then - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,aws-lc-rs -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,ring -- -D warnings for feature in aws-lc-rs ring tcp uds websocket; do cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings @@ -29,12 +29,12 @@ fi # shellcheck disable=SC2016 # the backticks are literal, as rustc prints them want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' -if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 +if output=$(cargo check --locked -p moq-tokio --no-default-features --features quic 2>&1); then + echo "rs: moq-tokio with only quic should reject its missing crypto provider" >&2 exit 1 fi if ! grep -qF "$want" <<<"$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "rs: moq-tokio with only quic should fail on: $want" >&2 echo "$output" | grep -E '^error' >&2 || true exit 1 fi From 866b13d62b58895086916ae116c4d9c958eaf04a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:42 -0700 Subject: [PATCH 04/51] fix(moq-tokio): log a CONNECT stream cut off by the close at debug The capsule reader maps a lost stream to UnexpectedEnd, so every ordinary session close warned "failed to read capsule". A read error while the connection is closed is now debug; one on a live connection still warns. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/web/session.rs | 49 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index ad0fc8c2d69..38e01d66847 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -154,7 +154,7 @@ impl Session { recv: crate::quic::driver::RecvStream, close: Arc, ) { - let close_info = Self::read_capsules(recv).await; + let close_info = Self::read_capsules(&conn, recv).await; let code = close_info.as_ref().map_or(0, |(c, _)| *c); let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); @@ -182,7 +182,10 @@ impl Session { // Keep reading capsules from the CONNECT recv stream until it's closed. // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, // or None if the stream closed without a capsule. - async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + async fn read_capsules( + conn: &crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + ) -> Option<(u32, String)> { let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); loop { match reader.read().await { @@ -194,8 +197,13 @@ impl Session { tracing::warn!(%typ, size = payload.len(), "unknown capsule"); } Ok(None) => return None, + // The capsule reader reports a lost stream as `UnexpectedEnd`, so ask the + // connection: a read cut short by its close is the ordinary teardown. Err(e) => { - tracing::warn!(?e, "failed to read capsule"); + match conn.close_reason() { + Some(reason) => tracing::debug!(?e, %reason, "CONNECT stream ended with the connection"), + None => tracing::warn!(?e, "failed to read capsule"), + } return None; } } @@ -1223,6 +1231,41 @@ mod tests { ); } + /// A CONNECT stream cut off by the connection's close is ordinary teardown, not a warning. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_ends_quietly_with_connection() { + let (client, server, _endpoints) = pair().await; + + // A zero-length reserved frame, so the server sees the stream without a capsule. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x21, 0x00]).await.unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!(!logs_contain("failed to read capsule"), "an ordinary close warned"); + } + + /// A CONNECT stream that ends mid-frame on a live connection still warns. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_truncated_warns() { + let (client, server, _endpoints) = pair().await; + + // A DATA frame that promises 5 bytes and carries 1. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x00, 0x05, 0x01]).await.unwrap(); + send.finish().unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!( + logs_contain("failed to read capsule"), + "a truncated capsule stream was not reported" + ); + } + /// A uni stream reset before its header arrived reports the reset, not a foreign session. #[tokio::test] async fn uni_reset_before_header() { From 88a6fae64385c685bacc5af8d6a1bde783375932 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:46 -0700 Subject: [PATCH 05/51] docs: point fork and advisory guidance at moq-quic Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 7 +++---- rs/AGENTS.md | 2 +- rs/moq-quic/README.md | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbfcde1df54..d168b341cee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,10 +74,9 @@ For non-trivial tasks, file an issue or offer to run `/quest-plan`. # Forks -[moq-dev/noq](https://github.com/moq-dev/noq) publishes `moq-noq*`, the QUIC stack every published MoQ crate builds on; iroh keeps upstream noq. -Its `moq-sync` workflow merges n0-computer/noq weekly as a PR; review it like any other, and `PARENT` names the upstream commit each release includes. -A carried change lists its upstream PR, or the reason it has none, in the fork PR. -For an advisory against noq or Quinn, compare the pinned release's `PARENT` with the fixing upstream commit, then sync, release the fork, and bump the pin here. +`rs/moq-quic` is our hard fork of quinn, the QUIC stack MoQ builds on; iroh keeps upstream noq. +It never merges upstream: fixes are cherry-picked by hand, and each carried change is listed in its README with its upstream PR, or the reason it has none. +For an advisory against Quinn, follow the advisory triage in [`rs/moq-quic/README.md`](rs/moq-quic/README.md). # Releases diff --git a/rs/AGENTS.md b/rs/AGENTS.md index d9bd16d6b4a..52ff1bc226f 100644 --- a/rs/AGENTS.md +++ b/rs/AGENTS.md @@ -5,7 +5,7 @@ The `/rs` Cargo workspace. Extends the root `AGENTS.md`. One crate per component from the root list, named `moq-` (`hang` is the exception). Keep them modular; a crate does one thing. Beyond that list: - `kio`: "easy async" primitives everything else polls through. -- `moq-tokio`: configures the QUIC backends (Quinn/Quiche/Noq/Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. +- `moq-tokio`: configures the QUIC backends (`moq-quic`, our quinn fork, and Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. - `moq-cli` builds the `moq` binary and owns the CLI surface for the gateway crates. Binaries never carry a `-cli` suffix. `moq-net`, `moq-mux`, `moq-relay` (including the config conventions every binary shares), and `moq-ffi` have their own `AGENTS.md`. diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index d47b7629b0b..b31ec65aae4 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -72,4 +72,4 @@ Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges `cargo audit` cannot match the renamed crate, so security fixes are tracked by hand. Watch quinn-rs/quinn's [security advisories](https://github.com/quinn-rs/quinn/security/advisories) and releases (including the `0.11.x` branch, which sometimes gets a fix `main` does not need). -For each quinn-proto or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. +For each quinn, quinn-proto, or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. From 2d5026751bea6cecf9e598ee73a0c9fb94a0f486 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:06:36 -0700 Subject: [PATCH 06/51] docs: format the upgrade note Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index e0e6502d998..2bd06786250 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -28,7 +28,7 @@ These land with the next breaking release, not the 2026-09-23 train. it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one - file per endpoint instead of one per connection; on io_uring workers, one per + file per endpoint instead of one per connection; on io\_uring workers, one per endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC From c2993ca71eeaed7bc3fa31d28832fddc96fc2a7b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:16:50 -0700 Subject: [PATCH 07/51] quest: finish quest/m1/quic/fork/switch Relay peak RSS on the fanout workloads matches moq-noq on origin/main, which was the last open part of the Goal. Co-Authored-By: Claude Opus 5.5 --- quest/m1/p2p/native-quic.md | 5 +-- quest/m1/p2p/udp-demux.md | 4 +-- quest/m1/quic/fork/README.md | 10 ++---- quest/m1/quic/fork/switch.md | 58 -------------------------------- quest/m1/quic/receive-credit.md | 4 --- quest/m1/time/README.md | 5 ++- quest/m1/time/relay-seam.md | 1 - quest/m1/time/uring.md | 1 - quest/m2/bbr-ack-cleanup.md | 1 - quest/m2/quic-bbr-loss-parity.md | 4 --- quest/m2/quic-sim.md | 3 +- 11 files changed, 8 insertions(+), 88 deletions(-) delete mode 100644 quest/m1/quic/fork/switch.md diff --git a/quest/m1/p2p/native-quic.md b/quest/m1/p2p/native-quic.md index 912de3082f5..81626f6fd3a 100644 --- a/quest/m1/p2p/native-quic.md +++ b/quest/m1/p2p/native-quic.md @@ -21,9 +21,7 @@ Decided with the maintainer on 2026-10-13: candidates through the module [webrtc](/quest/m1/p2p/webrtc.md) lifts out of `moq-rtc`, with host candidates, `.local` resolution, and the STUN Binding client; don't copy it. -- QUIC is `moq-quic`, fed datagrams from the nominated pair, so this waits - on [the switch](/quest/m1/quic/fork/switch.md) instead of building on - `moq-noq` and porting it. +- QUIC is `moq-quic`, fed datagrams from the nominated pair. - Authentication: each side generates a self-signed certificate and sends its SHA-256 fingerprint in its offer or answer, as DTLS fingerprints ride in SDP. Both sides pin it with the existing `tls.fingerprint` check. MoQ @@ -61,7 +59,6 @@ covers LAN and NAT pairs between two `moq-cli --p2p` nodes. ## Required -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - `moq-quic` replaces `moq-noq` in moq-tokio - [UDP demux](/quest/m1/p2p/udp-demux.md) - the classifier that splits STUN from QUIC - [Native data channel transport](/quest/m1/p2p/webrtc.md) - the shared ICE gathering module - [Signaling and policy](/quest/m1/p2p/signal.md) - the offer schema and roster this extends diff --git a/quest/m1/p2p/udp-demux.md b/quest/m1/p2p/udp-demux.md index f3d97c8776d..01bcd15e71f 100644 --- a/quest/m1/p2p/udp-demux.md +++ b/quest/m1/p2p/udp-demux.md @@ -32,9 +32,7 @@ one. Bound queues per stack with a per-source drop rather than unbounded growth; report drops as a counter. QUIC: the server takes the virtual socket through whatever the QUIC stack -accepts: noq's `new_with_abstract_socket` today, `moq-quic`'s fed datagrams -after [the switch](/quest/m1/quic/fork/switch.md). Build on whichever has -landed rather than both. QUIC-bit greasing is already off, from +accepts: the driver's `new_with_abstract_socket` on `moq-quic`. QUIC-bit greasing is already off, from [shard steering](/quest/m1/p2p/shard-steering.md). STUN: an ICE connectivity check is a Binding request carrying USERNAME, so diff --git a/quest/m1/quic/fork/README.md b/quest/m1/quic/fork/README.md index 65e1f272192..091f1bafe67 100644 --- a/quest/m1/quic/fork/README.md +++ b/quest/m1/quic/fork/README.md @@ -1,4 +1,4 @@ -# Hard-fork quinn into the monorepo as moq-quic +# [XS] Hard-fork quinn into the monorepo as moq-quic ## Goal @@ -55,12 +55,8 @@ them: This README's own work: delete moq-dev/noq's `moq-sync.yml` and mark its README frozen. Advisory triage is documented in `rs/moq-quic/README.md`. Every carried change in moq-dev/noq's `CHANGELOG-MOQ.md` -is either ported by a child quest or recorded as not applicable in the -[switch](/quest/m1/quic/fork/switch.md) PR. - -## Required - -- [Switch](/quest/m1/quic/fork/switch.md) - quinn's async layer and `web-transport-moq` join moq-tokio, both runtimes run on `moq-quic`, `moq-noq*` is gone, and relay memory matches `moq-noq` +is either ported by a child quest or recorded as not applicable in +[#5664](https://github.com/moq-dev/moq/pull/5664). ## Related diff --git a/quest/m1/quic/fork/switch.md b/quest/m1/quic/fork/switch.md deleted file mode 100644 index 32e547d53e7..00000000000 --- a/quest/m1/quic/fork/switch.md +++ /dev/null @@ -1,58 +0,0 @@ -# [L] Switch MoQ onto moq-quic - -## Goal - -quinn's async layer and `web-transport-moq` live in moq-tokio as modules, -moq-tokio and moq-uring run on `moq-quic`, and no workspace crate depends on -`moq-noq*`. Upstream `noq-proto` remains only for the `iroh` feature. Relay -memory on the bulk and fanout workloads matches `moq-noq`. - -## Plan - -Decided 2026-10-06: `moq-quic` (quinn-proto) is the only crate the fork adds. -quinn's async layer is imported into moq-tokio as a module rather than a -`moq-quic-tokio` crate, tokio-only: drop quinn's `Runtime` abstraction and -its smol and async-io implementations instead of carrying them. Its UDP -sockets come from `moq_sock::udp`. -Keep the first commit verbatim from upstream (quinn's `quinn/src`) so a -reviewer can diff it, and extend the cherry-pick recipe in -`rs/moq-quic/README.md` to map `quinn/src/` onto the module. - -Move `web-transport-moq` from moq-dev/noq into moq-tokio as a module too. It -derives from web-transport-quinn; its noq-only parts are `PathId::ZERO` and -`path_stats`. Report bandwidth from quinn's `PathStats::bandwidth_estimate` -instead of the current cwnd/rtt guess; it is in bytes per second and -`web-transport-trait` wants bits. moq-uring uses it only in tests, -through moq-tokio. - -BBR3 is already `moq-quic`'s default, with `Bbr3Config` and `Bbr3` named as -in noq, so the `Delay` family maps over unchanged. Run the benchmark matrix -against `moq-noq` 1.3.x on the same workloads and report any throughput or -latency difference; the BBR3 port could not, with no relay on `moq-quic` yet. - -moq-uring built against both quinn-proto and noq-proto until #3811, behind -about 20 `cfg` lines (stats fields, CID generator, qlog, BBR). Rename -`rs/moq-uring/src/quic/noq` by role and use that history as the map. Keep -moq-uring's `qlog` feature working on quinn's qlog. - -Rename the `noq` cargo features by role (`quic` is the recommendation); -confirm the name with the maintainer in the PR. Update every doc and example -that names noq, and run `just test interop --all`. - -Decided 2026-10-10: `moq-quic` takes `moq_clock::Instant` at its boundary in -place of `std::time::Instant`, so the sim can drive real QUIC later -([QUIC on the sim](/quest/m2/quic-sim.md)) and the io_uring worker's per-turn -sample passes straight through. `moq_clock::Instant` mirrors std's methods, so -alias it in `moq-quic` and keep the swap an import change; that keeps -upstream cherry-picks mechanical. - -In the PR, list each carried change from moq-dev/noq's `CHANGELOG-MOQ.md` as -ported (with its quest) or not applicable (with the reason). - -Lazy stream slots ([quinn#2601](https://github.com/quinn-rs/quinn/pull/2601), -the change noq took as noq#667) arrived with the import at quinn `7616e6b2`, -so only its measurement remains, and that needs the relay on `moq-quic`. -Re-run #3342's bulk and fanout relay memory workloads after the switch and -report them in the PR. On `moq-noq` they measured 75 MiB (bulk) and 31 MiB -(fanout), against 141 and 97 MiB without lazy slots; `moq-quic` should land -near the former. diff --git a/quest/m1/quic/receive-credit.md b/quest/m1/quic/receive-credit.md index c2bd474fbbd..9f939258768 100644 --- a/quest/m1/quic/receive-credit.md +++ b/quest/m1/quic/receive-credit.md @@ -47,10 +47,6 @@ with its consumer. Wire: existing QUIC flow-control and cancellation frames; no new MoQ message or negotiation. Browser and iroh receivers do not gain a capability their transports cannot provide. -## Required - -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - both native adapters use the in-tree transport - ## Related - [Bound message assembly](/quest/m1/quic/receive-memory.md) - retains connection credit until a complete message is handed off diff --git a/quest/m1/time/README.md b/quest/m1/time/README.md index 9b070511dd7..b4f507c457d 100644 --- a/quest/m1/time/README.md +++ b/quest/m1/time/README.md @@ -85,9 +85,8 @@ reasons, so later sessions don't reopen them: [moq-uring on moq-clock](/quest/m1/time/uring.md). The relay seam [bench-relay](/quest/m2/bench-relay.md) needed is [the relay seam](/quest/m1/time/relay-seam.md). Quests that already asked - for "mocked time" Require the core quest. moq-quic adopts the logical - instant in the [switch](/quest/m1/quic/fork/switch.md), which Requires the - crate. + for "mocked time" Require the core quest. moq-quic takes the logical + instant at its boundary. This milestone verifies the exported clock contract and production consumers needed by the QUIC switch and clustering regressions. Unrelated test/private diff --git a/quest/m1/time/relay-seam.md b/quest/m1/time/relay-seam.md index 2bf68fdcb57..5a938ca285e 100644 --- a/quest/m1/time/relay-seam.md +++ b/quest/m1/time/relay-seam.md @@ -33,4 +33,3 @@ Public API: the in-memory session and the relay seam. Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the sim and instant the relay tests run on -- [Switch](/quest/m1/quic/fork/switch.md) - moq-tokio's server types settle first diff --git a/quest/m1/time/uring.md b/quest/m1/time/uring.md index 6151101783f..6987317e340 100644 --- a/quest/m1/time/uring.md +++ b/quest/m1/time/uring.md @@ -36,7 +36,6 @@ Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the driver's instant becomes moq-clock's -- [Switch](/quest/m1/quic/fork/switch.md) - the QUIC driver this edits moves onto `moq-quic` ## Closes diff --git a/quest/m2/bbr-ack-cleanup.md b/quest/m2/bbr-ack-cleanup.md index 4791c2bb469..00462c87cc5 100644 --- a/quest/m2/bbr-ack-cleanup.md +++ b/quest/m2/bbr-ack-cleanup.md @@ -54,6 +54,5 @@ packet-lifetime comments inline; no new user guide is needed. ## Related -- [Switch](/quest/m1/quic/fork/switch.md) - not a blocker: the core and its BBR3 are already in `rs/moq-quic` - [Benchmark reporting](/bench/README.md) - reusable measurement guidance, not a prerequisite for this fix - [Loss sampling](/quest/m2/quic-bbr-loss-parity.md) - preserve packet metadata needed by the separate loss-sample repair; both edit `bbr3/mod.rs`, so sequence them diff --git a/quest/m2/quic-bbr-loss-parity.md b/quest/m2/quic-bbr-loss-parity.md index b936b04d1cf..83639152a56 100644 --- a/quest/m2/quic-bbr-loss-parity.md +++ b/quest/m2/quic-bbr-loss-parity.md @@ -39,7 +39,3 @@ traffic yet. ## Required - [BBR ACK cleanup](/quest/m2/bbr-ack-cleanup.md) - edits the same `bbr3/mod.rs` packet bookkeeping and lands first; this keeps the metadata it preserves - -## Related - -- [Switch](/quest/m1/quic/fork/switch.md) - MoQ runs on `moq-quic`; not a blocker, since this is a local `rs/moq-quic` fix diff --git a/quest/m2/quic-sim.md b/quest/m2/quic-sim.md index 9e3c21b8295..423cd102c06 100644 --- a/quest/m2/quic-sim.md +++ b/quest/m2/quic-sim.md @@ -11,8 +11,7 @@ real QUIC are tested deterministically with no sockets. Decided 2026-10-10: the in-memory session comes first ([the relay seam](/quest/m1/time/relay-seam.md)); simulated UDP follows. -`moq-quic` already takes `moq_clock::Instant` after the -[switch](/quest/m1/quic/fork/switch.md), so this is a datagram link plus a +`moq-quic` already takes `moq_clock::Instant`, so this is a datagram link plus a driver. Start from moq-shaper's in-memory `UdpSocket` (`rs/moq-shaper/src/mem.rs`). BBR's controller-level `Sim` in `bbr3/mod.rs` stays separate unless this replaces it cleanly. From 5c7093f3321dbed87a67b588fdf866b5973c9623 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:46:48 -0700 Subject: [PATCH 08/51] claim quest/m1/quic/receive-credit Co-Authored-By: Claude Opus 5.5 From 15d40762718147f5410060af525c78a30eb860b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:27:42 -0700 Subject: [PATCH 09/51] feat(quic): hold connection credit until the receiver releases it A moq-quic read can now keep its connection flow control credit: `Chunks::hold` takes it as a `Credit`, and `Connection::release` returns it. Stream credit still follows reads, so a finite stream window never forces a whole message to fit. Every terminal path already returned only unread bytes, so held bytes account once whether they are released before or after the stream is reclaimed. Credit too small to clear the usual MAX_DATA batching threshold is announced once the peer is within that threshold of its limit, on release or on the next arrival. Held credit may never add up to the threshold, and the receiver ignores DATA_BLOCKED, so without this a small release could leave the peer blocked indefinitely. The tokio and io_uring drivers opt in per receive stream with `hold` and return bytes with `release`; dropping the stream releases the rest. io_uring's read-ahead is not releasable by the application and returns on stop. `ConnectionStats::credit` reports held and still-available credit. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/README.md | 1 - quest/m1/quic/receive-credit.md | 54 ---- quest/m1/quic/receive-memory.md | 11 +- rs/moq-quic/src/connection/mod.rs | 19 +- rs/moq-quic/src/connection/stats.rs | 13 + rs/moq-quic/src/connection/streams/mod.rs | 2 +- rs/moq-quic/src/connection/streams/recv.rs | 58 +++++ rs/moq-quic/src/connection/streams/state.rs | 266 +++++++++++++++++++- rs/moq-quic/src/lib.rs | 8 +- rs/moq-quic/src/tests/mod.rs | 117 +++++++++ rs/moq-tokio/src/quic/driver/recv_stream.rs | 39 ++- rs/moq-tokio/src/quic/driver/tests.rs | 44 ++++ rs/moq-uring/src/quic/driver/connection.rs | 6 + rs/moq-uring/src/quic/driver/stream.rs | 70 +++++- rs/moq-uring/tests/credit.rs | 163 ++++++++++++ sh/gh/nightly.sh | 1 + 16 files changed, 790 insertions(+), 82 deletions(-) delete mode 100644 quest/m1/quic/receive-credit.md create mode 100644 rs/moq-uring/tests/credit.rs diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index 5b00ba7b8ea..5321e491f42 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -68,7 +68,6 @@ cache limits remain separate; bitrate pacing composes with this memory cap. ## Required - [Hard fork](/quest/m1/quic/fork/README.md) - quinn in-tree as `moq-quic`, with BBR3 and lazy stream slots, and MoQ switched onto it -- [Retain connection credit](/quest/m1/quic/receive-credit.md) - native receivers own connection credit independently of stream reads - [Bound message assembly](/quest/m1/quic/receive-memory.md) - complete headers and frames return credit, and receiver priority cancellation restores progress - [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - `RESET_STREAM_AT`, so a reset WebTransport stream still delivers its header diff --git a/quest/m1/quic/receive-credit.md b/quest/m1/quic/receive-credit.md deleted file mode 100644 index 9f939258768..00000000000 --- a/quest/m1/quic/receive-credit.md +++ /dev/null @@ -1,54 +0,0 @@ -# [L] Return connection credit when the receiver releases bytes - -## Goal - -Native receivers can read QUIC stream bytes without returning connection -credit until their consumer releases those bytes. The same ownership works -through the tokio and io_uring adapters, including cancellation and teardown. -Stream credit remains independent, so a finite stream window does not force -an entire application message to fit before more bytes arrive. - -## Plan - -Decided in the 2026-10-10 receive-memory interview: use `moq-quic` after the -fork switch, not a new feature in the frozen moq-dev/noq fork. This quest -provides the transport primitive; [message assembly](/quest/m1/quic/receive-memory.md) -activates it with the recovery policy needed for progress. - -Today `Chunks::finalize` returns both kinds of credit before the parser has -a complete message. Separate returning borrowed transport state from -returning connection credit. Carry ownership through the native adapters -without copying payloads solely to track credit. Prefer an owned handle or -explicit consumption accounting over callbacks; keep the surface limited to -what the message-assembly consumer needs. - -Replenish `MAX_STREAM_DATA` on reads. Defer `MAX_DATA` until the owner releases -the bytes, exactly once, even after FIN, STOP_SENDING, RESET_STREAM, or stream -state reclamation. Unread discarded bytes, late data on stopped streams, -reset final-size gaps, and already-read retained bytes must not leak or -return the same credit twice. Releasing useful credit must eventually wake -a blocked peer even below the normal window-update threshold; do not depend -on optional DATA_BLOCKED frames. Runtime window shrink debt still applies. - -Expose enough credit accounting and events for MoQ to detect assembly -pressure. QUIC performs cancellation but does not choose application victims. -A sender's write being blocked, or its data being acknowledged, does not -prove receiver assembly deadlock. No sender-side shedding is added. - -Tests use the transport simulator and controlled time: stream credit advances -while connection credit stays held, explicit release resumes the peer, and -all terminal paths account once even when retained bytes outlive the stream. -Cover a useful release smaller than the update threshold and compose with -reliable reset when available. Wire adapter tests into CI and measure the -accounting cost across stream counts and payload sizes. - -Public API: a native receive-credit ownership/observation seam, finalized -with its consumer. Wire: existing QUIC flow-control and cancellation frames; -no new MoQ message or negotiation. Browser and iroh receivers do not gain a -capability their transports cannot provide. - -## Related - -- [Bound message assembly](/quest/m1/quic/receive-memory.md) - retains connection credit until a complete message is handed off -- [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - compose retained credit with reliable prefixes and final-size accounting -- [Relay peers get wider limits](/quest/m1/quic/peer-limits.md) - runtime receive-window changes preserve retained-credit accounting diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md index f027c782966..5e217f71ffc 100644 --- a/quest/m1/quic/receive-memory.md +++ b/quest/m1/quic/receive-memory.md @@ -18,6 +18,13 @@ peer-visible QUIC flow control instead of a separate header quota error. The native transport primitive is a separate prerequisite; do not activate whole-message retention without its recovery policy in this quest. +That primitive landed as a provisional seam to finalize here: `moq-quic` +reads can hold their connection credit until released, both native drivers' +receive streams opt in per stream and release by byte count (dropping +releases the rest), and connection stats report held and still-available +credit. moq-net's transport trait and the WebTransport wrappers do not carry +it yet; reshape any of it to fit the consumer. + Hold connection credit through reader buffers, type/length prefixes, headers, extensions, and frame assembly. Attribute read-ahead bytes to the next unit rather than releasing them with the completed one. Release on complete-unit @@ -86,10 +93,6 @@ Browser/iroh receive-side guarantees and completed-cache limits are outside this quest. Update existing configuration and transport docs inline; no separate guide quest is needed. -## Required - -- [Retain connection credit](/quest/m1/quic/receive-credit.md) - core and native adapters support separate byte ownership and stream-credit release - ## Related - [Scope track priority](/quest/m1/track-priority-scope.md) - broadcast fairness domains and subscription priority semantics diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index e828b460551..50ac930e1b7 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -76,7 +76,7 @@ use spaces::Retransmits; use spaces::{PacketNumberFilter, PacketSpace, SendableFrames, SentPacket, ThinRetransmits}; mod stats; -pub use stats::{ConnectionStats, FrameStats, PathStats, UdpStats}; +pub use stats::{ConnectionStats, CreditStats, FrameStats, PathStats, UdpStats}; mod streams; #[cfg(fuzzing)] @@ -84,7 +84,7 @@ pub use streams::StreamsState; #[cfg(not(fuzzing))] use streams::StreamsState; pub use streams::{ - Chunks, ClosedStream, FinishError, ReadError, ReadableError, RecvStream, SendStream, + Chunks, ClosedStream, Credit, FinishError, ReadError, ReadableError, RecvStream, SendStream, ShouldTransmit, StreamEvent, Streams, WriteError, Written, }; @@ -1358,10 +1358,25 @@ impl Connection { stats.path.cwnd = self.path.congestion.window(); stats.path.bandwidth_estimate = self.path.congestion.metrics().bandwidth_estimate; stats.path.current_mtu = self.path.mtud.current_mtu(); + stats.credit = self.streams.credit_stats(); stats } + /// Return connection flow control credit taken by [`Chunks::hold()`] to the peer. + /// + /// If [`ShouldTransmit::should_transmit()`] returns `true`, call + /// [`Self::poll_transmit()`] soon so the peer learns of the larger window. + /// + /// # Panics + /// If more credit is released than this connection's reads have held, which means `credit` + /// came from another connection. + pub fn release(&mut self, credit: Credit) -> ShouldTransmit { + let transmit = self.streams.release(credit); + self.spaces[SpaceId::Data].pending.max_data |= transmit.should_transmit(); + transmit + } + /// Ping the remote endpoint /// /// Causes an ACK-eliciting packet to be transmitted. diff --git a/rs/moq-quic/src/connection/stats.rs b/rs/moq-quic/src/connection/stats.rs index b62cede34eb..1a6dab4ba3b 100644 --- a/rs/moq-quic/src/connection/stats.rs +++ b/rs/moq-quic/src/connection/stats.rs @@ -179,4 +179,17 @@ pub struct ConnectionStats { pub frame_rx: FrameStats, /// Statistics related to the current transmission path pub path: PathStats, + /// Connection-level receive flow control + pub credit: CreditStats, +} + +/// Connection-level receive flow control, from the receiver's side +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct CreditStats { + /// Bytes read with [`Chunks::hold()`](crate::Chunks::hold) whose credit is not released yet + pub held: u64, + /// Bytes the peer may still send before it needs more credit, including credit not yet + /// advertised + pub available: u64, } diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index 745555db11b..b9ef4c3dec0 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -12,7 +12,7 @@ use crate::{Dir, StreamId, VarInt, connection::streams::state::get_or_insert_rec mod recv; use recv::Recv; -pub use recv::{Chunks, ReadError, ReadableError}; +pub use recv::{Chunks, Credit, ReadError, ReadableError}; mod send; pub(crate) use send::{ByteSlice, BytesArray}; diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index c7fcf5d773b..8ab1e4d212b 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -336,10 +336,22 @@ impl<'a> Chunks<'a> { } } + /// Keep the connection flow control credit for the bytes read so far. + /// + /// Stream credit still advances on [`Self::finalize()`], but the connection credit for these + /// bytes is withheld from the peer until the returned [`Credit`] is passed to + /// [`Connection::release()`][crate::Connection::release]. + pub fn hold(&mut self) -> Credit { + let bytes = mem::take(&mut self.read); + self.streams.held += bytes; + Credit { bytes } + } + /// Mark the read data as consumed from the stream. /// /// The number of read bytes will be released from the congestion window, /// allowing the remote peer to send more data if it was previously blocked. + /// Bytes taken by [`Self::hold()`] keep their connection credit. /// /// If [`ShouldTransmit::should_transmit()`] returns `true`, /// a packet needs to be sent to the peer informing them that the stream is unblocked. @@ -388,6 +400,52 @@ impl Drop for Chunks<'_> { } } +/// Connection flow control credit for bytes read with [`Chunks::hold()`]. +/// +/// The peer cannot reuse this credit until it is passed to +/// [`Connection::release()`][crate::Connection::release] on the connection it came from, so +/// dropping it leaks receive window. +#[must_use = "dropping Credit leaks connection flow control credit"] +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Credit { + bytes: u64, +} + +impl Credit { + /// The number of bytes this credit covers. + pub fn len(&self) -> u64 { + self.bytes + } + + /// Whether this credit covers no bytes. + pub fn is_empty(&self) -> bool { + self.bytes == 0 + } + + /// Split off the credit for the first `bytes`, keeping the rest. + /// + /// # Panics + /// If `bytes` exceeds [`Self::len()`]. + pub fn split(&mut self, bytes: u64) -> Self { + assert!( + bytes <= self.bytes, + "split {bytes} of {} held bytes", + self.bytes + ); + self.bytes -= bytes; + Self { bytes } + } + + /// Take over the bytes covered by `other`. + pub fn merge(&mut self, other: Self) { + self.bytes += other.bytes; + } + + pub(super) fn into_bytes(self) -> u64 { + self.bytes + } +} + enum ChunksState { Readable(Box), Reset(VarInt), diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 0c47d3df7f2..0b792695d9a 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -9,13 +9,13 @@ use rustc_hash::FxHashMap; use tracing::{debug, trace}; use super::{ - PendingStreamsQueue, Recv, Retransmits, Send, SendState, ShouldTransmit, StreamEvent, + Credit, PendingStreamsQueue, Recv, Retransmits, Send, SendState, ShouldTransmit, StreamEvent, StreamHalf, ThinRetransmits, }; use crate::{ Dir, MAX_STREAM_COUNT, Side, StreamId, TransportError, VarInt, coding::BufMutExt, - connection::stats::FrameStats, + connection::stats::{CreditStats, FrameStats}, frame::{self, FrameStruct, StreamMetaVec}, transport_parameters::TransportParameters, }; @@ -121,6 +121,9 @@ pub struct StreamsState { pub(super) data_sent: u64, /// Sum of end offsets of all receive streams. Includes gaps, so it's an upper bound. data_recvd: u64, + /// Bytes read with [`Chunks::hold`](super::Chunks::hold) whose connection credit the + /// application has not released yet + pub(super) held: u64, /// Total quantity of outgoing data retained in send buffers, including acknowledged data pub(super) buffered_data: u64, /// Configured upper bound for `buffered_data`. @@ -180,6 +183,7 @@ impl StreamsState { sent_max_data: receive_window, data_sent: 0, data_recvd: 0, + held: 0, buffered_data: 0, send_window, stream_receive_window: stream_receive_window.into(), @@ -282,7 +286,9 @@ impl StreamsState { if !rs.stopped { self.events.push_back(StreamEvent::Readable { id }); - return Ok(ShouldTransmit(false)); + // Credit released while the peer still had room may not have been worth a frame + // then; it is now that the peer is about to block. + return Ok(ShouldTransmit(self.max_data_due())); } // Stopped streams become closed instantly on FIN, so check whether we need to clean up @@ -991,17 +997,42 @@ impl StreamsState { self.receive_window_shrink_debt -= credits; } + ShouldTransmit(self.max_data_due()) + } + + /// Return the connection credit held by `credit` to the peer. + pub(crate) fn release(&mut self, credit: Credit) -> ShouldTransmit { + let bytes = credit.into_bytes(); + assert!( + bytes <= self.held, + "released {bytes} bytes of credit with only {} held", + self.held + ); + self.held -= bytes; + self.add_read_credits(bytes) + } + + /// Whether the window has grown enough since the last `MAX_DATA` to be worth announcing. + /// + /// Updates are batched to a fraction of the receive window, which accommodates bigger windows + /// with fewer frames. Once the peer is within that fraction of its limit any increase is + /// worth a frame, because the credit an application holds may never add up to the fraction. + fn max_data_due(&self) -> bool { if self.local_max_data > VarInt::MAX.into_inner() { - return ShouldTransmit(false); + return false; } - // Only announce a window update if it's significant enough - // to make it worthwhile sending a MAX_DATA frame. - // We use a fraction of the configured connection receive window to make - // the decision, to accommodate for connection using bigger windows requiring - // less updates. - let diff = self.local_max_data - self.sent_max_data.into_inner(); - ShouldTransmit(diff >= (self.receive_window / 8)) + let sent = self.sent_max_data.into_inner(); + let threshold = self.receive_window / 8; + let diff = self.local_max_data - sent; + diff >= threshold || (diff > 0 && sent.saturating_sub(self.data_recvd) < threshold) + } + + pub(crate) fn credit_stats(&self) -> CreditStats { + CreditStats { + held: self.held, + available: self.local_max_data.saturating_sub(self.data_recvd), + } } /// Update counters for removal of a stream @@ -2528,4 +2559,217 @@ mod tests { server.next_remote[Dir::Bi as usize] ); } + + /// How a held stream ends in [`held_credit_returns_once`]. + #[derive(Debug, Clone, Copy)] + enum End { + Fin, + Stop, + Reset, + StopThenReset, + } + + fn data(client: &mut StreamsState, id: StreamId, offset: u64, len: usize, fin: bool) { + let _ = client + .received( + frame::Stream { + id, + offset, + fin, + data: Bytes::from(vec![0; len]), + }, + len, + ) + .unwrap(); + } + + fn reset(client: &mut StreamsState, id: StreamId, final_offset: u32) { + let _ = client + .received_reset(frame::ResetStream { + id, + error_code: 0u32.into(), + final_offset: final_offset.into(), + }) + .unwrap(); + } + + /// Read everything readable now, holding its connection credit. + fn read_held(client: &mut StreamsState, id: StreamId) -> Credit { + let mut pending = Retransmits::default(); + let mut recv = RecvStream { + id, + state: client, + pending: &mut pending, + }; + let mut chunks = recv.read(true).unwrap(); + while let Ok(Some(_)) = chunks.next(usize::MAX) {} + let credit = chunks.hold(); + let _ = chunks.finalize(); + credit + } + + /// Every byte of a stream returns its connection credit exactly once, whichever terminal path + /// it takes and whether the held bytes are released before or after its state is reclaimed. + #[test] + fn held_credit_returns_once() { + for end in [End::Fin, End::Stop, End::Reset, End::StopThenReset] { + let mut client = make(Side::Client); + let initial = client.local_max_data; + let id = StreamId::new(Side::Server, Dir::Uni, 0); + + // A readable prefix, then a gap the transport must account for itself. + data(&mut client, id, 0, 32, false); + data(&mut client, id, 64, 16, false); + let credit = read_held(&mut client, id); + assert_eq!(credit.len(), 32, "{end:?}"); + assert_eq!(client.credit_stats().held, 32, "{end:?}"); + assert_eq!( + client.local_max_data, initial, + "{end:?}: held, not returned" + ); + + let size = match end { + End::Fin => { + data(&mut client, id, 32, 32, false); + data(&mut client, id, 80, 16, true); + let rest = read_held(&mut client, id); + assert_eq!(rest.len(), 64, "{end:?}"); + let _ = client.release(rest); + 96 + } + End::Stop => { + let mut pending = Retransmits::default(); + RecvStream { + id, + state: &mut client, + pending: &mut pending, + } + .stop(0u32.into()) + .unwrap(); + // Late data below and beyond the stop is credited on arrival. + data(&mut client, id, 32, 32, false); + data(&mut client, id, 80, 16, true); + 96 + } + End::Reset => { + reset(&mut client, id, 128); + let mut pending = Retransmits::default(); + let mut recv = RecvStream { + id, + state: &mut client, + pending: &mut pending, + }; + let mut chunks = recv.read(true).unwrap(); + assert!(chunks.next(usize::MAX).is_err(), "{end:?}"); + let _ = chunks.finalize(); + 128 + } + End::StopThenReset => { + let mut pending = Retransmits::default(); + RecvStream { + id, + state: &mut client, + pending: &mut pending, + } + .stop(0u32.into()) + .unwrap(); + reset(&mut client, id, 128); + // A duplicate reset after reclamation changes nothing. + reset(&mut client, id, 128); + 128 + } + }; + assert!( + !client.recv.contains_key(&id), + "{end:?}: state reclaimed while credit is held" + ); + assert_eq!( + client.local_max_data, + initial + size - 32, + "{end:?}: everything but the held prefix" + ); + + let _ = client.release(credit); + assert_eq!(client.local_max_data, initial + size, "{end:?}"); + assert_eq!(client.credit_stats().held, 0, "{end:?}"); + } + } + + #[test] + fn held_credit_pays_shrink_debt() { + let mut client = make(Side::Client); + let initial = client.local_max_data; + let id = StreamId::new(Side::Server, Dir::Uni, 0); + data(&mut client, id, 0, 32, false); + let credit = read_held(&mut client, id); + client.set_receive_window((client.receive_window - 16).try_into().unwrap()); + let _ = client.release(credit); + assert_eq!(client.local_max_data, initial + 16); + } + + #[test] + #[should_panic(expected = "only 0 held")] + fn release_from_another_connection() { + let mut client = make(Side::Client); + let id = StreamId::new(Side::Server, Dir::Uni, 0); + data(&mut client, id, 0, 32, false); + let credit = read_held(&mut client, id); + let _ = make(Side::Client).release(credit); + } + + /// Ingest and read rounds across many streams, with and without holding and releasing each + /// read's connection credit. The held cost must stay flat as the stream count grows. + #[test] + #[ignore = "release benchmark, run by nightly"] + fn benchmark_receive_credit() { + const BYTES: usize = 1 << 30; + println!("streams,payload,plain_ns_per_read,held_ns_per_read"); + for streams in [1u64, 64, 4096] { + for payload in [1200usize, 64 * 1024] { + let rounds = (BYTES / payload / streams as usize).max(1) as u64; + let chunk = Bytes::from(vec![0; payload]); + let mut ns = [0f64; 2]; + for (mode, held) in [false, true].into_iter().enumerate() { + let mut client = StreamsState::new( + Side::Client, + VarInt::from_u64(streams).unwrap(), + 0u32.into(), + 0, + VarInt::from_u64(1 << 50).unwrap(), + VarInt::from_u64(1 << 40).unwrap(), + ); + let mut pending = Retransmits::default(); + let start = std::time::Instant::now(); + for round in 0..rounds { + for index in 0..streams { + let id = StreamId::new(Side::Server, Dir::Uni, index); + let frame = frame::Stream { + id, + offset: round * payload as u64, + fin: false, + data: chunk.clone(), + }; + let _ = client.received(frame, payload).unwrap(); + let mut recv = RecvStream { + id, + state: &mut client, + pending: &mut pending, + }; + let mut chunks = recv.read(true).unwrap(); + std::hint::black_box(chunks.next(usize::MAX).unwrap()); + if held { + let credit = chunks.hold(); + let _ = chunks.finalize(); + let _ = client.release(credit); + } else { + let _ = chunks.finalize(); + } + } + } + ns[mode] = start.elapsed().as_nanos() as f64 / (rounds * streams) as f64; + } + println!("{streams},{payload},{:.1},{:.1}", ns[0], ns[1]); + } + } + } } diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index a5c39cb9576..78a1d34c52e 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -44,10 +44,10 @@ pub use bloom_token_log::BloomTokenLog; mod connection; pub use crate::connection::{ - Chunk, Chunks, ClosedStream, Connection, ConnectionError, ConnectionStats, Datagrams, Event, - FinishError, FrameStats, PathStats, ReadError, ReadableError, RecvStream, RttEstimator, - SendDatagramError, SendStream, ShouldTransmit, StreamEvent, Streams, UdpStats, WriteError, - Written, + Chunk, Chunks, ClosedStream, Connection, ConnectionError, ConnectionStats, Credit, CreditStats, + Datagrams, Event, FinishError, FrameStats, PathStats, ReadError, ReadableError, RecvStream, + RttEstimator, SendDatagramError, SendStream, ShouldTransmit, StreamEvent, Streams, UdpStats, + WriteError, Written, }; #[cfg(feature = "qlog")] pub use connection::qlog::QlogStream; diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 60a8e74199a..30073fb152c 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -6032,6 +6032,123 @@ fn partial_poll_reports_starvation() { ); } +/// Write what the client can and read what the server can, holding its credit, until neither +/// makes progress. Returns the bytes read. +fn pump_held( + pair: &mut Pair, + (client_ch, server_ch): (ConnectionHandle, ConnectionHandle), + s: StreamId, + msg: &[u8], + sent: &mut usize, + held: &mut Credit, +) -> usize { + let mut total = 0; + loop { + // Deliver whatever credit the server queued since the last round. + pair.drive(); + let mut wrote = 0; + if *sent < msg.len() + && let Ok(n) = pair.client_send(client_ch, s).write(&msg[*sent..]) + { + wrote = n; + *sent += n; + if *sent == msg.len() { + pair.client_send(client_ch, s).finish().unwrap(); + } + } + pair.drive(); + + let mut read = 0; + let mut recv = pair.server_recv(server_ch, s); + if let Ok(mut chunks) = recv.read(true) { + while let Ok(Some(chunk)) = chunks.next(usize::MAX) { + read += chunk.bytes.len(); + } + held.merge(chunks.hold()); + let _ = chunks.finalize(); + } + if read == 0 && wrote == 0 { + return total; + } + total += read; + } +} + +/// Held reads keep replenishing a stream window much smaller than the connection's, while the +/// connection window stays spent until the bytes are released. A release far below the usual +/// `MAX_DATA` threshold still unblocks the peer, without relying on its DATA_BLOCKED. +#[test] +fn held_credit_spends_connection_window_only() { + const WINDOW: usize = 4000; + let _guard = subscribe(); + let mut transport = cubic_transport(); + transport.receive_window((WINDOW as u32).into()); + transport.stream_receive_window(1000u32.into()); + let mut pair = Pair::new( + Default::default(), + ServerConfig { + transport: Arc::new(transport), + ..server_config() + }, + ); + let chs = pair.connect(); + let (client_ch, server_ch) = chs; + let s = pair.client_streams(client_ch).open(Dir::Uni).unwrap(); + let msg = vec![0xAB; 3 * WINDOW]; + let (mut sent, mut held) = (0, Credit::default()); + + assert_eq!( + pump_held(&mut pair, chs, s, &msg, &mut sent, &mut held), + WINDOW + ); + assert_eq!(held.len(), WINDOW as u64); + let stats = pair.server_conn_mut(server_ch).stats().credit; + assert_eq!((stats.held, stats.available), (WINDOW as u64, 0)); + + let small = held.split(100); + let _ = pair.server_conn_mut(server_ch).release(small); + assert_eq!( + pump_held(&mut pair, chs, s, &msg, &mut sent, &mut held), + 100 + ); + + // Credit too small to announce while the peer still has room goes out once the peer spends + // that room, even though the receiver reads nothing in between. + let big = held.split(600); + let _ = pair.server_conn_mut(server_ch).release(big); + pair.drive_server(); + let small = held.split(100); + let _ = pair.server_conn_mut(server_ch).release(small); + let before = sent; + for _ in 0..4 { + if let Ok(n) = pair.client_send(client_ch, s).write(&msg[sent..]) { + sent += n; + } + pair.drive(); + } + assert_eq!(sent - before, 700); + assert_eq!( + pump_held(&mut pair, chs, s, &msg, &mut sent, &mut held), + 700 + ); + + let mut total = WINDOW + 100 + 700; + while total < msg.len() { + let all = mem::take(&mut held); + let _ = pair.server_conn_mut(server_ch).release(all); + let read = pump_held(&mut pair, chs, s, &msg, &mut sent, &mut held); + assert!(read > 0, "a release unblocks the peer"); + total += read; + } + assert_eq!( + pair.server_recv(server_ch, s).read(true).err(), + Some(ReadableError::ClosedStream), + "finished and reclaimed while its tail is still held" + ); + let _ = pair.server_conn_mut(server_ch).release(held); + assert_eq!(pair.server_conn_mut(server_ch).stats().credit.held, 0); +} + #[test] fn sender_window_is_not_starvation() { let _guard = subscribe(); diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 027349ae12d..4d3b82f4512 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -6,7 +6,7 @@ use std::{ }; use bytes::Bytes; -use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, Credit, ReadableError, StreamId}; use thiserror::Error; use tokio::io::ReadBuf; @@ -55,6 +55,8 @@ pub struct RecvStream { is_0rtt: bool, all_data_read: bool, reset: Option, + /// Connection credit for bytes read since [`Self::hold`], not yet released. + held: Option, } impl RecvStream { @@ -65,6 +67,32 @@ impl RecvStream { is_0rtt, all_data_read: false, reset: None, + held: None, + } + } + + /// Keep the connection flow control credit for every byte read from now on. + /// + /// Stream credit still follows reads, so a stream window smaller than an application message + /// keeps the message flowing. The connection window those bytes occupy stays spent until + /// [`Self::release`] returns it, or this stream is dropped. + pub fn hold(&mut self) { + self.held.get_or_insert_default(); + } + + /// Return the connection credit for `bytes` of the held bytes to the peer. + /// + /// # Panics + /// If fewer than `bytes` read bytes are held. + pub fn release(&mut self, bytes: u64) { + let credit = self + .held + .as_mut() + .expect("release requires hold") + .split(bytes); + let mut conn = self.conn.state.lock("RecvStream::release"); + if conn.inner.release(credit).should_transmit() { + conn.wake(); } } @@ -380,6 +408,9 @@ impl RecvStream { let mut recv = conn.inner.recv_stream(self.stream); let mut chunks = recv.read(ordered)?; let status = read_fn(&mut chunks); + if let Some(held) = &mut self.held { + held.merge(chunks.hold()); + } if chunks.finalize().should_transmit() { conn.wake(); } @@ -528,6 +559,12 @@ impl tokio::io::AsyncRead for RecvStream { impl Drop for RecvStream { fn drop(&mut self) { + if let Some(held) = self.held.take().filter(|held| !held.is_empty()) { + let mut conn = self.conn.state.lock("RecvStream::drop"); + if conn.inner.release(held).should_transmit() { + conn.wake(); + } + } if self.all_data_read { debug_assert!( !self diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0b284f17706..427bd98943f 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1743,6 +1743,50 @@ async fn dropped_connection_cleans_up() { endpoint.wait_idle().await; } +/// Held reads keep the connection window spent until released, while a stream window far +/// smaller than it keeps replenishing; dropping the stream releases what is still held. +#[tokio::test] +async fn held_reads_spend_the_connection_window() { + const WINDOW: usize = 32 * 1024; + let _guard = subscribe(); + let mut transport = TransportConfig::default(); + transport.receive_window((WINDOW as u32).into()); + transport.stream_receive_window(4096u32.into()); + let endpoint = endpoint_with_config(transport); + + let (client, server) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let mut send = client.open_uni().await.unwrap(); + let writer = tokio::spawn(async move { + // The receiver walks away before this finishes. + let _ = send.write_all(&[7; 4 * WINDOW]).await; + }); + + let mut recv = server.accept_uni().await.unwrap(); + recv.hold(); + let mut buf = vec![0; WINDOW]; + recv.read_exact(&mut buf).await.unwrap(); + let credit = server.stats().credit; + assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); + + // Far below the usual update threshold, and still enough to unblock. + recv.release(100); + recv.read_exact(&mut buf[..100]).await.unwrap(); + assert_eq!(server.stats().credit.held, WINDOW as u64); + + recv.release(WINDOW as u64); + recv.read_exact(&mut buf[..500]).await.unwrap(); + assert_eq!(server.stats().credit.held, 500); + drop(recv); + assert_eq!(server.stats().credit.held, 0, "dropping releases the rest"); + writer.await.unwrap(); +} + #[derive(Default)] struct WakeCounter { wakes: AtomicUsize, diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index fb70a5a1126..21f5c9f45b8 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -337,6 +337,12 @@ impl Connection { pub fn handshake(&self) -> &crate::quic::handshake::Info { &self.handshake } + + /// How much receive credit the application holds, and how much the peer + /// may still send. + pub fn credit(&self) -> moq_quic::CreditStats { + self.shared.conn.borrow().stats().credit + } } impl Clone for Connection { diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index e415f57f308..66af5782838 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -8,7 +8,7 @@ use std::task::{Context, Poll}; use bytes::{Buf, Bytes, BytesMut}; -use moq_quic::{StreamId, VarInt}; +use moq_quic::{Credit, StreamId, VarInt}; use super::super::Error; use super::{End, Shared}; @@ -223,6 +223,12 @@ pub struct RecvStream { stopped: bool, /// Bytes `poll_closed` read ahead, handed to `poll_read` before moq-quic's. backlog: BytesMut, + /// Connection credit for bytes read since [`Self::hold`], not yet released. + /// Includes the held part of `backlog`, which the application has not seen. + held: Option, + /// The front of `backlog` read before [`Self::hold`], whose credit is + /// already returned. + unheld: usize, } impl RecvStream { @@ -234,6 +240,49 @@ impl RecvStream { finished: false, stopped: false, backlog: BytesMut::new(), + held: None, + unheld: 0, + } + } + + /// Keep the connection flow control credit for every byte read from now + /// on. + /// + /// Stream credit still follows reads, so a stream window smaller than an + /// application message keeps the message flowing. The connection window + /// those bytes occupy stays spent until [`Self::release`] returns it, or + /// this stream is dropped. + pub fn hold(&mut self) { + if self.held.is_none() { + self.held = Some(Credit::default()); + self.unheld = self.backlog.len(); + } + } + + /// Return the connection credit for `bytes` of the held bytes this stream + /// has handed to the application. + /// + /// # Panics + /// If fewer than `bytes` read bytes are held. + pub fn release(&mut self, bytes: u64) { + let backlog = (self.backlog.len() - self.unheld) as u64; + let held = self.held.as_mut().expect("release requires hold"); + assert!( + bytes + backlog <= held.len(), + "released {bytes} bytes with {} handed out", + held.len() - backlog + ); + let credit = held.split(bytes); + self.give_back(credit); + } + + /// Hand `credit` to moq-quic, kicking the driver if the peer should hear. + fn give_back(&self, credit: Credit) { + if credit.is_empty() { + return; + } + if self.shared.conn.borrow_mut().release(credit).should_transmit() { + self.shared.kick(); } } @@ -246,7 +295,13 @@ impl RecvStream { /// [`stop`](web_transport_trait::poll::RecvStream::stop) with a /// full-width code, for the WebTransport HTTP/3 error mapping. pub(crate) fn stop_code(&mut self, code: u64) { + // The application never sees the read-ahead, so it can never release it. + if let Some(held) = &mut self.held { + let credit = held.split((self.backlog.len() - self.unheld) as u64); + self.give_back(credit); + } self.backlog.clear(); + self.unheld = 0; if self.stopped || self.finished { return; } @@ -268,7 +323,7 @@ impl RecvStream { /// /// Takes the shared state rather than `&mut self` so a caller can hold a /// waiter from `self.park` across the read. - fn read(shared: &Shared, id: StreamId, max: usize) -> Read { + fn read(shared: &Shared, id: StreamId, max: usize, held: Option<&mut Credit>) -> Read { let mut conn = shared.conn.borrow_mut(); let mut recv = conn.recv_stream(id); let mut chunks = match recv.read(true) { @@ -282,6 +337,9 @@ impl RecvStream { Err(moq_quic::ReadError::Blocked) => Read::Blocked, Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), }; + if let Some(held) = held { + held.merge(chunks.hold()); + } let transmit = chunks.finalize().should_transmit(); drop(conn); if transmit { @@ -298,6 +356,7 @@ impl RecvStream { let n = dst.len().min(self.backlog.len()); dst[..n].copy_from_slice(&self.backlog[..n]); self.backlog.advance(n); + self.unheld = self.unheld.saturating_sub(n); if n > 0 { self.shared.wake_readable(self.id); } @@ -319,7 +378,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { if self.finished { return Poll::Ready(Ok(None)); } - match Self::read(&self.shared, self.id, dst.len()) { + match Self::read(&self.shared, self.id, dst.len(), self.held.as_mut()) { Read::Chunk(bytes) => { let n = bytes.len().min(dst.len()); dst[..n].copy_from_slice(&bytes[..n]); @@ -367,7 +426,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { self.shared.park_readable(self.id, waiter); return Poll::Pending; } - match Self::read(&self.shared, self.id, READ_CHUNK) { + match Self::read(&self.shared, self.id, READ_CHUNK, self.held.as_mut()) { Read::Chunk(bytes) => self.backlog.extend_from_slice(&bytes), Read::Finished => { self.finished = true; @@ -388,6 +447,9 @@ impl web_transport_trait::poll::RecvStream for RecvStream { impl Drop for RecvStream { fn drop(&mut self) { + if let Some(held) = self.held.take() { + self.give_back(held); + } self.shared.forget_recv(self.id); if !self.finished && !self.stopped { let _ = self diff --git a/rs/moq-uring/tests/credit.rs b/rs/moq-uring/tests/credit.rs new file mode 100644 index 00000000000..d2a7cbb63d5 --- /dev/null +++ b/rs/moq-uring/tests/credit.rs @@ -0,0 +1,163 @@ +//! Held receive credit through the io_uring driver: reads keep the connection +//! window spent until the application releases them, including bytes +//! `poll_closed` reads ahead. +//! +//! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. + +#![cfg(all(target_os = "linux", feature = "quic"))] + +#[path = "support.rs"] +mod support; + +use std::net::UdpSocket; +use std::task::Poll; +use std::time::Duration; + +use moq_uring::{Config, Error, Worker, quic, udp}; +use web_transport_trait::poll::{RecvStream as _, SendStream as _, Session as _}; + +const ALPN: &str = "moq-uring-credit"; +/// The server's connection window; under `poll_closed`'s read-ahead cap so +/// one window fits in its backlog. +const WINDOW: usize = 32 * 1024; + +fn worker() -> Option { + match Worker::new(Config::default()) { + Ok(worker) => Some(worker), + Err(Error::Unsupported(reason)) if std::env::var_os("MOQ_STRICT").is_none() => { + eprintln!("skipping io_uring credit test: {reason}"); + None + } + Err(err) => panic!("worker setup failed: {err}"), + } +} + +/// Await `future`, failing loudly rather than hanging if it never resolves. +async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future) -> T { + let mut deadline = moq_uring::Timer::after(handle, Duration::from_secs(5)); + let mut future = std::pin::pin!(future); + kio::wait(|waiter| { + let mut cx = waiter.context(); + if let Poll::Ready(value) = future.as_mut().poll(&mut cx) { + return Poll::Ready(Some(value)); + } + deadline.poll(waiter).map(|()| None) + }) + .await + .unwrap_or_else(|| panic!("timed out waiting for {what}")) +} + +/// Read exactly `len` bytes. +async fn read(handle: &moq_uring::Handle, recv: &mut quic::RecvStream, len: usize) { + let mut buf = vec![0u8; len]; + let mut filled = 0; + while filled < len { + let n = within( + handle, + "stream data", + std::future::poll_fn(|cx| recv.poll_read(cx, &mut buf[filled..])), + ) + .await + .expect("read") + .expect("not finished"); + filled += n; + } +} + +#[test] +fn held_reads_spend_the_connection_window() { + let Some(mut worker) = worker() else { return }; + let handle = worker.handle(); + let certs = support::certs().expect("certificates"); + + let socket = |handle: &moq_uring::Handle| { + handle + .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) + .expect("socket") + }; + let mut server_config = quic::server::Config::new( + quic::Identity::open(&certs.cert, &certs.key) + .expect("identity") + .server(&[ALPN], quic::server::ClientAuth::None) + .expect("tls"), + ); + server_config.transport.receive_window = Some(WINDOW as u64); + // Far smaller than the window: held reads must keep replenishing it. + server_config.transport.stream_receive_window = Some(4096); + let server = quic::Endpoint::new( + socket(&handle), + quic::endpoint::Config::default().with_server(server_config), + ) + .expect("server endpoint"); + let client = quic::Endpoint::new(socket(&handle), quic::endpoint::Config::default()).expect("dial-only endpoint"); + let mut dial = quic::client::Config::new(server.local_addr(), "localhost"); + dial.alpn = vec![ALPN.to_string()]; + dial.verify = false; + + worker + .block_on(async move { + let mut dialed = client.connect(&dial).await.expect("dial"); + let mut accepted = server.accept().await.expect("accept"); + + let mut send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) + .await + .expect("open_uni"); + handle.spawn(async move { + let payload = vec![7u8; 4 * WINDOW]; + let mut buf = &payload[..]; + while !buf.is_empty() { + match std::future::poll_fn(|cx| send.poll_write(cx, buf)).await { + Ok(n) => buf = &buf[n..], + // The receiver stops once it has seen enough. + Err(_) => return, + } + } + let _ = send.finish(); + let _ = std::future::poll_fn(|cx| send.poll_closed(cx)).await; + }); + + let mut recv = within( + &handle, + "the stream", + std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)), + ) + .await + .expect("accept_uni"); + recv.hold(); + + read(&handle, &mut recv, WINDOW).await; + let credit = accepted.credit(); + assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); + + // Far below the usual update threshold, and still enough to unblock. + recv.release(100); + read(&handle, &mut recv, 100).await; + assert_eq!(accepted.credit().held, WINDOW as u64); + + // Read-ahead holds credit the application has not seen yet. + recv.release(WINDOW as u64); + within( + &handle, + "a window of read-ahead", + std::future::poll_fn(|cx| match recv.poll_closed(cx) { + Poll::Ready(res) => panic!("stream ended early: {res:?}"), + Poll::Pending if accepted.credit().held == WINDOW as u64 => Poll::Ready(()), + Poll::Pending => Poll::Pending, + }), + ) + .await; + + // Only what reached the application is releasable. + read(&handle, &mut recv, 1000).await; + recv.release(1000); + assert_eq!(accepted.credit().held, WINDOW as u64 - 1000); + + // Stopping returns the read-ahead; the application still owns what it read. + read(&handle, &mut recv, 500).await; + recv.stop(0); + assert_eq!(accepted.credit().held, 500); + drop(recv); + assert_eq!(accepted.credit().held, 0, "dropping releases the rest"); + }) + .expect("worker"); +} diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 8974deed54d..12e96d26f25 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -22,6 +22,7 @@ case "${1:-}" in run just rs bench-playback run just rs bench-feed-fetch run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate + run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test # Each kio routine once, including the contended bench's worker threads. From e111abf07ff42f489961501f0642cfd510642d5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:50:23 -0700 Subject: [PATCH 10/51] test(moq-tokio): gate the timed helper with its only caller Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/client.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 66166b129bc..70c9e0a9059 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -787,6 +787,7 @@ mod tests { use super::*; /// A millisecond track: the frames written here carry timestamps. + #[cfg(all(feature = "websocket", feature = "quic"))] fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } From 188b08c30dd4d3bae1ab33dfb8f71ba8a60c572b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 22:46:44 -0700 Subject: [PATCH 11/51] fix(tokio): mark quic::web and quic::driver error enums non_exhaustive Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1 + rs/moq-tokio/src/quic/driver/recv_stream.rs | 4 ++++ rs/moq-tokio/src/quic/driver/send_stream.rs | 2 ++ rs/moq-tokio/src/quic/web/connect.rs | 1 + rs/moq-tokio/src/quic/web/error.rs | 8 ++++++++ rs/moq-tokio/src/quic/web/settings.rs | 1 + 6 files changed, 17 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 605163a6736..e86cecef417 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -1422,6 +1422,7 @@ fn wake_all_notify(wakers: &mut FxHashMap>) { /// Errors that can arise when sending a datagram #[derive(Debug, Error, Clone, Eq, PartialEq)] +#[non_exhaustive] pub enum SendDatagramError { /// The peer does not support receiving datagram frames #[error("datagrams not supported by peer")] diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 027349ae12d..2bf91f62310 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -506,6 +506,7 @@ impl ReadToEndBuffer { /// Errors from [`RecvStream::read_to_end`] #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadToEndError { /// An error occurred during reading #[error("read error: {0}")] @@ -558,6 +559,7 @@ impl Drop for RecvStream { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadError { /// The peer abandoned transmitting data on this stream /// @@ -618,6 +620,7 @@ impl From for io::Error { /// Errors that arise while waiting for a stream to be reset #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ResetError { /// The connection was lost #[error("connection lost")] @@ -691,6 +694,7 @@ impl Future for ReadExact<'_> { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadExactError { /// The stream finished before all bytes were read #[error("stream finished early ({0} bytes read)")] diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index e51af7cf259..812f2d4c12c 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -358,6 +358,7 @@ impl Drop for SendStream { /// Errors that arise from writing to a stream #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum WriteError { /// The peer is no longer accepting data on this stream /// @@ -409,6 +410,7 @@ impl From for io::Error { /// Errors that arise while monitoring for a send stream stop from the peer #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum StoppedError { /// The connection was lost #[error("connection lost")] diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 471f5e416c7..d49228d5f44 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -5,6 +5,7 @@ use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ConnectError { #[error("quic stream was closed early")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 5697bf684ec..57d9eb1916b 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -6,6 +6,7 @@ use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ClientError { #[error("unexpected end of stream")] UnexpectedEnd, @@ -39,6 +40,7 @@ pub enum ClientError { /// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum SessionError { #[error("connection error: {0}")] ConnectionError(crate::quic::driver::ConnectionError), @@ -140,6 +142,7 @@ impl CloseReason { /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WebTransportError { #[error("closed: code={0} reason={1}")] Closed(u32, String), @@ -156,6 +159,7 @@ pub enum WebTransportError { /// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WriteError { #[error("STOP_SENDING: {0}")] Stopped(u32), @@ -188,6 +192,7 @@ impl From for WriteError { /// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadError { #[error("session error: {0}")] SessionError(#[from] SessionError), @@ -223,6 +228,7 @@ impl From for ReadError { /// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadExactError { #[error("finished early")] FinishedEarly(usize), @@ -242,6 +248,7 @@ impl From for ReadExactError { /// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadToEndError { #[error("too long")] TooLong, @@ -272,6 +279,7 @@ impl From for ClosedStream { /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ServerError { #[error("unexpected end of stream")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index fdee95f23c7..034c379ce74 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -3,6 +3,7 @@ use futures::try_join; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum SettingsError { #[error("quic stream was closed early")] UnexpectedEnd, From d35e2cd2dea328a3b77883b72a919b103fdb0ad2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:01:47 -0700 Subject: [PATCH 12/51] feat(quic): hand native receivers owned Credit handles Replace RecvStream::hold/release(bytes) in both drivers with a copying poll_read_held that yields a driver Credit for exactly the bytes read. Credit splits and merges, and dropping it returns the connection credit, so over-release and cross-connection release cannot be written. io_uring's poll_closed read-ahead now keeps its credit until the application reads it: a plain read returns it, a held read takes it. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/receive-memory.md | 15 +- rs/moq-tokio/src/quic/driver/connection.rs | 2 +- rs/moq-tokio/src/quic/driver/mod.rs | 2 +- rs/moq-tokio/src/quic/driver/recv_stream.rs | 193 ++++++++++++----- rs/moq-tokio/src/quic/driver/tests.rs | 35 ++- rs/moq-uring/src/quic/driver/mod.rs | 2 +- rs/moq-uring/src/quic/driver/stream.rs | 223 ++++++++++++-------- rs/moq-uring/src/quic/mod.rs | 2 +- rs/moq-uring/tests/credit.rs | 51 +++-- 9 files changed, 345 insertions(+), 180 deletions(-) diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md index 5e217f71ffc..5371a7525a3 100644 --- a/quest/m1/quic/receive-memory.md +++ b/quest/m1/quic/receive-memory.md @@ -18,12 +18,15 @@ peer-visible QUIC flow control instead of a separate header quota error. The native transport primitive is a separate prerequisite; do not activate whole-message retention without its recovery policy in this quest. -That primitive landed as a provisional seam to finalize here: `moq-quic` -reads can hold their connection credit until released, both native drivers' -receive streams opt in per stream and release by byte count (dropping -releases the rest), and connection stats report held and still-available -credit. moq-net's transport trait and the WebTransport wrappers do not carry -it yet; reshape any of it to fit the consumer. +That primitive landed as a provisional seam to finalize here: both native +drivers' receive streams have a copying `poll_read_held` that yields a +`Credit` for exactly the bytes read. Dropping a `Credit` returns it, so a +reader buffer keeps one merged `Credit` and drops `split(n)` for each +complete unit; read-ahead stays in the remainder. io_uring's `poll_closed` +read-ahead keeps its credit until the application reads it. Connection stats +report held and still-available credit. moq-net's transport trait and the +WebTransport wrappers do not carry it yet; reshape any of it to fit the +consumer. Hold connection credit through reader buffers, type/length prefixes, headers, extensions, and frame assembly. Attribute read-ahead bytes to the next unit diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 605163a6736..c25fe3540cc 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -1015,7 +1015,7 @@ impl RemoteAddressWatcher { } #[derive(Debug)] -pub(crate) struct ConnectionRef(Arc); +pub(crate) struct ConnectionRef(pub(crate) Arc); impl ConnectionRef { fn stable_id(&self) -> usize { diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index d758cc9c7c6..2a927ce4c8e 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -73,7 +73,7 @@ pub use crate::quic::driver::connection::{ pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; pub use crate::quic::driver::recv_stream::{ - ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, + Credit, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, }; #[cfg(test)] pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 4d3b82f4512..75c0d1edcb8 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -1,16 +1,21 @@ use std::{ + fmt, future::{Future, poll_fn}, - io, + io, mem, pin::Pin, + sync::Arc, task::{Context, Poll, ready}, }; use bytes::Bytes; -use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, Credit, ReadableError, StreamId}; +use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; use thiserror::Error; use tokio::io::ReadBuf; -use crate::quic::driver::{VarInt, connection::ConnectionRef}; +use crate::quic::driver::{ + VarInt, + connection::{ConnectionInner, ConnectionRef}, +}; /// A stream that can only be used to receive data /// @@ -55,8 +60,6 @@ pub struct RecvStream { is_0rtt: bool, all_data_read: bool, reset: Option, - /// Connection credit for bytes read since [`Self::hold`], not yet released. - held: Option, } impl RecvStream { @@ -67,32 +70,6 @@ impl RecvStream { is_0rtt, all_data_read: false, reset: None, - held: None, - } - } - - /// Keep the connection flow control credit for every byte read from now on. - /// - /// Stream credit still follows reads, so a stream window smaller than an application message - /// keeps the message flowing. The connection window those bytes occupy stays spent until - /// [`Self::release`] returns it, or this stream is dropped. - pub fn hold(&mut self) { - self.held.get_or_insert_default(); - } - - /// Return the connection credit for `bytes` of the held bytes to the peer. - /// - /// # Panics - /// If fewer than `bytes` read bytes are held. - pub fn release(&mut self, bytes: u64) { - let credit = self - .held - .as_mut() - .expect("release requires hold") - .split(bytes); - let mut conn = self.conn.state.lock("RecvStream::release"); - if conn.inner.release(credit).should_transmit() { - conn.wake(); } } @@ -122,6 +99,50 @@ impl RecvStream { .await } + /// Read data contiguously from the stream, keeping its connection flow control credit. + /// + /// Like [`read()`], but yields the [`Credit`] for exactly the bytes read into `buf`, or `None` + /// if the stream was finished. Stream credit still advances, so a stream window smaller than + /// an application message keeps the message flowing, while the connection window these bytes + /// occupy stays spent until the credit is dropped. + /// + /// This operation is cancel-safe. + /// + /// [`read()`]: RecvStream::read + pub async fn read_held(&mut self, buf: &mut [u8]) -> Result, ReadError> { + poll_fn(|cx| self.poll_read_held(cx, buf)).await + } + + /// Attempts to read from the stream into the provided buffer, keeping the bytes' connection + /// flow control credit. + /// + /// See [`read_held()`] for details. + /// + /// [`read_held()`]: RecvStream::read_held + pub fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll, ReadError>> { + let mut buf = ReadBuf::new(buf); + let mut held = moq_quic::Credit::default(); + let read = match buf.remaining() { + 0 => Poll::Ready(Ok(Some(()))), + _ => self.poll_read_generic(cx, true, |chunks| { + let status = fill(chunks, &mut buf); + held.merge(chunks.hold()); + status + }), + }; + // Bytes are read exactly when the read yields `Some`. + read.map_ok(|read| { + read.map(|()| Credit { + conn: self.conn.0.clone(), + inner: mem::take(&mut held), + }) + }) + } + /// Attempts to read from the stream into the provided buffer /// /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this @@ -164,24 +185,8 @@ impl RecvStream { return Poll::Ready(Ok(())); } - self.poll_read_generic(cx, true, |chunks| { - let mut read = false; - loop { - if buf.remaining() == 0 { - // We know `read` is `true` because `buf.remaining()` was not 0 before - return ReadStatus::Readable(()); - } - - match chunks.next(buf.remaining()) { - Ok(Some(chunk)) => { - buf.put_slice(&chunk.bytes); - read = true; - } - res => return (if read { Some(()) } else { None }, res.err()).into(), - } - } - }) - .map(|res| res.map(|_| ())) + self.poll_read_generic(cx, true, |chunks| fill(chunks, buf)) + .map(|res| res.map(|_| ())) } /// Read the next segment of data @@ -408,9 +413,6 @@ impl RecvStream { let mut recv = conn.inner.recv_stream(self.stream); let mut chunks = recv.read(ordered)?; let status = read_fn(&mut chunks); - if let Some(held) = &mut self.held { - held.merge(chunks.hold()); - } if chunks.finalize().should_transmit() { conn.wake(); } @@ -460,6 +462,87 @@ impl RecvStream { } } +/// Copy from `chunks` until `buf` is full or nothing more is readable. +fn fill(chunks: &mut Chunks<'_>, buf: &mut ReadBuf<'_>) -> ReadStatus<()> { + let mut read = false; + loop { + if buf.remaining() == 0 { + // We know `read` is `true` because `buf.remaining()` was not 0 before + return ReadStatus::Readable(()); + } + + match chunks.next(buf.remaining()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read = true; + } + res => return (if read { Some(()) } else { None }, res.err()).into(), + } + } +} + +/// Connection flow control credit for bytes read with [`RecvStream::read_held`]. +/// +/// The peer cannot reuse the connection window these bytes occupy until this is dropped, which +/// returns it even after the stream or connection is gone. +pub struct Credit { + conn: Arc, + inner: moq_quic::Credit, +} + +impl Credit { + /// The number of bytes this credit covers. + pub fn len(&self) -> u64 { + self.inner.len() + } + + /// Whether this credit covers no bytes. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Split off the credit for the first `bytes`, keeping the rest. + /// + /// # Panics + /// If `bytes` exceeds [`Self::len()`]. + pub fn split(&mut self, bytes: u64) -> Self { + Self { + conn: self.conn.clone(), + inner: self.inner.split(bytes), + } + } + + /// Take over the bytes covered by `other`. + /// + /// # Panics + /// If `other` came from another connection. + pub fn merge(&mut self, mut other: Self) { + assert!( + Arc::ptr_eq(&self.conn, &other.conn), + "merged credit from another connection" + ); + self.inner.merge(mem::take(&mut other.inner)); + } +} + +impl Drop for Credit { + fn drop(&mut self) { + if self.inner.is_empty() { + return; + } + let mut conn = self.conn.state.lock("Credit::drop"); + if conn.inner.release(mem::take(&mut self.inner)).should_transmit() { + conn.wake(); + } + } +} + +impl fmt::Debug for Credit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Credit").field("len", &self.len()).finish() + } +} + enum ReadStatus { Readable(T), Finished(Option), @@ -559,12 +642,6 @@ impl tokio::io::AsyncRead for RecvStream { impl Drop for RecvStream { fn drop(&mut self) { - if let Some(held) = self.held.take().filter(|held| !held.is_empty()) { - let mut conn = self.conn.state.lock("RecvStream::drop"); - if conn.inner.release(held).should_transmit() { - conn.wake(); - } - } if self.all_data_read { debug_assert!( !self diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 427bd98943f..9e6bab5e0bb 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -43,7 +43,9 @@ use tracing::instrument::Instrument as _; use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; -use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; +use super::{ + ClientConfig, Credit, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig, +}; #[test] fn handshake_timeout() { @@ -1743,8 +1745,8 @@ async fn dropped_connection_cleans_up() { endpoint.wait_idle().await; } -/// Held reads keep the connection window spent until released, while a stream window far -/// smaller than it keeps replenishing; dropping the stream releases what is still held. +/// Held reads keep the connection window spent until their credit drops, while a stream window +/// far smaller than it keeps replenishing; credit outlives its stream. #[tokio::test] async fn held_reads_spend_the_connection_window() { const WINDOW: usize = 32 * 1024; @@ -1768,25 +1770,36 @@ async fn held_reads_spend_the_connection_window() { }); let mut recv = server.accept_uni().await.unwrap(); - recv.hold(); let mut buf = vec![0; WINDOW]; - recv.read_exact(&mut buf).await.unwrap(); + let mut held = read_held_exact(&mut recv, &mut buf).await; + assert_eq!(held.len(), WINDOW as u64); let credit = server.stats().credit; assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); // Far below the usual update threshold, and still enough to unblock. - recv.release(100); - recv.read_exact(&mut buf[..100]).await.unwrap(); + drop(held.split(100)); + held.merge(read_held_exact(&mut recv, &mut buf[..100]).await); assert_eq!(server.stats().credit.held, WINDOW as u64); - recv.release(WINDOW as u64); - recv.read_exact(&mut buf[..500]).await.unwrap(); - assert_eq!(server.stats().credit.held, 500); + drop(held); + let tail = read_held_exact(&mut recv, &mut buf[..500]).await; drop(recv); - assert_eq!(server.stats().credit.held, 0, "dropping releases the rest"); + assert_eq!(server.stats().credit.held, 500, "credit outlives its stream"); + drop(tail); + assert_eq!(server.stats().credit.held, 0); writer.await.unwrap(); } +/// Fill `buf` with held reads, returning the credit for all of it. +async fn read_held_exact(recv: &mut RecvStream, buf: &mut [u8]) -> Credit { + let mut held = recv.read_held(buf).await.unwrap().unwrap(); + while (held.len() as usize) < buf.len() { + let read = held.len() as usize; + held.merge(recv.read_held(&mut buf[read..]).await.unwrap().unwrap()); + } + held +} + #[derive(Default)] struct WakeCounter { wakes: AtomicUsize, diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index b1dc8fc86de..67d6dbc3994 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -15,7 +15,7 @@ mod stream; pub use connection::Connection; pub use endpoint::Endpoint; -pub use stream::{RecvStream, SendStream}; +pub use stream::{Credit, RecvStream, SendStream}; pub(crate) use connection::{End, Shared}; diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index 66af5782838..1be7b4e6a99 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -8,7 +8,7 @@ use std::task::{Context, Poll}; use bytes::{Buf, Bytes, BytesMut}; -use moq_quic::{Credit, StreamId, VarInt}; +use moq_quic::{StreamId, VarInt}; use super::super::Error; use super::{End, Shared}; @@ -223,12 +223,9 @@ pub struct RecvStream { stopped: bool, /// Bytes `poll_closed` read ahead, handed to `poll_read` before moq-quic's. backlog: BytesMut, - /// Connection credit for bytes read since [`Self::hold`], not yet released. - /// Includes the held part of `backlog`, which the application has not seen. - held: Option, - /// The front of `backlog` read before [`Self::hold`], whose credit is - /// already returned. - unheld: usize, + /// The connection credit for `backlog`, kept until the application reads + /// it, as if it were still in moq-quic's reassembly buffer. + backlog_credit: moq_quic::Credit, } impl RecvStream { @@ -240,49 +237,63 @@ impl RecvStream { finished: false, stopped: false, backlog: BytesMut::new(), - held: None, - unheld: 0, + backlog_credit: moq_quic::Credit::default(), } } - /// Keep the connection flow control credit for every byte read from now - /// on. + /// Read into `dst`, keeping the connection flow control credit for exactly + /// the bytes read, or `None` once the peer has finished the stream. /// - /// Stream credit still follows reads, so a stream window smaller than an - /// application message keeps the message flowing. The connection window - /// those bytes occupy stays spent until [`Self::release`] returns it, or - /// this stream is dropped. - pub fn hold(&mut self) { - if self.held.is_none() { - self.held = Some(Credit::default()); - self.unheld = self.backlog.len(); - } - } - - /// Return the connection credit for `bytes` of the held bytes this stream - /// has handed to the application. - /// - /// # Panics - /// If fewer than `bytes` read bytes are held. - pub fn release(&mut self, bytes: u64) { - let backlog = (self.backlog.len() - self.unheld) as u64; - let held = self.held.as_mut().expect("release requires hold"); - assert!( - bytes + backlog <= held.len(), - "released {bytes} bytes with {} handed out", - held.len() - backlog - ); - let credit = held.split(bytes); - self.give_back(credit); + /// Stream credit still advances, so a stream window smaller than an + /// application message keeps the message flowing, while the connection + /// window these bytes occupy stays spent until the credit is dropped. + pub fn poll_read_held(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { + let mut held = moq_quic::Credit::default(); + self.poll_read_into(cx, dst, Some(&mut held)).map_ok(|read| { + read.map(|_| Credit { + shared: self.shared.clone(), + inner: held, + }) + }) } - /// Hand `credit` to moq-quic, kicking the driver if the peer should hear. - fn give_back(&self, credit: Credit) { - if credit.is_empty() { - return; + /// [`poll_read`](web_transport_trait::poll::RecvStream::poll_read), with + /// the credit for the bytes read moved into `held` instead of returned to + /// the peer. + fn poll_read_into( + &mut self, + cx: &mut Context<'_>, + dst: &mut [u8], + held: Option<&mut moq_quic::Credit>, + ) -> Poll, Error>> { + let waiter = self.park.hold(cx); + if dst.is_empty() { + return Poll::Ready(Ok(Some(0))); } - if self.shared.conn.borrow_mut().release(credit).should_transmit() { - self.shared.kick(); + if !self.backlog.is_empty() { + return Poll::Ready(Ok(Some(self.drain(dst, held)))); + } + if self.finished { + return Poll::Ready(Ok(None)); + } + match Self::read(&self.shared, self.id, dst.len(), held) { + Read::Chunk(bytes) => { + // moq-quic hands out at most `dst.len()`. + dst[..bytes.len()].copy_from_slice(&bytes); + Poll::Ready(Ok(Some(bytes.len()))) + } + Read::Finished => { + self.finished = true; + Poll::Ready(Ok(None)) + } + Read::Blocked => { + if let Some(err) = self.shared.closed() { + return Poll::Ready(Err(err)); + } + self.shared.park_readable(self.id, waiter); + Poll::Pending + } + Read::Reset(code) => Poll::Ready(Err(Error::Reset(code))), } } @@ -295,13 +306,8 @@ impl RecvStream { /// [`stop`](web_transport_trait::poll::RecvStream::stop) with a /// full-width code, for the WebTransport HTTP/3 error mapping. pub(crate) fn stop_code(&mut self, code: u64) { - // The application never sees the read-ahead, so it can never release it. - if let Some(held) = &mut self.held { - let credit = held.split((self.backlog.len() - self.unheld) as u64); - self.give_back(credit); - } + release(&self.shared, std::mem::take(&mut self.backlog_credit)); self.backlog.clear(); - self.unheld = 0; if self.stopped || self.finished { return; } @@ -319,11 +325,12 @@ impl RecvStream { /// Take up to `max` bytes out of moq-quic's reassembly buffer. /// /// Reading is what returns the peer's flow control credit, so a read that - /// owes it a frame kicks the driver. + /// owes it a frame kicks the driver. The connection credit for the bytes + /// read moves into `held` instead, when given. /// /// Takes the shared state rather than `&mut self` so a caller can hold a /// waiter from `self.park` across the read. - fn read(shared: &Shared, id: StreamId, max: usize, held: Option<&mut Credit>) -> Read { + fn read(shared: &Shared, id: StreamId, max: usize, held: Option<&mut moq_quic::Credit>) -> Read { let mut conn = shared.conn.borrow_mut(); let mut recv = conn.recv_stream(id); let mut chunks = match recv.read(true) { @@ -348,15 +355,20 @@ impl RecvStream { read } - /// Move up to `dst.len()` read-ahead bytes out of the backlog. + /// Move up to `dst.len()` read-ahead bytes out of the backlog, with their + /// credit moved into `held` or else returned to the peer. /// /// Wakes a `poll_closed` parked at the read-ahead cap: the room it was /// waiting for is what this just made. - fn drain(&mut self, dst: &mut [u8]) -> usize { + fn drain(&mut self, dst: &mut [u8], held: Option<&mut moq_quic::Credit>) -> usize { let n = dst.len().min(self.backlog.len()); dst[..n].copy_from_slice(&self.backlog[..n]); self.backlog.advance(n); - self.unheld = self.unheld.saturating_sub(n); + let credit = self.backlog_credit.split(n as u64); + match held { + Some(held) => held.merge(credit), + None => release(&self.shared, credit), + } if n > 0 { self.shared.wake_readable(self.id); } @@ -368,35 +380,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { type Error = Error; fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - let waiter = self.park.hold(cx); - if dst.is_empty() { - return Poll::Ready(Ok(Some(0))); - } - if !self.backlog.is_empty() { - return Poll::Ready(Ok(Some(self.drain(dst)))); - } - if self.finished { - return Poll::Ready(Ok(None)); - } - match Self::read(&self.shared, self.id, dst.len(), self.held.as_mut()) { - Read::Chunk(bytes) => { - let n = bytes.len().min(dst.len()); - dst[..n].copy_from_slice(&bytes[..n]); - Poll::Ready(Ok(Some(n))) - } - Read::Finished => { - self.finished = true; - Poll::Ready(Ok(None)) - } - Read::Blocked => { - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - self.shared.park_readable(self.id, waiter); - Poll::Pending - } - Read::Reset(code) => Poll::Ready(Err(Error::Reset(code))), - } + self.poll_read_into(cx, dst, None) } fn stop(&mut self, code: u32) { @@ -426,7 +410,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { self.shared.park_readable(self.id, waiter); return Poll::Pending; } - match Self::read(&self.shared, self.id, READ_CHUNK, self.held.as_mut()) { + match Self::read(&self.shared, self.id, READ_CHUNK, Some(&mut self.backlog_credit)) { Read::Chunk(bytes) => self.backlog.extend_from_slice(&bytes), Read::Finished => { self.finished = true; @@ -447,9 +431,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { impl Drop for RecvStream { fn drop(&mut self) { - if let Some(held) = self.held.take() { - self.give_back(held); - } + release(&self.shared, std::mem::take(&mut self.backlog_credit)); self.shared.forget_recv(self.id); if !self.finished && !self.stopped { let _ = self @@ -468,3 +450,70 @@ impl std::fmt::Debug for RecvStream { f.debug_struct("RecvStream").field("id", &self.id).finish() } } + +/// Return `credit` to the peer, kicking the driver if the peer should hear. +fn release(shared: &Shared, credit: moq_quic::Credit) { + if credit.is_empty() { + return; + } + if shared.conn.borrow_mut().release(credit).should_transmit() { + shared.kick(); + } +} + +/// Connection flow control credit for bytes read with +/// [`RecvStream::poll_read_held`]. +/// +/// The peer cannot reuse the connection window these bytes occupy until this +/// is dropped, which returns it even after the stream or connection is gone. +pub struct Credit { + shared: Shared, + inner: moq_quic::Credit, +} + +impl Credit { + /// The number of bytes this credit covers. + pub fn len(&self) -> u64 { + self.inner.len() + } + + /// Whether this credit covers no bytes. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Split off the credit for the first `bytes`, keeping the rest. + /// + /// # Panics + /// If `bytes` exceeds [`Self::len()`]. + pub fn split(&mut self, bytes: u64) -> Self { + Self { + shared: self.shared.clone(), + inner: self.inner.split(bytes), + } + } + + /// Take over the bytes covered by `other`. + /// + /// # Panics + /// If `other` came from another connection. + pub fn merge(&mut self, mut other: Self) { + assert!( + std::rc::Rc::ptr_eq(&self.shared, &other.shared), + "merged credit from another connection" + ); + self.inner.merge(std::mem::take(&mut other.inner)); + } +} + +impl Drop for Credit { + fn drop(&mut self) { + release(&self.shared, std::mem::take(&mut self.inner)); + } +} + +impl std::fmt::Debug for Credit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credit").field("len", &self.len()).finish() + } +} diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index f6e84c8959f..b8ccd230f33 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -30,7 +30,7 @@ pub mod web; mod driver; -pub use driver::{Connection, RecvStream, SendStream}; +pub use driver::{Connection, Credit, RecvStream, SendStream}; pub use endpoint::Endpoint; /// The QUIC payload size every full datagram in a GSO train uses, and the diff --git a/rs/moq-uring/tests/credit.rs b/rs/moq-uring/tests/credit.rs index d2a7cbb63d5..de11d66dcc7 100644 --- a/rs/moq-uring/tests/credit.rs +++ b/rs/moq-uring/tests/credit.rs @@ -1,6 +1,6 @@ -//! Held receive credit through the io_uring driver: reads keep the connection -//! window spent until the application releases them, including bytes -//! `poll_closed` reads ahead. +//! Held receive credit through the io_uring driver: held reads keep the +//! connection window spent until their credit drops, and bytes `poll_closed` +//! reads ahead keep theirs until the application reads them. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. @@ -47,6 +47,28 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future quic::Credit { + let mut buf = vec![0u8; len]; + let mut held: Option = None; + while held.as_ref().map_or(0, quic::Credit::len) < len as u64 { + let filled = held.as_ref().map_or(0, |held| held.len() as usize); + let credit = within( + handle, + "stream data", + std::future::poll_fn(|cx| recv.poll_read_held(cx, &mut buf[filled..])), + ) + .await + .expect("read") + .expect("not finished"); + match &mut held { + Some(held) => held.merge(credit), + None => held = Some(credit), + } + } + held.expect("len > 0") +} + /// Read exactly `len` bytes. async fn read(handle: &moq_uring::Handle, recv: &mut quic::RecvStream, len: usize) { let mut buf = vec![0u8; len]; @@ -123,19 +145,18 @@ fn held_reads_spend_the_connection_window() { ) .await .expect("accept_uni"); - recv.hold(); - read(&handle, &mut recv, WINDOW).await; + let mut held = read_held(&handle, &mut recv, WINDOW).await; let credit = accepted.credit(); assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); // Far below the usual update threshold, and still enough to unblock. - recv.release(100); - read(&handle, &mut recv, 100).await; + drop(held.split(100)); + held.merge(read_held(&handle, &mut recv, 100).await); assert_eq!(accepted.credit().held, WINDOW as u64); - // Read-ahead holds credit the application has not seen yet. - recv.release(WINDOW as u64); + // Read-ahead keeps the credit the application has not taken yet. + drop(held); within( &handle, "a window of read-ahead", @@ -147,17 +168,19 @@ fn held_reads_spend_the_connection_window() { ) .await; - // Only what reached the application is releasable. + // A plain read returns it as it takes the bytes; a held read keeps it. read(&handle, &mut recv, 1000).await; - recv.release(1000); + assert_eq!(accepted.credit().held, WINDOW as u64 - 1000); + let taken = read_held(&handle, &mut recv, 500).await; assert_eq!(accepted.credit().held, WINDOW as u64 - 1000); - // Stopping returns the read-ahead; the application still owns what it read. - read(&handle, &mut recv, 500).await; + // Stopping returns the read-ahead; the application still owns what it took. recv.stop(0); assert_eq!(accepted.credit().held, 500); drop(recv); - assert_eq!(accepted.credit().held, 0, "dropping releases the rest"); + assert_eq!(accepted.credit().held, 500, "credit outlives its stream"); + drop(taken); + assert_eq!(accepted.credit().held, 0); }) .expect("worker"); } From 5d21b4fe829af8857f114dcebb36767c7d288579 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:03:33 -0700 Subject: [PATCH 13/51] fix(tokio): count in-flight accepts in wait_all_draining Endpoint::close now refuses attempts still queued for accept, and wait_all_draining waits for accepts partway through Incoming::accept, so a server shutdown cannot release the socket before those peers hear back. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 60 +++++++++++++++++--- rs/moq-tokio/src/quic/driver/tests.rs | 70 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 8 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2bae9031dc0..9093082720b 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -292,7 +292,7 @@ impl Endpoint { /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); - let mut endpoint = self.inner.state.lock().unwrap(); + let endpoint = &mut *self.inner.state.lock().unwrap(); endpoint.recv_state.connections.close = Some((error_code, reason.clone())); for sender in endpoint.recv_state.connections.senders.values() { // Ignoring errors from dropped connections @@ -301,7 +301,19 @@ impl Endpoint { reason: reason.clone(), }); } + // Refuse queued attempts now, the same as ones arriving after the close, so the peer + // hears back before a caller of `wait_all_draining` releases the socket. + let mut response_buffer = Vec::new(); + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.stats.refused_handshakes += 1; + response_buffer.clear(); + let transmit = endpoint.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut endpoint.sender); + } self.inner.shared.incoming.notify_waiters(); + if endpoint.all_draining() { + self.inner.shared.all_draining.notify_waiters(); + } } /// Wait for all connections on the endpoint to be cleanly shut down @@ -331,20 +343,22 @@ impl Endpoint { /// Wait until no connection on the endpoint is still active /// /// A connection stops being active once its peer's close arrives, or once its closing - /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out - /// the closing period for a peer that already has the close, nor wait for the + /// period ends if the peer never answers. An attempt queued for [`accept()`] or partway + /// through [`Incoming::accept`] still counts as active. Unlike [`wait_idle()`], this does + /// not sit out the closing period for a peer that already has the close, nor wait for the /// [`Connection`](crate::quic::driver::Connection) handles to drop. /// /// Does not proactively close existing connections or cause incoming connections to be /// rejected. Consider calling [`close()`] if that is desired. /// + /// [`accept()`]: Endpoint::accept /// [`close()`]: Endpoint::close /// [`wait_idle()`]: Endpoint::wait_idle pub async fn wait_all_draining(&self) { loop { { let endpoint = &mut *self.inner.state.lock().unwrap(); - if endpoint.recv_state.connections.all_draining() { + if endpoint.all_draining() { break; } // Construct future while lock is held to avoid race @@ -353,6 +367,22 @@ impl Endpoint { .await; } } + + /// Wait until a connection attempt is queued for [`accept()`](Endpoint::accept) + #[cfg(test)] + pub(crate) async fn wait_queued(&self) { + loop { + { + let endpoint = self.inner.state.lock().unwrap(); + if !endpoint.recv_state.incoming.is_empty() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.incoming.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -419,13 +449,18 @@ impl Future for EndpointDriver { impl Drop for EndpointDriver { fn drop(&mut self) { - let mut endpoint = self.0.state.lock().unwrap(); + let endpoint = &mut *self.0.state.lock().unwrap(); endpoint.driver_lost = true; self.0.shared.incoming.notify_waiters(); // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); endpoint.recv_state.connections.draining.clear(); + // `accept` yields nothing once the driver is lost, so queued attempts would hold off + // `wait_all_draining` forever. + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.inner.ignore(incoming); + } self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else @@ -500,6 +535,10 @@ impl EndpointInner { if state.is_idle() { self.shared.idle.notify_waiters(); } + // The pending accept held off `wait_all_draining`, and a failed one sends no event. + if state.all_draining() { + self.shared.all_draining.notify_waiters(); + } result } @@ -609,9 +648,7 @@ impl State { shared.idle.notify_waiters(); } } - if (event.is_draining() || event.is_drained()) - && self.recv_state.connections.all_draining() - { + if (event.is_draining() || event.is_drained()) && self.all_draining() { shared.all_draining.notify_waiters(); } let Some(event) = self.inner.handle_event(ch, event) else { @@ -633,6 +670,13 @@ impl State { fn is_idle(&self) -> bool { self.recv_state.connections.is_empty() && self.pending_accepts == 0 } + + /// Whether no connection is still active, nor any attempt left that could become one + fn all_draining(&self) -> bool { + self.recv_state.connections.all_draining() + && self.pending_accepts == 0 + && self.recv_state.incoming.is_empty() + } } impl Drop for State { diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0b284f17706..5d50d7a004a 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1285,6 +1285,76 @@ async fn split_accept_close_during_pending_accept() { assert_eq!(server.open_connections(), 0); } +/// An accept still in progress when the endpoint closes will send a close of its own, so +/// `wait_all_draining` must not report the endpoint drained until that accept finishes. +#[tokio::test] +async fn split_accept_close_holds_off_all_draining() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, _) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + + blocker.release(); + // The waiter parked above resolves only if the accept's close is accounted for and announced. + timeout(Duration::from_secs(5), draining).await.unwrap(); + assert!(await_connection(connect).await.is_err()); + + let _ = await_connection(accept).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// `close` refuses attempts still queued for `accept`, so the peer hears back before +/// `wait_all_draining` lets the caller release the socket. +#[tokio::test] +async fn close_refuses_queued_incoming() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + server.close(0u32.into(), b"closing"); + timeout(Duration::from_secs(5), server.wait_all_draining()) + .await + .unwrap(); + assert!(server.accept().await.is_none()); + + // Without the refusal, the attempt sits in the queue and the client waits out its timeout. + let error = timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .unwrap_err(); + assert!(matches!( + error, + crate::quic::driver::ConnectionError::ConnectionClosed(close) + if close.error_code == moq_quic::TransportErrorCode::CONNECTION_REFUSED + )); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From 50ebb4e22a81d50a2301ea8d3bd0dd30d37ed146 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:23:26 -0700 Subject: [PATCH 14/51] style(moq-tokio): apply quinn formatting to the held credit code Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/recv_stream.rs | 6 +++++- rs/moq-tokio/src/quic/driver/tests.rs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 75c0d1edcb8..6733220179c 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -531,7 +531,11 @@ impl Drop for Credit { return; } let mut conn = self.conn.state.lock("Credit::drop"); - if conn.inner.release(mem::take(&mut self.inner)).should_transmit() { + if conn + .inner + .release(mem::take(&mut self.inner)) + .should_transmit() + { conn.wake(); } } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 9e6bab5e0bb..5a0635fe623 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1784,7 +1784,11 @@ async fn held_reads_spend_the_connection_window() { drop(held); let tail = read_held_exact(&mut recv, &mut buf[..500]).await; drop(recv); - assert_eq!(server.stats().credit.held, 500, "credit outlives its stream"); + assert_eq!( + server.stats().credit.held, + 500, + "credit outlives its stream" + ); drop(tail); assert_eq!(server.stats().credit.held, 0); writer.await.unwrap(); From 21cac2b27435c4b2084018c98b0a3cb5be467cc6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 00:06:45 -0700 Subject: [PATCH 15/51] fix(tokio): wake draining waiters when accept empties the queue Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 5 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 38 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 9093082720b..2a214d54661 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -821,6 +821,11 @@ impl Future for Accept<'_> { return Poll::Ready(None); } if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // A handed-out attempt no longer holds off `wait_all_draining`, and refusing, + // ignoring, or dropping it sends no event that would wake the waiter later. + if endpoint.all_draining() { + this.endpoint.inner.shared.all_draining.notify_waiters(); + } // Release the mutex lock on endpoint so cloning it doesn't deadlock drop(endpoint); let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 5d50d7a004a..aae7054d83a 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1355,6 +1355,44 @@ async fn close_refuses_queued_incoming() { )); } +/// Taking the last queued attempt and refusing it sends no event, so `accept` itself must wake a +/// parked `wait_all_draining`. +#[tokio::test] +async fn refused_incoming_wakes_all_draining_waiter() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, wake_counter) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + server.accept().await.unwrap().refuse(); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), draining).await.unwrap(); + + assert!( + timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .is_err() + ); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From c1f4cfd3ef7ad1c6ef7136e1d323ec272708148f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 07:47:22 -0700 Subject: [PATCH 16/51] fix(tokio): surface non-WouldBlock receive errors, cap moq_quic logs The server socket retried every failed recv, so a persistent error spun the endpoint task while tokio kept readiness asserted. Only WouldBlock retries now, matching the driver's tokio runtime. Also cap moq_quic at info like noq was, fix the driver doc example, and point the moq-uring README at the just recipe. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/log.rs | 1 + rs/moq-tokio/src/quic/backend.rs | 8 ++++++-- rs/moq-tokio/src/quic/driver/endpoint.rs | 4 ++-- rs/moq-uring/README.md | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index 5d8a5e4f198..6b4833f9d16 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,6 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) + .add_directive("moq_quic=info".parse()?) .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 666040d3246..e743af6ea7c 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -627,8 +627,12 @@ impl driver::AsyncUdpSocket for Socket { // Only the endpoint driver receives, so the one readiness waker is enough. loop { ready!(io.poll_recv_ready(cx))?; - if let Ok(res) = io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { - return Poll::Ready(Ok(res)); + match io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`; retrying any other error + // would spin, since `poll_recv_ready` completes at once and the recv fails again. + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} + Err(err) => return Poll::Ready(Err(err)), } } } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2a214d54661..eeaaea5e3db 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -59,8 +59,8 @@ impl Endpoint { /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other /// address. For example: /// - /// ``` - /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ```no_run + /// moq_tokio::quic::driver::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); /// ``` /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index db5a5a8aeca..ec84787ea01 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -82,7 +82,7 @@ timers, and UDP socket can be built without QUIC. Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features quic +just rs uring ``` `moq-relay` enables it with `--features io-uring`. From b38976329a7585240eeb6ae0a6e1c60ff6c2f2cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:35:17 -0700 Subject: [PATCH 17/51] refactor(moq-uring): rename Connection::credit to credit_stats Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/tests.rs | 4 ++-- rs/moq-uring/src/quic/driver/connection.rs | 2 +- rs/moq-uring/tests/credit.rs | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 4f93ec1e645..4b63539caf3 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1881,8 +1881,8 @@ async fn held_reads_spend_the_connection_window() { let mut buf = vec![0; WINDOW]; let mut held = read_held_exact(&mut recv, &mut buf).await; assert_eq!(held.len(), WINDOW as u64); - let credit = server.stats().credit; - assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); + let stats = server.stats().credit; + assert_eq!((stats.held, stats.available), (WINDOW as u64, 0)); // Far below the usual update threshold, and still enough to unblock. drop(held.split(100)); diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 21f5c9f45b8..82a6d357c1c 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -340,7 +340,7 @@ impl Connection { /// How much receive credit the application holds, and how much the peer /// may still send. - pub fn credit(&self) -> moq_quic::CreditStats { + pub fn credit_stats(&self) -> moq_quic::CreditStats { self.shared.conn.borrow().stats().credit } } diff --git a/rs/moq-uring/tests/credit.rs b/rs/moq-uring/tests/credit.rs index de11d66dcc7..4a6e5b352e4 100644 --- a/rs/moq-uring/tests/credit.rs +++ b/rs/moq-uring/tests/credit.rs @@ -147,13 +147,13 @@ fn held_reads_spend_the_connection_window() { .expect("accept_uni"); let mut held = read_held(&handle, &mut recv, WINDOW).await; - let credit = accepted.credit(); - assert_eq!((credit.held, credit.available), (WINDOW as u64, 0)); + let stats = accepted.credit_stats(); + assert_eq!((stats.held, stats.available), (WINDOW as u64, 0)); // Far below the usual update threshold, and still enough to unblock. drop(held.split(100)); held.merge(read_held(&handle, &mut recv, 100).await); - assert_eq!(accepted.credit().held, WINDOW as u64); + assert_eq!(accepted.credit_stats().held, WINDOW as u64); // Read-ahead keeps the credit the application has not taken yet. drop(held); @@ -162,7 +162,7 @@ fn held_reads_spend_the_connection_window() { "a window of read-ahead", std::future::poll_fn(|cx| match recv.poll_closed(cx) { Poll::Ready(res) => panic!("stream ended early: {res:?}"), - Poll::Pending if accepted.credit().held == WINDOW as u64 => Poll::Ready(()), + Poll::Pending if accepted.credit_stats().held == WINDOW as u64 => Poll::Ready(()), Poll::Pending => Poll::Pending, }), ) @@ -170,17 +170,17 @@ fn held_reads_spend_the_connection_window() { // A plain read returns it as it takes the bytes; a held read keeps it. read(&handle, &mut recv, 1000).await; - assert_eq!(accepted.credit().held, WINDOW as u64 - 1000); + assert_eq!(accepted.credit_stats().held, WINDOW as u64 - 1000); let taken = read_held(&handle, &mut recv, 500).await; - assert_eq!(accepted.credit().held, WINDOW as u64 - 1000); + assert_eq!(accepted.credit_stats().held, WINDOW as u64 - 1000); // Stopping returns the read-ahead; the application still owns what it took. recv.stop(0); - assert_eq!(accepted.credit().held, 500); + assert_eq!(accepted.credit_stats().held, 500); drop(recv); - assert_eq!(accepted.credit().held, 500, "credit outlives its stream"); + assert_eq!(accepted.credit_stats().held, 500, "credit outlives its stream"); drop(taken); - assert_eq!(accepted.credit().held, 0); + assert_eq!(accepted.credit_stats().held, 0); }) .expect("worker"); } From c9b84039808894c1385f94fae6d47a98933c7444 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:35:17 -0700 Subject: [PATCH 18/51] quest: claim quest/m1/quic/receive-memory Co-Authored-By: Claude Opus 5.5 From 738c16ec109b76223b624f8c6b1dfe9038037dd2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:57:03 -0700 Subject: [PATCH 19/51] refactor(moq-uring): expose Connection::stats instead of credit_stats Match the tokio driver and quinn: credit is read as stats().credit in both drivers. The inherent stats() shadows the poll::Session trait's on concrete calls, so the web test reads UDP bytes from it directly. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/receive-memory.md | 9 +++++---- rs/moq-uring/src/quic/driver/connection.rs | 7 +++---- rs/moq-uring/tests/credit.rs | 16 ++++++++-------- rs/moq-uring/tests/web.rs | 5 ++--- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md index 5371a7525a3..5e2bd90a1b7 100644 --- a/quest/m1/quic/receive-memory.md +++ b/quest/m1/quic/receive-memory.md @@ -23,10 +23,11 @@ drivers' receive streams have a copying `poll_read_held` that yields a `Credit` for exactly the bytes read. Dropping a `Credit` returns it, so a reader buffer keeps one merged `Credit` and drops `split(n)` for each complete unit; read-ahead stays in the remainder. io_uring's `poll_closed` -read-ahead keeps its credit until the application reads it. Connection stats -report held and still-available credit. moq-net's transport trait and the -WebTransport wrappers do not carry it yet; reshape any of it to fit the -consumer. +read-ahead keeps its credit until the application reads it. Both drivers' +`Connection::stats().credit` reports held and still-available credit. +`poll_read_held` stays separate until this quest folds it into `poll_read`. +moq-net's transport trait and the WebTransport wrappers do not carry it yet; +reshape any of it to fit the consumer. Hold connection credit through reader buffers, type/length prefixes, headers, extensions, and frame assembly. Attribute read-ahead bytes to the next unit diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 82a6d357c1c..a672a9417b2 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -338,10 +338,9 @@ impl Connection { &self.handshake } - /// How much receive credit the application holds, and how much the peer - /// may still send. - pub fn credit_stats(&self) -> moq_quic::CreditStats { - self.shared.conn.borrow().stats().credit + /// Returns connection statistics. + pub fn stats(&self) -> moq_quic::ConnectionStats { + self.shared.conn.borrow().stats() } } diff --git a/rs/moq-uring/tests/credit.rs b/rs/moq-uring/tests/credit.rs index 4a6e5b352e4..6a9fcf3787b 100644 --- a/rs/moq-uring/tests/credit.rs +++ b/rs/moq-uring/tests/credit.rs @@ -147,13 +147,13 @@ fn held_reads_spend_the_connection_window() { .expect("accept_uni"); let mut held = read_held(&handle, &mut recv, WINDOW).await; - let stats = accepted.credit_stats(); + let stats = accepted.stats().credit; assert_eq!((stats.held, stats.available), (WINDOW as u64, 0)); // Far below the usual update threshold, and still enough to unblock. drop(held.split(100)); held.merge(read_held(&handle, &mut recv, 100).await); - assert_eq!(accepted.credit_stats().held, WINDOW as u64); + assert_eq!(accepted.stats().credit.held, WINDOW as u64); // Read-ahead keeps the credit the application has not taken yet. drop(held); @@ -162,7 +162,7 @@ fn held_reads_spend_the_connection_window() { "a window of read-ahead", std::future::poll_fn(|cx| match recv.poll_closed(cx) { Poll::Ready(res) => panic!("stream ended early: {res:?}"), - Poll::Pending if accepted.credit_stats().held == WINDOW as u64 => Poll::Ready(()), + Poll::Pending if accepted.stats().credit.held == WINDOW as u64 => Poll::Ready(()), Poll::Pending => Poll::Pending, }), ) @@ -170,17 +170,17 @@ fn held_reads_spend_the_connection_window() { // A plain read returns it as it takes the bytes; a held read keeps it. read(&handle, &mut recv, 1000).await; - assert_eq!(accepted.credit_stats().held, WINDOW as u64 - 1000); + assert_eq!(accepted.stats().credit.held, WINDOW as u64 - 1000); let taken = read_held(&handle, &mut recv, 500).await; - assert_eq!(accepted.credit_stats().held, WINDOW as u64 - 1000); + assert_eq!(accepted.stats().credit.held, WINDOW as u64 - 1000); // Stopping returns the read-ahead; the application still owns what it took. recv.stop(0); - assert_eq!(accepted.credit_stats().held, 500); + assert_eq!(accepted.stats().credit.held, 500); drop(recv); - assert_eq!(accepted.credit_stats().held, 500, "credit outlives its stream"); + assert_eq!(accepted.stats().credit.held, 500, "credit outlives its stream"); drop(taken); - assert_eq!(accepted.credit_stats().held, 0); + assert_eq!(accepted.stats().credit.held, 0); }) .expect("worker"); } diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index 2fbf2157ba4..ffdd71de508 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -737,8 +737,7 @@ fn cancelling_a_pending_web_handshake(send_settings: bool) { let mut watch = conn.clone(); let mut control = None; if send_settings { - use web_transport_trait::Stats as _; - let before = watch.stats().bytes_received().expect("receive stats"); + let before = watch.stats().udp_rx.bytes; let mut stream = std::future::poll_fn(|cx| peer.poll_open_uni(cx)) .await .expect("control stream"); @@ -756,7 +755,7 @@ fn cancelling_a_pending_web_handshake(send_settings: bool) { control = Some(stream); within(&handle, "peer SETTINGS to arrive", async { loop { - if watch.stats().bytes_received().expect("receive stats") > before { + if watch.stats().udp_rx.bytes > before { break; } let mut tick = moq_uring::Timer::after(&handle, std::time::Duration::from_millis(10)); From c0f87053971d9c0a2cdbdba5790dcf66ba659884 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:08:47 -0700 Subject: [PATCH 20/51] wip(moq-net): carry receive credit through transport and Reader Adds an associated Credit to transport::poll::RecvStream, an edge-triggered poll_credit_exhausted on Session, a memory transport with a real receive window, and a crate-private pressure registry for STOP_SENDING recovery. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/client.rs | 16 +- rs/moq-net/src/coding/reader.rs | 167 ++++++++++++++++-- rs/moq-net/src/fuzz.rs | 12 +- rs/moq-net/src/ietf/adapter.rs | 112 +++++++++--- rs/moq-net/src/lib.rs | 2 + rs/moq-net/src/lite/test_transport.rs | 23 ++- rs/moq-net/src/pressure.rs | 236 ++++++++++++++++++++++++++ rs/moq-net/src/server.rs | 9 +- rs/moq-net/src/transport.rs | 148 ++++++++++++---- rs/moq-net/src/transport/memory.rs | 198 ++++++++++++++++++--- rs/moq-net/tests/subscribe_tracks.rs | 2 +- rs/moq-net/tests/support/mock.rs | 51 +++--- 12 files changed, 840 insertions(+), 136 deletions(-) create mode 100644 rs/moq-net/src/pressure.rs diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 395fde91f38..91eefbd3f76 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -648,6 +648,10 @@ mod tests { Poll::Ready(FakeError) } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { FakeStats { send_rate: *self.state.send_rate.lock().unwrap(), @@ -703,8 +707,9 @@ mod tests { impl crate::transport::poll::RecvStream for FakeRecvStream { type Error = FakeError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { if self.data.is_empty() { return Poll::Ready(Ok(None)); } @@ -713,7 +718,7 @@ mod tests { for slot in dst.iter_mut().take(size) { *slot = self.data.pop_front().unwrap(); } - Poll::Ready(Ok(Some(size))) + Poll::Ready(Ok(Some(crate::transport::Unheld(size as u64)))) } fn stop(&mut self, _code: u32) {} @@ -1033,6 +1038,10 @@ mod tests { self.inner.poll_closed(cx) } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { self.inner.stats() } @@ -1064,8 +1073,9 @@ mod tests { impl crate::transport::poll::RecvStream for LocalRecv { type Error = FakeError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { self.inner.poll_read(cx, dst) } diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 5c3bf08ec8a..b6c2a78da03 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -6,7 +6,13 @@ use std::{ use bytes::{Buf, Bytes, BytesMut}; -use crate::{Error, StreamError, coding::*}; +use crate::{ + Error, StreamError, + coding::*, + pressure::{Pressure, Rank, Ticket}, + + transport::Credit, +}; /// A reader for decoding messages from a stream. /// @@ -14,10 +20,24 @@ use crate::{Error, StreamError, coding::*}; /// wrappers, so the reader can be driven from another poll function without /// pinning a future. Partial bytes accumulate in the internal buffer, so a /// `Pending` (or a cancelled wrapper) never desynchronizes the stream. +/// +/// A [tracked](Self::track) reader holds the transport's connection credit for its +/// buffered bytes and for the unit it is assembling, releasing each unit's bytes once +/// it is decoded and handed off, so read-ahead stays with the next unit. An untracked +/// reader returns credit as soon as it reads. pub struct Reader { stream: S, buffer: BytesMut, version: V, + /// Covers `buffer` plus `unit`, less `free`, while tracked. + credit: Option, + /// Bytes of the incomplete frame already taken out of `buffer`. + unit: u64, + /// Leading bytes read before tracking began, whose credit already returned. + free: u64, + ticket: Option, + /// The receive allowance filled and this stream was chosen to free it. + evicted: bool, } /// How much payload may pile up behind a parked consumer before @@ -37,9 +57,79 @@ impl Reader { stream, buffer: Default::default(), version, + credit: None, + unit: 0, + free: 0, + ticket: None, + evicted: false, + } + } + + /// Hold connection credit until each unit completes, cancellable by `pressure` at + /// `rank`. + pub(crate) fn track(&mut self, pressure: &Pressure, rank: Rank) { + if self.ticket.is_some() { + return self.rank(rank); + } + self.free = self.buffer.len() as u64 + self.unit; + self.ticket = Some(pressure.track(rank)); + } + + /// Re-rank a tracked stream once its header identifies it. + pub(crate) fn rank(&self, rank: Rank) { + if let Some(ticket) = &self.ticket { + ticket.rank(rank); } } + /// Keep the credit for bytes just read, if tracked; otherwise it returns now. + fn take(&mut self, credit: S::Credit) { + let Some(ticket) = &self.ticket else { return }; + let held = match &mut self.credit { + Some(held) => { + held.merge(credit); + held.len() + } + None => { + let len = credit.len(); + self.credit = Some(credit); + len + } + }; + ticket.held(held); + } + + /// Return the credit for `bytes` that completed a unit. + fn release(&mut self, bytes: u64) { + let free = self.free.min(bytes); + self.free -= free; + let (Some(ticket), Some(credit)) = (&self.ticket, &mut self.credit) else { + return; + }; + drop(credit.split(bytes - free)); + ticket.held(credit.len()); + } + + /// Drop `used` decoded bytes from the buffer, returning their credit. + fn consume(&mut self, used: usize) { + self.buffer.advance(used); + self.release(used as u64); + } + + /// Whether this stream was chosen to free the receive allowance. The first time, + /// stop it and let go of everything it holds. + fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { + if self.evicted { + return true; + } + if !self.ticket.as_ref().is_some_and(|ticket| ticket.poll_evicted(cx)) { + return false; + } + self.evicted = true; + self.abort(&Error::Evicted); + true + } + /// Poll `decode` against the buffered bytes, reading more until it stops coming up /// short. `consume` drops what it read; otherwise the bytes stay for the next read. fn poll_with( @@ -57,7 +147,7 @@ impl Reader { Ok(value) => { if consume { let used = self.buffer.len() - r.remaining(); - self.buffer.advance(used); + self.consume(used); } return Poll::Ready(Ok(value)); } @@ -190,22 +280,42 @@ impl Reader { std::future::poll_fn(|cx| self.poll_varint_peek(cx)).await } - /// Poll for the next chunk, draining the reader's internal buffer first. + /// Poll for the next chunk, draining the reader's internal buffer first. The chunk is + /// a complete unit: its credit returns as it is handed over. pub fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { + let chunk = ready!(self.poll_unit_chunk(cx, max))?; + let size = chunk.as_ref().map_or(0, |chunk| chunk.len() as u64); + self.release(size); + self.unit -= size; + Poll::Ready(Ok(chunk)) + } + + /// The next chunk of the current unit, draining the buffer first. Its credit stays + /// held in `unit` until the caller releases it. + fn poll_unit_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { if !self.buffer.is_empty() { let n = cmp::min(self.buffer.len(), max); + self.unit += n as u64; return Poll::Ready(Ok(Some(self.buffer.split_to(n).freeze()))); } - self.stream - .poll_read_chunk(cx, max) - .map_err(|err| self.version.transport_error(err)) + if self.poll_evicted(cx) { + return Poll::Ready(Err(Error::Evicted)); + } + let Some((chunk, credit)) = ready!(self.stream.poll_read_chunk(cx, max)).map_err(|err| self.version.transport_error(err))? + else { + return Poll::Ready(Ok(None)); + }; + self.unit += chunk.len() as u64; + self.take(credit); + Poll::Ready(Ok(Some(chunk))) } /// Fill a frame's payload from the stream, returning `Pending` once it would block. /// /// `Ready(Ok(()))` means the frame is full and the caller should commit it; an error /// means the caller should abort it. Either way the frame is untouched otherwise, so - /// the caller keeps ownership of the commit/abort decision. + /// the caller keeps ownership of the commit/abort decision. The payload's credit is + /// held until then, whichever way the frame ends. /// /// Consumers are woken where the loop yields rather than once per chunk. The wire /// hands over everything it has already buffered in one turn, and each wake pays a @@ -227,7 +337,7 @@ impl Reader { break Ok(()); } - match self.poll_read_chunk(cx, frame.remaining()) { + match self.poll_unit_chunk(cx, frame.remaining()) { Poll::Pending => { if owed > 0 { frame.notify(); @@ -250,6 +360,9 @@ impl Reader { } }; + // The frame is handed off or discarded either way. + let unit = std::mem::take(&mut self.unit); + self.release(unit); Poll::Ready(result) } @@ -260,7 +373,9 @@ impl Reader { return Poll::Ready(Err(DecodeError::Short.into())); } } - Poll::Ready(Ok(self.buffer.split_to(size).freeze())) + let bytes = self.buffer.split_to(size).freeze(); + self.release(size as u64); + Poll::Ready(Ok(bytes)) } /// Poll to discard `remaining` bytes, counting it down as they arrive. Nothing is @@ -269,7 +384,7 @@ impl Reader { pub(crate) fn poll_skip(&mut self, cx: &mut Context<'_>, remaining: &mut usize) -> Poll> { loop { let skipped = self.buffer.len().min(*remaining); - self.buffer.advance(skipped); + self.consume(skipped); *remaining -= skipped; if *remaining == 0 { return Poll::Ready(Ok(())); @@ -306,19 +421,31 @@ impl Reader { /// Poll for more data from the stream. `true` if data was read, `false` if the /// stream finished. fn poll_read_more(&mut self, cx: &mut Context<'_>) -> Poll> { + if self.poll_evicted(cx) { + return Poll::Ready(Err(Error::Evicted)); + } match ready!(self.stream.poll_read_buf(cx, &mut self.buffer)) { - Ok(Some(_)) => Poll::Ready(Ok(true)), + Ok(Some(credit)) => { + self.take(credit); + Poll::Ready(Ok(true)) + } Ok(None) => Poll::Ready(Ok(false)), Err(e) => Poll::Ready(Err(self.version.transport_error(e))), } } - /// Abort the stream with the given error. + /// Abort the stream with the given error, returning the credit it holds. pub fn abort(&mut self, err: impl Into) { // STOP_SENDING is a stream operation, so it carries a stream code from the // negotiated protocol's registry. A session code, or the other protocol's, would be // read against the wrong table and mean something else. self.stream.stop(self.version.encode_stream_code(&err.into())); + // What is buffered stays readable, but nothing more is worth holding for. + self.free = self.buffer.len() as u64 + self.unit; + self.credit = None; + if let Some(ticket) = &self.ticket { + ticket.held(0); + } } /// Cast the reader to a different version, used during version negotiation. @@ -327,6 +454,11 @@ impl Reader { stream: self.stream, buffer: self.buffer, version, + credit: self.credit, + unit: self.unit, + free: self.free, + ticket: self.ticket, + evicted: self.evicted, } } } @@ -348,12 +480,13 @@ mod tests { impl crate::transport::poll::RecvStream for StopLog { type Error = crate::lite::test_transport::SinkError; + type Credit = crate::transport::Unheld; fn poll_read( &mut self, _cx: &mut std::task::Context<'_>, _dst: &mut [u8], - ) -> std::task::Poll, Self::Error>> { + ) -> std::task::Poll, Self::Error>> { std::task::Poll::Ready(Ok(None)) } @@ -454,8 +587,9 @@ mod tests { impl crate::transport::poll::RecvStream for Chunks { type Error = crate::lite::test_transport::SinkError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let Some(chunk) = self.0.pop_front() else { return Poll::Pending; }; @@ -467,7 +601,7 @@ mod tests { self.0.push_front(&chunk[n..]); } - Poll::Ready(Ok(Some(n))) + Poll::Ready(Ok(Some(crate::transport::Unheld(n as u64)))) } fn stop(&mut self, _code: u32) {} @@ -655,8 +789,9 @@ mod tests { impl crate::transport::poll::RecvStream for Burst { type Error = crate::lite::test_transport::SinkError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { // Start each turn with none of the last turn's registrations left. self.waiter = kio::Waiter::new(self.waiter.waker().clone()); while self.payload.poll_read_chunk(&self.waiter).is_ready() {} diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 4eb964978a7..cc7ae0fac20 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1162,17 +1162,18 @@ struct Chunks { impl crate::transport::poll::RecvStream for Chunks { type Error = NoError; + type Credit = crate::transport::Unheld; fn poll_read( &mut self, cx: &mut std::task::Context<'_>, dst: &mut [u8], - ) -> std::task::Poll, NoError>> { + ) -> std::task::Poll, NoError>> { let max = dst.len(); self.poll_read_chunk(cx, max).map_ok(|chunk| { - chunk.map(|chunk| { + chunk.map(|(chunk, credit)| { dst[..chunk.len()].copy_from_slice(&chunk); - chunk.len() + credit }) }) } @@ -1182,7 +1183,7 @@ impl crate::transport::poll::RecvStream for Chunks { &mut self, _cx: &mut std::task::Context<'_>, max: usize, - ) -> std::task::Poll, NoError>> { + ) -> std::task::Poll, NoError>> { self.ready = !self.ready; if self.ready { return std::task::Poll::Pending; @@ -1191,7 +1192,8 @@ impl crate::transport::poll::RecvStream for Chunks { return std::task::Poll::Ready(Ok(None)); } let n = self.chunk.min(max).min(self.payload.len()); - std::task::Poll::Ready(Ok(Some(self.payload.split_to(n)))) + std::task::Poll::Ready(Ok(Some((self.payload.split_to(n), crate::transport::Unheld(n as u64))))) + } fn stop(&mut self, _code: u32) {} diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 03ab85c3f76..9f11276b241 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -10,8 +10,10 @@ use crate::{ Error, PathOwned, coding::{Decode, Decoder, Encoder, Reader, Writer}, ietf::{self, RequestId}, + transport::{Credit, Unheld}, }; + use super::{Control, Message, Version, control::RequestPermit}; // === Message Queues === @@ -159,12 +161,15 @@ impl VirtualRecvStream { } } +/// The routed control messages a virtual stream holds were complete when the real control +/// stream handed them off, so their connection credit is already returned. impl crate::transport::poll::RecvStream for VirtualRecvStream { type Error = crate::Error; + type Credit = Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { if dst.is_empty() { - return Poll::Ready(Ok(Some(0))); + return Poll::Ready(Ok(Some(Unheld(0)))); } std::task::ready!(self.poll_fill(cx)); @@ -175,16 +180,16 @@ impl crate::transport::poll::RecvStream for VirtualRecvStream { let n = dst.len().min(self.buffer.len()); dst[..n].copy_from_slice(&self.buffer[..n]); self.buffer.advance(n); - Poll::Ready(Ok(Some(n))) + Poll::Ready(Ok(Some(Unheld(n as u64)))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { + ) -> Poll, Self::Error>> { if !buf.has_remaining_mut() { - return Poll::Ready(Ok(Some(0))); + return Poll::Ready(Ok(Some(Unheld(0)))); } std::task::ready!(self.poll_fill(cx)); @@ -194,12 +199,16 @@ impl crate::transport::poll::RecvStream for VirtualRecvStream { let n = buf.remaining_mut().min(self.buffer.len()); buf.put(self.buffer.split_to(n)); - Poll::Ready(Ok(Some(n))) + Poll::Ready(Ok(Some(Unheld(n as u64)))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); + return Poll::Ready(Ok(Some((Bytes::new(), Unheld(0))))); } std::task::ready!(self.poll_fill(cx)); @@ -208,7 +217,7 @@ impl crate::transport::poll::RecvStream for VirtualRecvStream { } let n = max.min(self.buffer.len()); - Poll::Ready(Ok(Some(self.buffer.split_to(n)))) + Poll::Ready(Ok(Some((self.buffer.split_to(n), Unheld(n as u64))))) } fn stop(&mut self, _code: u32) { @@ -455,13 +464,48 @@ pub enum AdapterRecv { Virtual(VirtualRecvStream), } +/// The credit of an [`AdapterRecv`]: the real stream's, or none for a virtual one. +pub enum AdapterCredit { + Real(C), + Virtual(Unheld), +} + +impl Credit for AdapterCredit { + fn len(&self) -> u64 { + match self { + Self::Real(c) => c.len(), + Self::Virtual(c) => c.len(), + } + } + + fn split(&mut self, bytes: u64) -> Self { + match self { + Self::Real(c) => Self::Real(c.split(bytes)), + Self::Virtual(c) => Self::Virtual(c.split(bytes)), + } + } + + fn merge(&mut self, other: Self) { + match (self, other) { + (Self::Real(c), Self::Real(other)) => c.merge(other), + (Self::Virtual(c), Self::Virtual(other)) => c.merge(other), + _ => panic!("merged a real stream's credit with a virtual one's"), + } + } +} + +type RealCredit = <::RecvStream as crate::transport::poll::RecvStream>::Credit; + impl crate::transport::poll::RecvStream for AdapterRecv { type Error = crate::Error; + type Credit = AdapterCredit>; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read(cx, dst).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read(cx, dst), + Self::Real(s) => s + .poll_read(cx, dst) + .map(|r| r.map(|c| c.map(AdapterCredit::Real)).map_err(|_| crate::Error::Closed)), + Self::Virtual(s) => s.poll_read(cx, dst).map_ok(|c| c.map(AdapterCredit::Virtual)), } } @@ -469,20 +513,32 @@ impl crate::transport::poll::RecvStream for &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { + ) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read_buf(cx, buf).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read_buf(cx, buf), + Self::Real(s) => s + .poll_read_buf(cx, buf) + .map(|r| r.map(|c| c.map(AdapterCredit::Real)).map_err(|_| crate::Error::Closed)), + Self::Virtual(s) => s.poll_read_buf(cx, buf).map_ok(|c| c.map(AdapterCredit::Virtual)), } } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { match self { - Self::Real(s) => s.poll_read_chunk(cx, max).map(|r| r.map_err(|_| crate::Error::Closed)), - Self::Virtual(s) => s.poll_read_chunk(cx, max), + Self::Real(s) => s.poll_read_chunk(cx, max).map(|r| { + r.map(|c| c.map(|(b, c)| (b, AdapterCredit::Real(c)))) + .map_err(|_| crate::Error::Closed) + }), + Self::Virtual(s) => s + .poll_read_chunk(cx, max) + .map_ok(|c| c.map(|(b, c)| (b, AdapterCredit::Virtual(c)))), } } + fn stop(&mut self, code: u32) { match self { Self::Real(s) => s.stop(code), @@ -1189,6 +1245,10 @@ impl crate::transport::poll::Session for Con self.inner.poll_closed(cx).map(|_| crate::Error::Closed) } + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + self.inner.poll_credit_exhausted(cx) + } + fn stats(&self) -> impl crate::transport::Stats { self.inner.stats() } @@ -1367,14 +1427,14 @@ mod tests { .unwrap(); let (_, mut recv) = shared.incoming.pop().await.unwrap(); assert_eq!( - recv.read_chunk(usize::MAX).await.unwrap(), + recv.read_chunk(usize::MAX).await.unwrap().map(|(b, _)| b), Some(Bytes::from_static(b"request")) ); assert_eq!(routed, target_id, "{version}: update {update_id} routed to {routed}"); shared.push(routed, raw.clone()); assert_eq!( - recv.read_chunk(usize::MAX).await.unwrap(), + recv.read_chunk(usize::MAX).await.unwrap().map(|(b, _)| b), Some(raw), "{version}: update {update_id} did not reach {target_id}" ); @@ -1438,12 +1498,12 @@ mod tests { // Read initial data let mut buf = [0u8; 32]; - let n = stream.read(&mut buf).await.unwrap().unwrap(); + let n = stream.read(&mut buf).await.unwrap().unwrap().0 as usize; assert_eq!(&buf[..n], b"initial"); // Send follow-up tx.push(Bytes::from_static(b"followup")); - let n = stream.read(&mut buf).await.unwrap().unwrap(); + let n = stream.read(&mut buf).await.unwrap().unwrap().0 as usize; assert_eq!(&buf[..n], b"followup"); // Drop the writer → FIN @@ -1459,14 +1519,14 @@ mod tests { // Read small chunks let mut buf = [0u8; 5]; - let n = stream.read(&mut buf).await.unwrap().unwrap(); + let n = stream.read(&mut buf).await.unwrap().unwrap().0 as usize; assert_eq!(&buf[..n], b"hello"); - let n = stream.read(&mut buf).await.unwrap().unwrap(); + let n = stream.read(&mut buf).await.unwrap().unwrap().0 as usize; assert_eq!(&buf[..n], b" worl"); let mut buf = [0u8; 1]; - let n = stream.read(&mut buf).await.unwrap().unwrap(); + let n = stream.read(&mut buf).await.unwrap().unwrap().0 as usize; assert_eq!(&buf[..n], b"d"); } @@ -1564,7 +1624,7 @@ mod tests { // Drain the advertisement the stream opened with, so both halves are quiescent // and a later read only sees what the terminal message routes. assert_eq!( - recv.read_chunk(usize::MAX).await.unwrap(), + recv.read_chunk(usize::MAX).await.unwrap().map(|(b, _)| b), Some(encode_msg(&msg, version)) ); recv diff --git a/rs/moq-net/src/lib.rs b/rs/moq-net/src/lib.rs index 63ff8b119ce..b35d964e573 100644 --- a/rs/moq-net/src/lib.rs +++ b/rs/moq-net/src/lib.rs @@ -100,6 +100,8 @@ mod lite; mod model; pub mod path; #[cfg(feature = "async")] +mod pressure; +#[cfg(feature = "async")] mod priority; #[cfg(feature = "async")] mod recv; diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index f47528eec5d..ceb1ab5072d 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -212,8 +212,9 @@ pub struct PendingRecv; impl poll::RecvStream for PendingRecv { type Error = SinkError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { Poll::Pending } @@ -259,8 +260,9 @@ pub struct DeadRecv(ResetError); impl poll::RecvStream for DeadRecv { type Error = ResetError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { Poll::Ready(Err(self.0)) } @@ -378,6 +380,10 @@ impl poll::Session for DeadStreamSession { Poll::Pending } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { SinkStats::default() } @@ -566,6 +572,10 @@ impl poll::Session for SinkSession { Poll::Pending } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { *self.stats.lock().unwrap() } @@ -624,8 +634,9 @@ pub enum Close { impl poll::RecvStream for ScriptedRecv { type Error = SinkError; + type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let take = { let mut script = self.script.lock().unwrap(); if script.is_empty() { @@ -644,7 +655,7 @@ impl poll::RecvStream for ScriptedRecv { Some(Close::Reset) => Poll::Ready(Err(SinkError)), None => Poll::Pending, }, - take => Poll::Ready(Ok(Some(take))), + take => Poll::Ready(Ok(Some(crate::transport::Unheld(take as u64)))), } } @@ -892,6 +903,10 @@ impl poll::Session for ScriptedSession { Poll::Pending } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { self.stats } diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs new file mode 100644 index 00000000000..9597c0557a1 --- /dev/null +++ b/rs/moq-net/src/pressure.rs @@ -0,0 +1,236 @@ +//! Receive-side recovery for when held connection credit fills the allowance. +//! +//! A [`Reader`](crate::coding::Reader) tracked here holds the connection credit for its +//! buffered bytes and its incomplete unit, so incomplete messages are bounded by the +//! transport's receive allowance. When the transport reports that the allowance is +//! spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), +//! nothing can arrive to complete any unit, so the [`Monitor`] cancels the least useful +//! holder with STOP_SENDING and the session continues with the credit it returns. +//! +//! Victims, first to last: +//! - Media, from the broadcast holding the most credit, lowest priority first. Within a +//! broadcast the subscription's priority and group order rank the streams, as on the +//! send side; raw priorities of unrelated broadcasts are never compared. +//! - Unidentified streams, whose header has yet to name a subscription, so a header +//! gets the chance to identify itself before it is cancelled. +//! - Control streams. Cancelling one may end the session under its protocol, which the +//! stream's handler reports like any other error. +//! +//! Ties go to the larger holder, then to the newest stream. There is no protected class: +//! the allowance fits the largest supported unit, so cancelling every other holder +//! always leaves room for one to complete. + +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + task::{Context, Poll, Waker}, +}; + +use slab::Slab; + +use crate::{PathOwned, priority::Priority}; + +/// How useful a tracked stream is, which orders the victims. +#[derive(Clone, Debug)] +pub(crate) enum Rank { + /// A stream whose header has not identified it yet. + Unidentified, + /// A media stream of one broadcast, ranked within it. + Media { + /// The broadcast, the domain inside which priorities compare. + broadcast: PathOwned, + /// The stream's rank within its broadcast. + priority: Priority, + }, + /// A control stream. + Control, +} + +/// One session's tracked receivers. +#[derive(Clone, Default)] +pub(crate) struct Pressure { + state: kio::Lock, +} + +#[derive(Default)] +struct State { + entries: Slab, + /// Registration order, the last tiebreak: the newest stream goes first. + next: u64, +} + +struct Entry { + slot: Arc, + rank: Rank, + seq: u64, +} + +/// What a [`Ticket`] shares with the registry without taking its lock. +#[derive(Default)] +struct Slot { + /// Bytes of credit the reader holds. + held: AtomicU64, + /// The monitor chose this stream; its reader stops and lets go. + evicted: AtomicBool, + /// The reader's task, woken on eviction. + waker: Mutex>, +} + +impl Pressure { + /// Track a new receiver at `rank`. + pub fn track(&self, rank: Rank) -> Ticket { + let slot = Arc::new(Slot::default()); + let mut state = self.state.lock(); + let seq = state.next; + state.next += 1; + let id = state.entries.insert(Entry { + slot: slot.clone(), + rank, + seq, + }); + Ticket { + pressure: self.clone(), + id, + slot, + } + } + + /// Cancel the least useful holder of credit, returning whether there was one. + pub fn evict(&self) -> bool { + let state = self.state.lock(); + let Some(id) = victim(&state.entries) else { + return false; + }; + let slot = &state.entries[id].slot; + slot.evicted.store(true, Ordering::Release); + let waker = slot.waker.lock().unwrap_or_else(|err| err.into_inner()).take(); + tracing::debug!(rank = ?state.entries[id].rank, held = slot.held.load(Ordering::Relaxed), "receive credit exhausted, cancelling a stream"); + drop(state); + if let Some(waker) = waker { + waker.wake(); + } + true + } + + /// The bytes every tracked receiver holds, for tests. + #[cfg(test)] + pub fn held(&self) -> u64 { + let state = self.state.lock(); + state.entries.iter().map(|(_, e)| e.slot.held.load(Ordering::Relaxed)).sum() + } +} + +/// Pick the victim: see the module docs for the order. +fn victim(entries: &Slab) -> Option { + let held = |entry: &Entry| entry.slot.held.load(Ordering::Relaxed); + let holding = || { + entries + .iter() + .filter(|(_, entry)| held(entry) > 0 && !entry.slot.evicted.load(Ordering::Relaxed)) + }; + + // Media: the broadcast holding the most, then its least useful stream. + let mut domains: Vec<(&PathOwned, u64)> = Vec::new(); + for (_, entry) in holding() { + if let Rank::Media { broadcast, .. } = &entry.rank { + match domains.iter_mut().find(|(path, _)| *path == broadcast) { + Some((_, total)) => *total += held(entry), + None => domains.push((broadcast, held(entry))), + } + } + } + // The larger holder, then the lower path, so the choice never depends on order. + let domain = domains + .into_iter() + .max_by(|(a, a_held), (b, b_held)| a_held.cmp(b_held).then_with(|| b.as_str().cmp(a.as_str()))); + if let Some((domain, _)) = domain { + return holding() + .filter_map(|(id, entry)| match &entry.rank { + Rank::Media { broadcast, priority } if broadcast == domain => Some((id, priority, entry)), + _ => None, + }) + // `Priority` sorts the most urgent first, so the greatest is the least useful. + .max_by(|(_, a, a_entry), (_, b, b_entry)| { + a.cmp(b) + .then_with(|| held(a_entry).cmp(&held(b_entry))) + .then_with(|| a_entry.seq.cmp(&b_entry.seq)) + }) + .map(|(id, ..)| id); + } + + let largest = |control: bool| { + holding() + .filter(|(_, entry)| matches!(entry.rank, Rank::Control) == control) + .max_by(|(_, a), (_, b)| held(a).cmp(&held(b)).then_with(|| a.seq.cmp(&b.seq))) + .map(|(id, _)| id) + }; + largest(false).or_else(|| largest(true)) +} + +/// A tracked receiver's registration, removed on drop. +pub(crate) struct Ticket { + pressure: Pressure, + id: usize, + slot: Arc, +} + +impl Ticket { + /// Re-rank the stream, once its header identifies it. + pub fn rank(&self, rank: Rank) { + self.pressure.state.lock().entries[self.id].rank = rank; + } + + /// Record how many bytes of credit the reader holds. + pub fn held(&self, bytes: u64) { + self.slot.held.store(bytes, Ordering::Relaxed); + } + + /// Whether the monitor chose this stream, registering `cx` to hear when it does. + pub fn poll_evicted(&self, cx: &Context<'_>) -> bool { + if self.slot.evicted.load(Ordering::Acquire) { + return true; + } + let mut waker = self.slot.waker.lock().unwrap_or_else(|err| err.into_inner()); + match waker.as_mut() { + Some(waker) => waker.clone_from(cx.waker()), + None => *waker = Some(cx.waker().clone()), + } + drop(waker); + // Checked again: an eviction between the load and the store took no waker. + self.slot.evicted.load(Ordering::Acquire) + } +} + +impl Drop for Ticket { + fn drop(&mut self) { + self.pressure.state.lock().entries.remove(self.id); + } +} + +/// Cancels a stream each time the session's receive allowance fills. +pub(crate) struct Monitor { + session: S, + pressure: Pressure, +} + +impl Monitor { + /// Watch `session`, cancelling streams tracked by `pressure`. + pub fn new(session: S, pressure: Pressure) -> Self { + Self { session, pressure } + } + + /// Run until dropped. Each fill cancels one stream: the next fill can only come after + /// it let go of its credit and the peer spent what that freed. + pub fn poll(&mut self, cx: &mut Context<'_>) -> Poll<()> { + while self.session.poll_credit_exhausted(cx).is_ready() { + if !self.pressure.evict() { + // Untracked readers or transport read-ahead hold it all; whoever holds it + // lets go on their own, and a later fill is reported again. + tracing::warn!("receive credit exhausted with no tracked holder to cancel"); + } + } + Poll::Pending + } +} diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 371e347a161..531c6668cf8 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1093,6 +1093,10 @@ mod tests { fn poll_closed(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll { std::task::Poll::Pending } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl crate::transport::Stats { crate::transport::StatsUnavailable } @@ -1125,11 +1129,12 @@ mod tests { } impl crate::transport::poll::RecvStream for FakeRecv { type Error = FakeError; + type Credit = crate::transport::Unheld; fn poll_read( &mut self, _cx: &mut std::task::Context<'_>, dst: &mut [u8], - ) -> std::task::Poll, Self::Error>> { + ) -> std::task::Poll, Self::Error>> { if self.data.is_empty() { return std::task::Poll::Ready(Ok(None)); } @@ -1137,7 +1142,7 @@ mod tests { for slot in dst.iter_mut().take(size) { *slot = self.data.pop_front().unwrap(); } - std::task::Poll::Ready(Ok(Some(size))) + std::task::Poll::Ready(Ok(Some(crate::transport::Unheld(size as u64)))) } fn stop(&mut self, code: u32) { self.stops.lock().unwrap().push(code); diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 409ed432e7b..cdbf4ec1419 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -97,6 +97,51 @@ pub trait Stats { pub struct StatsUnavailable; impl Stats for StatsUnavailable {} +/// Connection flow-control credit for bytes read from a [`poll::RecvStream`]. +/// +/// A transport that bounds receive memory keeps the connection window those bytes +/// occupy spent until their credit is dropped, so a receiver holds it until the unit +/// the bytes belong to is complete. Stream credit is unaffected and advances on reads. +/// Dropping a credit returns it, even after its stream is gone. +pub trait Credit: 'static { + /// The number of bytes this credit covers. + fn len(&self) -> u64; + + /// Whether this credit covers no bytes. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Split off the credit for the first `bytes`, keeping the rest. + /// + /// Panics if `bytes` exceeds [`len`](Self::len). + fn split(&mut self, bytes: u64) -> Self; + + /// Take over the bytes covered by `other`, read from the same session. + fn merge(&mut self, other: Self); +} + +/// The credit of a transport that returns connection credit as soon as bytes are read: +/// only a byte count. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Unheld(pub u64); + +impl Credit for Unheld { + fn len(&self) -> u64 { + self.0 + } + + fn split(&mut self, bytes: u64) -> Self { + assert!(bytes <= self.0, "split {bytes} of {} bytes", self.0); + self.0 -= bytes; + Self(bytes) + } + + fn merge(&mut self, other: Self) { + self.0 += other.0; + } +} + /// A transport failure with optional session and stream application codes. /// /// Session codes and stream codes belong to separate registries. @@ -118,7 +163,7 @@ pub mod poll { pub type BiStreams = (::SendStream, ::RecvStream); use std::task::{Context, Poll, ready}; - use super::{Error, MaybeSend, MaybeSync, Stats}; + use super::{Credit, Error, MaybeSend, MaybeSync, Stats}; use bytes::BytesMut; use bytes::{Buf, BufMut, Bytes}; @@ -201,6 +246,17 @@ pub mod poll { /// Poll until the connection is closed by either side. fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll; + /// Poll until held [`Credit`] fills the receive allowance: the peer may send + /// nothing more and every byte it sent is held, so no stream progresses until + /// some credit is dropped. + /// + /// Resolves once each time it starts holding, then pends until it holds again + /// after some credit was dropped, so a receiver evicts once per stall. A transport whose reads + /// return their credit (an [`Unheld`](super::Unheld) credit) never fills it, so + /// it always pends. Required rather than defaulted: a transport that holds + /// credit and forgets to report this parks its receiver forever. + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()>; + /// Return connection-level statistics. /// /// Return [`super::StatsUnavailable`] if the transport does not track them. Required @@ -295,12 +351,16 @@ pub mod poll { /// boxed drivers (`Send` boxes on native): what the moq-transport path /// requires until it too becomes named machines. Implemented automatically. pub trait Boxable: - Session + MaybeSend + MaybeSync + Session, Error: MaybeSend> + + MaybeSend + + MaybeSync { } impl Boxable for S where - S: Session + MaybeSend + MaybeSync + S: Session, Error: MaybeSend> + + MaybeSend + + MaybeSync { } @@ -452,19 +512,25 @@ pub mod poll { } /// An incoming stream with reads, stop codes, and closure notification. + /// + /// Every read returns the [`Credit`] for exactly the bytes it read. A caller that + /// does not bound its receive memory drops it straight away. pub trait RecvStream: 'static { /// The error type for every operation on this stream. type Error: Error; + /// The connection credit each read returns. + type Credit: Credit; + /// Poll to read some data into the provided slice. /// - /// Returns the number of bytes read, or `None` once the peer has finished the - /// stream. An empty `dst` reads nothing and returns `Some(0)`: asking for no - /// bytes is not end of stream. - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>>; + /// Returns the credit for the bytes read ([`Credit::len`] counts them), or + /// `None` once the peer has finished the stream. An empty `dst` reads nothing + /// and returns an empty credit: asking for no bytes is not end of stream. + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>>; - /// Poll to read some data into the provided buffer, advancing it by the number - /// of bytes read. + /// Poll to read some data into the provided buffer, advancing it by the bytes + /// read, which the returned credit covers. /// /// Override this to avoid a copy when the underlying transport already owns the /// bytes as a [`Bytes`], which can be handed to [`BufMut::put`] directly. @@ -472,7 +538,7 @@ pub mod poll { &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { + ) -> Poll, Self::Error>> { // Cap the slice: it is zeroed on every poll, Pending included, and a // read may be partial anyway. let len = buf.chunk_mut().len().min(64 * 1024); @@ -480,7 +546,7 @@ pub mod poll { // A destination with no room is not a closed stream. Collapsing the two // would turn "buffer full" into "stream ended", which reads as truncation. if len == 0 { - return Poll::Ready(Ok(Some(0))); + return self.poll_read(cx, &mut []); } // `poll_read` is safe and may inspect its input, so initialize spare @@ -491,26 +557,32 @@ pub mod poll { std::slice::from_raw_parts_mut(chunk.as_mut_ptr(), len) }; - let size = match ready!(self.poll_read(cx, dst))? { - Some(size) if size > 0 => size, - Some(_) => return Poll::Ready(Ok(Some(0))), - None => return Poll::Ready(Ok(None)), + let Some(credit) = ready!(self.poll_read(cx, dst))? else { + return Poll::Ready(Ok(None)); }; + let size = credit.len() as usize; assert!(size <= len, "transport read exceeded its destination"); unsafe { buf.advance_mut(size) }; - Poll::Ready(Ok(Some(size))) + Poll::Ready(Ok(Some(credit))) } - /// Poll for the next chunk of data, up to `max` bytes. + /// Poll for the next chunk of data, up to `max` bytes, with its credit. /// /// Override this when the transport can hand over a [`Bytes`] it already owns; /// the default allocates and copies. - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { + #[allow(clippy::type_complexity)] + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { // As in `poll_read_buf`: asking for nothing is not end of stream. if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); + return self + .poll_read(cx, &mut []) + .map_ok(|read| read.map(|credit| (Bytes::new(), credit))); } // Don't allocate too much. Override this to avoid the copy, or to use a @@ -518,16 +590,15 @@ pub mod poll { let capacity = max.min(8 * 1024); let mut buf = BytesMut::zeroed(capacity); - let size = match ready!(self.poll_read(cx, &mut buf))? { - Some(size) if size > 0 => size, - Some(_) => return Poll::Ready(Ok(Some(Bytes::new()))), - None => return Poll::Ready(Ok(None)), + let Some(credit) = ready!(self.poll_read(cx, &mut buf))? else { + return Poll::Ready(Ok(None)); }; + let size = credit.len() as usize; assert!(size <= capacity, "transport read exceeded its destination"); buf.truncate(size); - Poll::Ready(Ok(Some(buf.freeze()))) + Poll::Ready(Ok(Some((buf.freeze(), credit)))) } /// Send a `STOP_SENDING` QUIC code, informing the peer that no more data will be @@ -545,17 +616,17 @@ pub mod poll { /// - We received a FIN via [`SendStream::finish`] and read all data. fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll>; - /// Read some bytes into the slice, or `None` once the stream is finished. + /// Read some bytes into the slice, returning their credit, or `None` once finished. fn read<'a>(&'a mut self, dst: &'a mut [u8]) -> Read<'a, Self> { Read { stream: self, dst } } - /// Read some bytes into the buffer, advancing it, or `None` once finished. + /// Read some bytes into the buffer, advancing it and returning their credit, or `None`. fn read_buf<'a, B: BufMut>(&'a mut self, buf: &'a mut B) -> ReadBuf<'a, Self, B> { ReadBuf { stream: self, buf } } - /// Read the next chunk of data, up to `max` bytes, or `None` once finished. + /// Read the next chunk, up to `max` bytes, with its credit, or `None` once finished. fn read_chunk(&mut self, max: usize) -> ReadChunk<'_, Self> { ReadChunk { stream: self, max } } @@ -573,7 +644,7 @@ pub mod poll { } impl Future for Read<'_, S> { - type Output = Result, S::Error>; + type Output = Result, S::Error>; fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = &mut *self; @@ -588,7 +659,7 @@ pub mod poll { } impl Future for ReadBuf<'_, S, B> { - type Output = Result, S::Error>; + type Output = Result, S::Error>; fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = &mut *self; @@ -603,7 +674,7 @@ pub mod poll { } impl Future for ReadChunk<'_, S> { - type Output = Result, S::Error>; + type Output = Result, S::Error>; fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = &mut *self; @@ -614,18 +685,19 @@ pub mod poll { #[cfg(test)] mod tests { - use super::poll::RecvStream as _; + use super::{Unheld, poll::RecvStream as _}; use bytes::{Bytes, BytesMut}; use std::task::{Context, Poll}; struct Read; impl super::poll::RecvStream for Read { type Error = crate::Error; - fn poll_read(&mut self, _: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + type Credit = Unheld; + fn poll_read(&mut self, _: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { // A safe transport may read its destination before overwriting it. assert!(dst.iter().all(|byte| *byte == 0)); let size = dst.len().min(2); dst[..size].copy_from_slice(&b"hi"[..size]); - Poll::Ready(Ok(Some(size))) + Poll::Ready(Ok(Some(Unheld(size as u64)))) } fn stop(&mut self, _: u32) {} fn poll_closed(&mut self, _: &mut Context<'_>) -> Poll> { @@ -639,21 +711,23 @@ mod tests { let mut buf = BytesMut::with_capacity(8); assert!(matches!( read.poll_read_buf(&mut cx, &mut buf), - Poll::Ready(Ok(Some(2))) + Poll::Ready(Ok(Some(Unheld(2)))) )); assert_eq!(buf, b"hi"[..]); - let Poll::Ready(Ok(Some(chunk))) = read.poll_read_chunk(&mut cx, 1) else { + let Poll::Ready(Ok(Some((chunk, credit)))) = read.poll_read_chunk(&mut cx, 1) else { panic!("missing chunk") }; assert_eq!(chunk, Bytes::from_static(b"h")); - let Poll::Ready(Ok(Some(chunk))) = read.poll_read_chunk(&mut cx, 0) else { + assert_eq!(credit, Unheld(1)); + let Poll::Ready(Ok(Some((chunk, credit)))) = read.poll_read_chunk(&mut cx, 0) else { panic!("missing empty chunk") }; assert!(chunk.is_empty()); + assert_eq!(credit, Unheld(0)); let mut empty: &mut [u8] = &mut []; assert!(matches!( read.poll_read_buf(&mut cx, &mut empty), - Poll::Ready(Ok(Some(0))) + Poll::Ready(Ok(Some(Unheld(0)))) )); } } diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 1bccc9948bb..4d4dc18d574 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -6,7 +6,13 @@ //! //! Each stream's channel is bounded by [`Config::stream_buffer`] queued bytes, and a write //! pends while it is full, so a slow reader throttles only its own stream. A write is -//! never split: one accepted while the channel has room may overshoot the bound. +//! never split for it: one accepted while the channel has room may overshoot the bound. +//! +//! [`Config::receive_window`] optionally bounds each side's incoming bytes across every +//! stream, like QUIC's connection flow control: queued bytes and bytes whose read +//! [`Credit`] is still held count against it, a write takes only what fits, and +//! [`poll_credit_exhausted`](poll::Session::poll_credit_exhausted) reports each time held +//! credit fills it. //! //! [`finish`](poll::SendStream::finish) closes the channel cleanly once its messages are //! read. A [`reset`](poll::SendStream::reset) or [`stop`](poll::RecvStream::stop) closes it @@ -42,6 +48,9 @@ pub struct Config { pub stream_buffer: usize, /// The application protocol both sides report, standing in for a negotiated ALPN. pub protocol: Option, + /// Incoming bytes, queued or held by an undropped [`Credit`], each side accepts across + /// all of its streams before the peer's writes pend. `None` is unbounded. + pub receive_window: Option, } impl Default for Config { @@ -51,15 +60,23 @@ impl Default for Config { Self { stream_buffer: 1024 * 1024, protocol: None, + receive_window: None, } } } /// Create two connected sessions: streams one opens, the other accepts. pub fn pair(config: Config) -> (Session, Session) { + let window = || { + kio::Shared::new(Window { + limit: config.receive_window, + ..Default::default() + }) + }; let conn = Arc::new(Conn { close: Default::default(), incoming: [Incoming::default(), Incoming::default()], + windows: [window(), window()], config, }); @@ -72,6 +89,7 @@ pub fn pair(config: Config) -> (Session, Session) { accept_bi: Default::default(), datagram: Default::default(), closed: Default::default(), + exhausted: Default::default(), }; (side(0), side(1)) @@ -144,6 +162,95 @@ struct Conn { close: Close, /// What each side has yet to accept. incoming: [Incoming; 2], + /// Each side's receive allowance. + windows: [kio::Shared; 2], +} + +/// One side's receive allowance, shared by every stream it reads. +#[derive(Default)] +struct Window { + /// The most bytes `queued + held` may reach, if bounded. + limit: Option, + /// Bytes written to this side's streams and not yet read. + queued: usize, + /// Bytes read whose credit is not dropped yet. + held: usize, + /// Held credit fills the limit. + exhausted: bool, + /// A fill not yet reported to [`poll::Session::poll_credit_exhausted`]. + unreported: bool, +} + +impl Window { + /// How many more bytes a writer may queue. + fn room(&self) -> usize { + match self.limit { + Some(limit) => limit.saturating_sub(self.queued + self.held), + None => usize::MAX, + } + } + + /// Latch a fill: everything the peer may send is held. + fn check(&mut self) { + let full = self.limit.is_some_and(|limit| self.held > 0 && self.held >= limit); + if full && !self.exhausted { + self.unreported = true; + } + self.exhausted = full; + } +} + +/// The receive allowance taken by bytes read from an in-memory stream, returned on drop. +pub struct Credit { + window: kio::Shared, + bytes: usize, +} + +impl Credit { + fn new(window: &kio::Shared, bytes: usize) -> Self { + Self { + window: window.clone(), + bytes, + } + } +} + +impl super::Credit for Credit { + fn len(&self) -> u64 { + self.bytes as u64 + } + + fn split(&mut self, bytes: u64) -> Self { + let bytes = bytes as usize; + assert!(bytes <= self.bytes, "split {bytes} of {} bytes", self.bytes); + self.bytes -= bytes; + Self::new(&self.window, bytes) + } + + fn merge(&mut self, mut other: Self) { + assert!( + self.window.same_channel(&other.window), + "merged credit from another session" + ); + self.bytes += std::mem::take(&mut other.bytes); + } +} + +impl Drop for Credit { + fn drop(&mut self) { + if self.bytes == 0 { + return; + } + let mut window = self.window.lock(); + window.held -= self.bytes; + window.check(); + } +} + +impl std::fmt::Debug for Credit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credit").field("len", &self.bytes).finish() + } } impl Conn { @@ -154,10 +261,13 @@ impl Conn { } } - fn stream(&self) -> (SendStream, RecvStream) { + /// A stream that side `reader` reads. + fn stream(&self, reader: usize) -> (SendStream, RecvStream) { let channel = kio::Shared::::default(); + let window = self.windows[reader].clone(); let send = SendStream { channel: channel.clone(), + window: window.clone(), close: self.close.clone(), buffer: self.config.stream_buffer, done: false, @@ -165,6 +275,7 @@ impl Conn { }; let recv = RecvStream { channel, + window, close: self.close.clone(), stopped: false, park: Default::default(), @@ -238,6 +349,7 @@ pub struct Session { accept_bi: kio::Park, datagram: kio::Park, closed: kio::Park, + exhausted: kio::Park, } impl Session { @@ -283,13 +395,13 @@ impl poll::Session for Session { } fn poll_open_uni(&mut self, _cx: &mut Context<'_>) -> Poll> { - let (send, recv) = self.conn().stream(); + let (send, recv) = self.conn().stream(1 - self.side.local); Poll::Ready(self.open(&self.side.outgoing().uni, recv).map(|()| send)) } fn poll_open_bi(&mut self, _cx: &mut Context<'_>) -> Poll> { - let (send, peer_recv) = self.conn().stream(); - let (peer_send, recv) = self.conn().stream(); + let (send, peer_recv) = self.conn().stream(1 - self.side.local); + let (peer_send, recv) = self.conn().stream(self.side.local); let opened = self.open(&self.side.outgoing().bi, (peer_send, peer_recv)); Poll::Ready(opened.map(|()| (send, recv))) } @@ -331,6 +443,17 @@ impl poll::Session for Session { poll_close(&self.side.conn.close, self.closed.hold(cx)) } + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + let waiter = self.exhausted.hold(cx); + let window = &self.side.conn.windows[self.side.local]; + let mut window = ready!(window.poll(waiter, |window| match window.unreported { + true => Poll::Ready(()), + false => Poll::Pending, + })); + window.unreported = false; + Poll::Ready(()) + } + fn stats(&self) -> impl super::Stats { StatsUnavailable } @@ -357,13 +480,16 @@ enum End { } impl Channel { - /// Close with a reset or stop code and drop what is queued. An earlier end stands, - /// unless it was a finish the reader has not fully read. - fn abort(&mut self, end: End) { + /// Close with a reset or stop code and drop what is queued, returning its room to + /// `window`. An earlier end stands, unless it was a finish the reader has not fully read. + fn abort(&mut self, end: End, window: &kio::Shared) { if self.end.is_none() || !self.messages.is_empty() { self.end = Some(end); } self.messages.clear(); + if self.queued > 0 { + window.lock().queued -= self.queued; + } self.queued = 0; } @@ -381,6 +507,8 @@ impl Channel { /// The writing half of an in-memory stream. pub struct SendStream { channel: kio::Shared, + /// The reading side's receive allowance. + window: kio::Shared, close: Close, buffer: usize, /// This side finished or reset the stream. @@ -396,7 +524,8 @@ impl poll::SendStream for SendStream { self.poll_write_buf(cx, &mut buf) } - /// Queue all of `buf` as one message, waiting while the channel is full. + /// Queue as much of `buf` as the receive window allows as one message, waiting while + /// the channel or the window is full. fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { if self.done { return Poll::Ready(Err(Error::Closed)); @@ -420,9 +549,20 @@ impl poll::SendStream for SendStream { if let Some(End::Stop(code)) = channel.end { return Poll::Ready(Err(Error::Stopped(code))); } + // A reset stream's bytes are dropped, never read, so they take no room. Otherwise + // wait for some; the channel lock keeps the stream from ending meanwhile. + let mut window = match channel.end { + Some(_) => self.window.lock(), + None => ready!(self.window.poll(waiter, |window| match window.room() > 0 { + true => Poll::Ready(()), + false => Poll::Pending, + })), + }; // Take ownership rather than copy: free for a `Bytes` source. - let size = buf.remaining(); + let size = buf.remaining().min(window.room()); + window.queued += size; + drop(window); channel.messages.push_back(buf.copy_to_bytes(size)); channel.queued += size; @@ -456,7 +596,7 @@ impl poll::SendStream for SendStream { // close is checked under the channel lock so a concurrent close can't land between. let mut channel = self.channel.lock(); if closed(&self.close).is_none() { - channel.abort(End::Reset(code)); + channel.abort(End::Reset(code), &self.window); } } @@ -484,6 +624,8 @@ impl Drop for SendStream { /// The reading half of an in-memory stream. pub struct RecvStream { channel: kio::Shared, + /// This side's receive allowance. + window: kio::Shared, close: Close, /// This side stopped the stream. stopped: bool, @@ -492,33 +634,33 @@ pub struct RecvStream { impl poll::RecvStream for RecvStream { type Error = Error; + type Credit = Credit; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { - let Some(chunk) = ready!(self.poll_read_chunk(cx, dst.len()))? else { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { + let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, dst.len()))? else { return Poll::Ready(Ok(None)); }; dst[..chunk.len()].copy_from_slice(&chunk); - Poll::Ready(Ok(Some(chunk.len()))) + Poll::Ready(Ok(Some(credit))) } - fn poll_read_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll, Error>> { + fn poll_read_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll, Error>> { // The same cap the default uses, but without zeroing a destination first. let max = buf.remaining_mut().min(64 * 1024); - let Some(chunk) = ready!(self.poll_read_chunk(cx, max))? else { + let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, max))? else { return Poll::Ready(Ok(None)); }; - let size = chunk.len(); buf.put(chunk); - Poll::Ready(Ok(Some(size))) + Poll::Ready(Ok(Some(credit))) } /// Pop the next message, or split off its first `max` bytes. - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { + fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { if self.stopped { return Poll::Ready(Err(Error::Closed)); } if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); + return Poll::Ready(Ok(Some((Bytes::new(), Credit::new(&self.window, 0))))); } let waiter = self.park.hold(cx); @@ -547,7 +689,15 @@ impl poll::RecvStream for RecvStream { }; channel.queued -= chunk.len(); - Poll::Ready(Ok(Some(chunk))) + // Read bytes move from queued to held, still spending the window. + let mut window = self.window.lock(); + window.queued -= chunk.len(); + window.held += chunk.len(); + window.check(); + drop(window); + + let credit = Credit::new(&self.window, chunk.len()); + Poll::Ready(Ok(Some((chunk, credit)))) } fn stop(&mut self, code: u32) { @@ -558,10 +708,11 @@ impl poll::RecvStream for RecvStream { // as in `reset`. let mut channel = self.channel.lock(); if closed(&self.close).is_none() { - channel.abort(End::Stop(code)); + channel.abort(End::Stop(code), &self.window); } } + /// Ready once the writer finished or reset the stream, or this side stopped it. fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { if self.stopped { @@ -638,10 +789,13 @@ mod tests { send.poll_write_buf(&mut noop(), &mut chunk.clone()) } + /// Read a chunk, returning its credit straight away. fn read(recv: &mut RecvStream, max: usize) -> Poll, Error>> { recv.poll_read_chunk(&mut noop(), max) + .map_ok(|chunk| chunk.map(|(chunk, _)| chunk)) } + fn text(chunk: Poll, Error>>) -> Bytes { ready(chunk).unwrap().unwrap() } diff --git a/rs/moq-net/tests/subscribe_tracks.rs b/rs/moq-net/tests/subscribe_tracks.rs index 892faba5b9b..da39d8f20e4 100644 --- a/rs/moq-net/tests/subscribe_tracks.rs +++ b/rs/moq-net/tests/subscribe_tracks.rs @@ -116,7 +116,7 @@ async fn subscribe_tracks(pair: &MockPair) -> Vec { .expect("reply timed out") .expect("read") { - reply.extend_from_slice(&buf[..n]); + reply.extend_from_slice(&buf[..moq_net::transport::Credit::len(&n) as usize]); } reply } diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index da22d32c665..0f7bb67745b 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -23,7 +23,7 @@ use std::{ use bytes::{Buf, BufMut, Bytes}; use moq_net::{ time::Instant, - transport::{memory, poll}, + transport::{Credit as _, memory, poll}, }; use super::harness::now; @@ -271,7 +271,7 @@ impl Drop for MockSendStream { // ── RecvStream ────────────────────────────────────────────────────── /// What the transport handed over: a chunk, the FIN, or an error. -type Event = Result, memory::Error>; +type Event = Result, memory::Error>; /// A receive stream that delivers each event once its arrival has passed. pub struct MockRecvStream { @@ -283,8 +283,8 @@ pub struct MockRecvStream { flight: Option<(Instant, Event)>, /// The arrival a wake is already scheduled for, so each flight schedules one. landing: Option, - /// The rest of a chunk a smaller read split. - buf: Bytes, + /// The rest of a chunk a smaller read split, with its credit. + buf: Option<(Bytes, memory::Credit)>, /// Data chunks handed to the reader so far. chunks: usize, park: kio::Park, @@ -298,7 +298,7 @@ impl MockRecvStream { conn, flight: None, landing: None, - buf: Bytes::new(), + buf: None, chunks: 0, park: Default::default(), } @@ -371,43 +371,50 @@ impl MockRecvStream { impl poll::RecvStream for MockRecvStream { type Error = memory::Error; + type Credit = memory::Credit; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - let Some(chunk) = ready!(self.poll_read_chunk(cx, dst.len()))? else { + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, dst.len()))? else { return Poll::Ready(Ok(None)); }; dst[..chunk.len()].copy_from_slice(&chunk); - Poll::Ready(Ok(Some(chunk.len()))) + Poll::Ready(Ok(Some(credit))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { + ) -> Poll, Self::Error>> { let max = buf.remaining_mut().min(64 * 1024); - let Some(chunk) = ready!(self.poll_read_chunk(cx, max))? else { + let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, max))? else { return Poll::Ready(Ok(None)); }; - let size = chunk.len(); buf.put(chunk); - Poll::Ready(Ok(Some(size))) + Poll::Ready(Ok(Some(credit))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); - } - if self.buf.is_empty() { + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + if self.buf.is_none() { match ready!(self.poll_event(cx))? { - Some(chunk) => self.buf = chunk, + Some(chunk) => self.buf = Some(chunk), None => return Poll::Ready(Ok(None)), } } - let size = max.min(self.buf.len()); - Poll::Ready(Ok(Some(self.buf.split_to(size)))) + let (buf, credit) = self.buf.as_mut().expect("filled above"); + let size = max.min(buf.len()); + let chunk = (buf.split_to(size), credit.split(size as u64)); + if buf.is_empty() { + self.buf = None; + } + Poll::Ready(Ok(Some(chunk))) } + fn stop(&mut self, code: u32) { self.stop_after_latency(code); } @@ -612,6 +619,10 @@ impl poll::Session for MockSession { self.inner.poll_closed(cx) } + fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + fn stats(&self) -> impl moq_net::transport::Stats { assert!( !self.side.panic_on_stats.load(Ordering::Relaxed), From 0d565cf56c00221b00df2f0b6d685c94064beca5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:16:35 -0700 Subject: [PATCH 21/51] wip: Unheld credit for the WebTransport adapters Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/transport/adapter.rs | 29 ++++++++++++++++++++------- rs/moq-tokio/src/transport/owned.rs | 27 +++++++++++++++++++------ rs/moq-uring/src/transport/adapter.rs | 29 ++++++++++++++++++++------- rs/moq-wasm/src/transport/adapter.rs | 29 ++++++++++++++++++++------- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 0ebe75b7b14..4b176d13e34 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -98,6 +98,9 @@ impl + Clone + 'st fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(&mut self.0, cx).map(Error) } + fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(&self.0)) } @@ -127,20 +130,32 @@ impl poll::SendStream for SendStream { } /// An incoming WebTransport stream adapted to moq-net. pub struct RecvStream(S); +// The WebTransport backend returns connection credit as it reads, so there is none to hold. impl poll::RecvStream for RecvStream { type Error = Error; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(&mut self.0, cx, dst).map_err(Error) + type Credit = moq::Unheld; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + backend::RecvStream::poll_read(&mut self.0, cx, dst) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(&mut self.0, cx, buf).map_err(Error) + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_buf(&mut self.0, cx, buf) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(&mut self.0, cx, max).map_err(Error) + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) + .map_err(Error) + .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) @@ -295,7 +310,7 @@ mod tests { let Poll::Ready(Ok(Some(chunk))) = poll::RecvStream::poll_read_chunk(&mut recv, &mut cx, 2) else { panic!("read failed") }; - assert_eq!(chunk, b"hi"[..]); + assert_eq!(chunk.0, b"hi"[..]); poll::RecvStream::stop(&mut recv, 13); assert_eq!(state.get(), 13); let Poll::Ready(Err(error)) = poll::SendStream::poll_closed(&mut send, &mut cx) else { diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 44d98ef0d7e..225574189e8 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -81,6 +81,9 @@ impl> poll::Session for fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(self, cx).map(Error) } + fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(self)) } @@ -106,20 +109,32 @@ impl poll::SendStream for super::SendStream { backend::SendStream::poll_closed(self, cx).map_err(Error) } } +// The WebTransport backend returns connection credit as it reads, so there is none to hold. impl poll::RecvStream for super::RecvStream { type Error = Error; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(self, cx, dst).map_err(Error) + type Credit = moq::Unheld; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + backend::RecvStream::poll_read(self, cx, dst) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(self, cx, buf).map_err(Error) + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_buf(self, cx, buf) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(self, cx, max).map_err(Error) + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_chunk(self, cx, max) + .map_err(Error) + .map_ok(|chunk| chunk.map(|b| (b.clone(), moq::Unheld(b.len() as u64)))) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(self, code) diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 0a751d76718..6cfc014254e 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -98,6 +98,9 @@ impl + Clone + 'st fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(&mut self.0, cx).map(Error) } + fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(&self.0)) } @@ -127,20 +130,32 @@ impl poll::SendStream for SendStream { } /// An incoming WebTransport stream adapted to moq-net. pub struct RecvStream(S); +// The WebTransport backend returns connection credit as it reads, so there is none to hold. impl poll::RecvStream for RecvStream { type Error = Error; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(&mut self.0, cx, dst).map_err(Error) + type Credit = moq::Unheld; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + backend::RecvStream::poll_read(&mut self.0, cx, dst) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(&mut self.0, cx, buf).map_err(Error) + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_buf(&mut self.0, cx, buf) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(&mut self.0, cx, max).map_err(Error) + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) + .map_err(Error) + .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) @@ -295,7 +310,7 @@ mod tests { let Poll::Ready(Ok(Some(chunk))) = poll::RecvStream::poll_read_chunk(&mut recv, &mut cx, 2) else { panic!("read failed") }; - assert_eq!(chunk, b"hi"[..]); + assert_eq!(chunk.0, b"hi"[..]); poll::RecvStream::stop(&mut recv, 13); assert_eq!(state.get(), 13); let Poll::Ready(Err(error)) = poll::SendStream::poll_closed(&mut send, &mut cx) else { diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 0ebe75b7b14..4b176d13e34 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -98,6 +98,9 @@ impl + Clone + 'st fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(&mut self.0, cx).map(Error) } + fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(&self.0)) } @@ -127,20 +130,32 @@ impl poll::SendStream for SendStream { } /// An incoming WebTransport stream adapted to moq-net. pub struct RecvStream(S); +// The WebTransport backend returns connection credit as it reads, so there is none to hold. impl poll::RecvStream for RecvStream { type Error = Error; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(&mut self.0, cx, dst).map_err(Error) + type Credit = moq::Unheld; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + backend::RecvStream::poll_read(&mut self.0, cx, dst) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(&mut self.0, cx, buf).map_err(Error) + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_buf(&mut self.0, cx, buf) + .map_err(Error) + .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) } - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(&mut self.0, cx, max).map_err(Error) + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) + .map_err(Error) + .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) @@ -295,7 +310,7 @@ mod tests { let Poll::Ready(Ok(Some(chunk))) = poll::RecvStream::poll_read_chunk(&mut recv, &mut cx, 2) else { panic!("read failed") }; - assert_eq!(chunk, b"hi"[..]); + assert_eq!(chunk.0, b"hi"[..]); poll::RecvStream::stop(&mut recv, 13); assert_eq!(state.get(), 13); let Poll::Ready(Err(error)) = poll::SendStream::poll_closed(&mut send, &mut cx) else { From ecd05da4e834581609a3eaabba34872702f0b545 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:29:14 -0700 Subject: [PATCH 22/51] wip(moq-net): track every lite and ietf reader for receive-credit recovery Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 47 ++++++++++++------ rs/moq-net/src/ietf/auth.rs | 4 ++ rs/moq-net/src/ietf/publisher.rs | 10 +++- rs/moq-net/src/ietf/session.rs | 36 +++++++++++--- rs/moq-net/src/ietf/subscriber.rs | 80 ++++++++++++++++++++++++++----- rs/moq-net/src/lite/publisher.rs | 6 ++- rs/moq-net/src/lite/session.rs | 26 +++++++++- rs/moq-net/src/lite/subscriber.rs | 66 +++++++++++++++++++++++-- rs/moq-net/src/pressure.rs | 12 +++-- 9 files changed, 243 insertions(+), 44 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index b6c2a78da03..58d1cb7a92a 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -10,7 +10,6 @@ use crate::{ Error, StreamError, coding::*, pressure::{Pressure, Rank, Ticket}, - transport::Credit, }; @@ -116,18 +115,29 @@ impl Reader { self.release(used as u64); } - /// Whether this stream was chosen to free the receive allowance. The first time, - /// stop it and let go of everything it holds. - fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { + /// Whether this stream was chosen to free the receive allowance, registering `cx` to + /// hear when it is. A caller that parks the reader on something else races this, since + /// only a pending read registers on its own. + pub(crate) fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { + self.evicted(Some(cx)) + } + + /// Whether this stream was evicted, registering `cx` when given. The first time, stop + /// it and let go of everything it holds. + fn evicted(&mut self, cx: Option<&Context<'_>>) -> bool { if self.evicted { return true; } - if !self.ticket.as_ref().is_some_and(|ticket| ticket.poll_evicted(cx)) { - return false; + let evicted = match (&self.ticket, cx) { + (None, _) => false, + (Some(ticket), Some(cx)) => ticket.poll_evicted(cx), + (Some(ticket), None) => ticket.is_evicted(), + }; + if evicted { + self.evicted = true; + self.abort(&Error::Evicted); } - self.evicted = true; - self.abort(&Error::Evicted); - true + evicted } /// Poll `decode` against the buffered bytes, reading more until it stops coming up @@ -298,11 +308,15 @@ impl Reader { self.unit += n as u64; return Poll::Ready(Ok(Some(self.buffer.split_to(n).freeze()))); } - if self.poll_evicted(cx) { + if self.evicted(None) { return Poll::Ready(Err(Error::Evicted)); } - let Some((chunk, credit)) = ready!(self.stream.poll_read_chunk(cx, max)).map_err(|err| self.version.transport_error(err))? - else { + let read = match self.stream.poll_read_chunk(cx, max) { + Poll::Pending if self.evicted(Some(cx)) => return Poll::Ready(Err(Error::Evicted)), + Poll::Pending => return Poll::Pending, + Poll::Ready(read) => read, + }; + let Some((chunk, credit)) = read.map_err(|err| self.version.transport_error(err))? else { return Poll::Ready(Ok(None)); }; self.unit += chunk.len() as u64; @@ -421,10 +435,15 @@ impl Reader { /// Poll for more data from the stream. `true` if data was read, `false` if the /// stream finished. fn poll_read_more(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.poll_evicted(cx) { + if self.evicted(None) { return Poll::Ready(Err(Error::Evicted)); } - match ready!(self.stream.poll_read_buf(cx, &mut self.buffer)) { + let read = match self.stream.poll_read_buf(cx, &mut self.buffer) { + Poll::Pending if self.evicted(Some(cx)) => return Poll::Ready(Err(Error::Evicted)), + Poll::Pending => return Poll::Pending, + Poll::Ready(read) => read, + }; + match read { Ok(Some(credit)) => { self.take(credit); Poll::Ready(Ok(true)) diff --git a/rs/moq-net/src/ietf/auth.rs b/rs/moq-net/src/ietf/auth.rs index 53ae67d2c59..9306244a0e6 100644 --- a/rs/moq-net/src/ietf/auth.rs +++ b/rs/moq-net/src/ietf/auth.rs @@ -241,6 +241,7 @@ pub(super) async fn run_present( peer_setup: peer::PeerSetup, version: Version, going_away: crate::goaway::GoingAway, + pressure: crate::pressure::Pressure, ) -> Result<(), Error> where S: crate::transport::poll::Boxable, @@ -271,6 +272,7 @@ where handle: handle.clone(), version, id, + pressure: pressure.clone(), }; let going_away = going_away.clone(); let violation = violation.clone(); @@ -306,6 +308,7 @@ struct Present { handle: Handle, version: Version, id: u64, + pressure: crate::pressure::Pressure, } #[cfg(feature = "async")] @@ -321,6 +324,7 @@ impl Present { Ok(stream) => stream, Err(err) => return err, }; + stream.reader.track(&self.pressure, crate::pressure::Rank::Control); if let Err(err) = stream.writer.encode_message(&Auth { request_id, token }).await { return err; } diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index ec5fef0dc40..6f683ede1ca 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -522,6 +522,8 @@ pub(super) struct Publisher { // Subscriptions the peer may hold at once (`session::Limits::subscriptions`). pub(super) subscriptions: crate::session::Slots, pub(super) fetches: crate::session::Slots, + /// Tracks each stream this half opens. + pub(super) pressure: crate::pressure::Pressure, // Ranks every subscription's group streams against each other for the transport. priority: PriorityQueue, } @@ -598,6 +600,7 @@ where owed: Default::default(), subscriptions: Default::default(), fetches: Default::default(), + pressure: Default::default(), priority: Default::default(), } } @@ -2997,7 +3000,12 @@ where /// peer is waiting on us to retire. Failing instead keeps the loop moving. async fn open_request(&self) -> Result>, Error> { let mut session = self.session.clone(); - let mut open = std::pin::pin!(Stream::open(&mut session, self.version)); + let pressure = self.pressure.clone(); + let mut open = std::pin::pin!(async { + let mut stream = Stream::open(&mut session, self.version).await?; + stream.reader.track(&pressure, crate::pressure::Rank::Control); + Ok(stream) + }); let mut timeout = crate::time::Deadline::after(&self.runtime, ADVERTISE_TIMEOUT); kio::wait(|waiter| { diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index af9a871587a..ac709b5ae6b 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -3,6 +3,7 @@ use crate::{ Error, Hop, SessionError, StreamError, coding::{Decode, DecodeError, Encode, Reader, Stream, Writer}, ietf::{self, FetchHeader, RequestId}, + pressure::Rank, setup, util::{MaybeBoxedExt, MaybeSendBox, TaskSet, err_only}, }; @@ -178,9 +179,14 @@ where None => {} } + // Every stream the session reads holds its credit until a whole message arrives, and + // the monitor cancels one by rank whenever that fills the receive allowance. + let pressure = crate::pressure::Pressure::default(); + let driver = async move { // Held for the life of the session. let _setup_token = setup_token; + let mut monitor = crate::pressure::Monitor::new(session.clone(), pressure.clone()); // Released on any exit, so nothing waits on a token the session will never answer. let _auth_close = AuthClose(auth.clone()); // Decided once, before any Auth request can be accepted: the app took the requests @@ -205,7 +211,7 @@ where let res = match version { Version::Draft14 | Version::Draft15 | Version::Draft16 => { - let Some(setup) = setup else { + let Some(mut setup) = setup else { let err = Error::ProtocolViolation; session.close( super::error::to_session_code(&SessionError::from(&err)), @@ -213,6 +219,7 @@ where ); return Err(err); }; + setup.reader.track(&pressure, Rank::Control); let control = Control::new(request_id_max, client).with_window(limits.requests(), client, version); let adapter = ControlStreamAdapter::new(session.clone(), control.clone(), version); @@ -228,6 +235,7 @@ where ) .with_auth(auth.clone()); let (tasks, mut task_set) = TaskSet::new(); + publisher.pressure = pressure.clone(); publisher.withdrawal = withdrawing.clone(); publisher.owed = serving.clone(); publisher.subscriptions = subscriptions; @@ -246,6 +254,7 @@ where tasks.clone(), goaway.going_away.clone(), ) + .with_pressure(pressure.clone()) .with_auth(auth.clone()) .with_solicit(extensions.solicit); subscriber.announces = announces; @@ -314,7 +323,7 @@ where let mut sub_ns = sub_ns.clone(); let sub_ns_adapter = sub_ns_adapter.clone(); prefixes.push(async move { - let stream = match version { + let mut stream = match version { Version::Draft16 => { let mut sub_ns_adapter = sub_ns_adapter; let (send, recv) = sub_ns_adapter.open_native_bi().await?; @@ -325,6 +334,7 @@ where } _ => Stream::open(&mut sub_ns_adapter.clone(), version).await?, }; + stream.reader.track(&sub_ns.pressure, Rank::Control); if let Err(err) = sub_ns.run_subscribe_namespace(stream, prefix).await { // The peer breaking the protocol is fatal, and the driver // below turns this into the session close the draft wants. @@ -344,6 +354,7 @@ where let res = kio::wait(|waiter| { use std::task::Poll; + let _ = monitor.poll(&mut waiter.context()); if let Poll::Ready(err) = waiter.poll_future(adapter_run.as_mut()) { return Poll::Ready(Err::<(), Error>(err)); } @@ -432,6 +443,7 @@ where ) .with_auth(auth.clone()); let (tasks, mut task_set) = TaskSet::new(); + publisher.pressure = pressure.clone(); publisher.withdrawal = withdrawing.clone(); publisher.owed = serving.clone(); publisher.subscriptions = subscriptions; @@ -450,6 +462,7 @@ where tasks, goaway.going_away.clone(), ) + .with_pressure(pressure.clone()) .with_auth(auth.clone()) .with_solicit(extensions.solicit); @@ -462,6 +475,7 @@ where peer_setup.clone(), version, goaway.going_away.clone(), + pressure.clone(), ); // Only a side that offered MoQ Auth answers the peer's Auth requests: one that // arrives otherwise falls through to the protocol violation in `run_dispatch`. @@ -480,9 +494,14 @@ where // arrives on the wire. let goaway_recv = { let goaway = goaway.clone(); + let pressure = pressure.clone(); async move { match peer_setup_stream { - Some(reader) => run_goaway(reader.with_version(version), version, goaway).await, + Some(reader) => { + let mut reader = reader.with_version(version); + reader.track(&pressure, Rank::Control); + run_goaway(reader, version, goaway).await + } None => std::future::pending().await, } } @@ -526,7 +545,8 @@ where let sub_ns_session = sub_ns_session.clone(); prefixes.push(async move { let mut sub_ns_session = sub_ns_session; - let stream = Stream::open(&mut sub_ns_session, version).await?; + let mut stream = Stream::open(&mut sub_ns_session, version).await?; + stream.reader.track(&sub_ns.pressure, Rank::Control); if let Err(err) = sub_ns.run_subscribe_namespace(stream, prefix).await { // The peer breaking the protocol is fatal, and the driver // below turns this into the session close the draft wants. @@ -546,6 +566,7 @@ where let res = kio::wait(|waiter| { use std::task::Poll; + let _ = monitor.poll(&mut waiter.context()); if let Poll::Ready(err) = waiter.poll_future(unis.as_mut()) { return Poll::Ready(Err::<(), Error>(err)); } @@ -913,7 +934,7 @@ where let mut early = early.into_iter(); loop { - let reader = match early.next() { + let mut reader = match early.next() { Some(reader) => reader, None => { let recv = tasks @@ -949,6 +970,7 @@ where Reader::new(recv, outer_version) } }; + reader.track(&subscriber.pressure, Rank::Unidentified); let uni = uni.clone(); let fatal = fatal.clone(); @@ -1034,6 +1056,7 @@ impl Uni { if seen_setup.swap(true, std::sync::atomic::Ordering::Relaxed) { return Err(Error::ProtocolViolation); } + reader.rank(Rank::Control); // The negotiation gates the announce and dispatch loops, so a SETUP we // cannot read must end the session rather than leave them parked on a @@ -1205,10 +1228,11 @@ where Stream::poll_accept(&mut session, version, &mut cx) }) .await; - let stream = match accepted { + let mut stream = match accepted { Ok(stream) => stream, Err(err) => break err, }; + stream.reader.track(&bidi.subscriber.pressure, Rank::Control); // QUIC may expose an earlier stream before its bytes arrive. Reading the // header here could block later streams whose unread data holds flow control. diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 98e00ea0427..f6ad91aa05a 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -9,7 +9,9 @@ use crate::{ coding::{Decode, DecodeError, Decoder, Reader, Stream}, frame, group, ietf::{self, Control, FetchType, Filter, GroupOrder, RequestId}, - origin, track, + origin, + pressure::{Pressure, Rank}, + track, util::{MaybeBoxedExt, MaybeSendBox, TaskSet, Tasks}, }; @@ -509,6 +511,11 @@ struct TrackState { // The data streams read so far, which PUBLISH_DONE's Stream Count is checked against. tail: kio::Producer, + + // The priority and group order this subscription was sent with, which rank its data + // streams when held receive credit fills. + priority: u8, + order: track::Order, } impl TrackState { @@ -538,6 +545,18 @@ impl TrackState { joining, resume: None, tail: Default::default(), + priority: 0, + order: Default::default(), + } + } + + /// The rank of this subscription's data stream for `group`, starting at `start`. + fn rank(&self, subscribe: RequestId, group: u64, start: u64) -> Rank { + Rank::Media { + broadcast: std::sync::Arc::new(self.broadcast.clone()), + priority: crate::priority::Priority::new(self.priority, subscribe.0, group) + .with_order(self.order) + .with_start(start), } } } @@ -683,6 +702,8 @@ pub(super) struct Subscriber { frames: frame::Budget, // Namespaces the peer may have announced at once (`session::Limits::announces`). pub(super) announces: crate::session::Slots, + // Every stream this half reads, cancelled by rank when held receive credit fills. + pub(super) pressure: Pressure, } /// The prefixes to issue SUBSCRIBE_NAMESPACE for: `origin`'s permitted scope, @@ -767,10 +788,18 @@ where auth: crate::auth::Handle::new(false), frames: Default::default(), announces: Default::default(), + pressure: Default::default(), validity: Default::default(), } } + /// Open a request stream, its reads tracked as control. + async fn open_stream(&self) -> Result, Error> { + let mut stream = Stream::open(&mut self.session.clone(), self.version).await?; + stream.reader.track(&self.pressure, Rank::Control); + Ok(stream) + } + /// Bound what we subscribe to by the grant this session's tokens earn (MoQ Auth), /// and what the peer may publish to us by the session's limit, as either changes. pub fn with_auth(mut self, auth: crate::auth::Handle) -> Self { @@ -780,6 +809,13 @@ where self } + /// Track the streams this half reads with the session's `pressure`. Set before the + /// halves that clone this subscriber start. + pub fn with_pressure(mut self, pressure: Pressure) -> Self { + self.pressure = pressure; + self + } + /// Whether our SETUP declared MoQ Solicit. Without it an unsolicited PUBLISH_NAMESPACE /// is what we invited, even from a peer that implements the extension. pub fn with_solicit(mut self, solicit: bool) -> Self { @@ -2506,7 +2542,7 @@ where } }; - let mut stream = match Stream::open(&mut self.session.clone(), self.version).await { + let mut stream = match self.open_stream().await { Ok(s) => s, Err(err) => { tracing::debug!(%err, "failed to open subscribe stream"); @@ -2522,6 +2558,11 @@ where true => Fill::Requested, false => Fill::Done, }); + // The aggregate is read now: a subscriber can join while the request ID and stream + // were awaited, and nothing updates the priority or order after. + let aggregate = target.subscription(); + let priority = aggregate.as_ref().map(|s| s.priority).unwrap_or(0); + let order = aggregate.map(|s| s.order).unwrap_or_default(); { let mut state = self.state.lock(); state.subscribes.insert( @@ -2533,17 +2574,14 @@ where .filter(|start| start.frame != 0), // A resumed copy is the session's to abort from here on. producer: target.producer().cloned(), + priority, + order, ..TrackState::pending(track_name.to_owned(), broadcast_path.to_owned(), fill.clone(), joining) }, ); } - // Write Subscribe message. The aggregate is read now: a subscriber can join while - // the request ID and stream were awaited, and nothing updates the priority or - // order after. - let aggregate = target.subscription(); - let priority = aggregate.as_ref().map(|s| s.priority).unwrap_or(0); - let order = aggregate.map(|s| s.order).unwrap_or_default(); + // Write Subscribe message. let written = async { stream.writer.varint(ietf::Subscribe::ID).await?; stream @@ -3157,7 +3195,7 @@ where let work = async move { // No ID is spent while opening a stream, so dropping a blocked open // cannot leave a gap in the peer's sequential request IDs. - let mut stream = match Stream::open(&mut this.session.clone(), this.version).await { + let mut stream = match this.open_stream().await { Ok(stream) => stream, Err(err) => { tracing::debug!(%err, "failed to open joining FETCH stream"); @@ -3360,7 +3398,7 @@ where return None; } }; - let mut stream = match Stream::open(&mut self.session.clone(), self.version).await { + let mut stream = match self.open_stream().await { Ok(stream) => stream, Err(err) => { request.reject(err); @@ -3488,7 +3526,17 @@ where // SUBSCRIBE_OK or PUBLISH can be reordered behind this stream. Hold only the // subgroup header while waiting so the data stream cannot consume flow control. let aliases = self.state.lock().aliases.consume(); - let request_id = match resolve_track_alias(&self.runtime, aliases, group.track_alias).await { + let resolved = { + let mut resolve = std::pin::pin!(resolve_track_alias(&self.runtime, aliases, group.track_alias)); + // Parked off the stream, so an eviction would otherwise wait out the alias. + std::future::poll_fn(|cx| match stream.poll_evicted(cx) { + true => Poll::Ready(Err(Error::Evicted)), + false => resolve.as_mut().poll(cx), + }) + .await + }; + let request_id = match resolved { + Err(Error::Evicted) => return Err(Error::Evicted), Ok(request_id) => request_id, // Ours: we cancelled the subscription and the publisher has not stopped yet. Err(err @ Error::Cancel) => { @@ -3510,6 +3558,7 @@ where let (mut track, timescale, fill, resume, mut reading) = { let state = self.state.lock(); let track = state.subscribes.get(&request_id).ok_or(Error::NotFound)?; + stream.rank(track.rank(request_id, group.group_id, 0)); ( track.producer.clone().ok_or(Error::NotFound)?, track.timescale, @@ -4029,6 +4078,9 @@ where let joined = state.fetches.get(&header.request_id).copied(); let subscribe_id = joined.unwrap_or(header.request_id); let track = state.subscribes.get(&subscribe_id).ok_or(Error::NotFound)?; + // The head of the group the subscription joined, as urgent as its live edge. + let group = track.largest.map_or(0, |largest| largest.group); + stream.rank(track.rank(subscribe_id, group, 0)); // A fill is one of the subscription's own data streams, so PUBLISH_DONE counts // it. A joining FETCH is a request of its own. let counted = joined @@ -4048,6 +4100,10 @@ where // be reordered ahead of it. Taking the fill in the same step is what refuses a second // stream for a request that asked for one fill. let timescale = kio::wait(|waiter| { + // Parked off the stream, so an eviction would otherwise wait for SUBSCRIBE_OK. + if stream.poll_evicted(&waiter.context()) { + return Poll::Ready(Err(Error::Evicted)); + } let accepted = fill.poll(waiter, |fill| match **fill { Fill::Requested => Poll::Pending, _ => Poll::Ready(()), @@ -4402,7 +4458,7 @@ where let slot = kio::Producer::new(GroupFetch::Pending); let registered = GroupFetchEntry::new(&self.state, fetch_id, slot.clone()); - let mut stream = match Stream::open(&mut self.session.clone(), self.version).await { + let mut stream = match self.open_stream().await { Ok(stream) => stream, Err(err) => return request.reject(err), }; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 75a6cb6fa70..5612f6e8400 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -175,6 +175,8 @@ pub(super) struct Publisher { children: kio::Tasks>, /// Aborts the session on a publication outside our grant (dialing side only). enforce: Option, + /// Tracks each accepted control stream's reads. + pub(super) pressure: crate::pressure::Pressure, } impl Publisher { @@ -210,6 +212,7 @@ impl Publisher { runtime: config.runtime, accept, children: kio::Tasks::new(), + pressure: Default::default(), } } } @@ -231,7 +234,8 @@ where let mut cx = waiter.context(); loop { match Stream::poll_accept(&mut self.accept, self.shared.version, &mut cx) { - Poll::Ready(Ok(stream)) => { + Poll::Ready(Ok(mut stream)) => { + stream.reader.track(&self.pressure, crate::pressure::Rank::Control); self.children.push(Control { shared: self.shared.clone(), runtime: self.runtime.clone(), diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 4395977ff34..50b206195ee 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -3,6 +3,7 @@ use crate::{ Error, Hop, SessionError, bandwidth, coding::{Reader, Stream, Writer}, lite::SessionInfo, + pressure::{Monitor, Pressure, Rank}, }; use std::task::{Context, Poll, ready}; @@ -201,7 +202,7 @@ where runtime, client, session, - setup_stream, + mut setup_stream, publish, subscribe, peer_hop, @@ -292,8 +293,15 @@ where false => None, }; + // Every stream the session reads holds its credit until a whole message arrives, and + // the monitor cancels one by rank whenever that fills the receive allowance. + let pressure = Pressure::default(); + if let Some(stream) = &mut setup_stream { + stream.reader.track(&pressure, Rank::Control); + } + let estimate = kio::Producer::default(); - let publisher = Publisher::new(PublisherConfig { + let mut publisher = Publisher::new(PublisherConfig { estimate: estimate.weak(), runtime: runtime.clone(), session: session.clone(), @@ -324,6 +332,8 @@ where auth: auth.clone(), }); subscriber.announces = announces; + subscriber.pressure = pressure.clone(); + publisher.pressure = pressure.clone(); let driver = Driver { auth: Present { @@ -336,7 +346,9 @@ where violation: Default::default(), started: false, _setup: setup_token, + pressure: pressure.clone(), }, + monitor: Monitor::new(session.clone(), pressure), local_close: Default::default(), setup: version .has_setup_stream() @@ -362,6 +374,8 @@ where pub(crate) struct Driver { /// Presenting our tokens, one AUTH stream each. auth: Present, + /// Cancels a stream whenever held receive credit fills the allowance. + monitor: Monitor, pub(crate) local_close: std::sync::Arc, /// Advertising our capabilities, or `None` once sent (or on a version with no /// Setup Stream). @@ -427,6 +441,9 @@ where fn poll_protocol(&mut self, waiter: &kio::Waiter) -> Poll> { let mut cx = waiter.context(); + // Never resolves; it only cancels streams, whose own handlers report it. + let _ = self.monitor.poll(&mut cx); + // Presenting tokens ends the session only when the peer breaks the protocol. if let Poll::Ready(err) = self.auth.poll(waiter) { return Poll::Ready(Err(err)); @@ -480,6 +497,8 @@ struct Present { started: bool, /// The connection's own credential, held for the life of the session. _setup: Option, + /// Tracks each AUTH stream's reads. + pressure: Pressure, } impl Present { @@ -499,6 +518,7 @@ impl Present { handle: self.handle.clone(), going_away: self.going_away.clone(), violation: self.violation.clone(), + pressure: self.pressure.clone(), id, state: PresentState::Open { token }, }); @@ -521,6 +541,7 @@ struct PresentToken { handle: crate::auth::Handle, going_away: crate::goaway::GoingAway, violation: kio::Shared>, + pressure: Pressure, id: u64, state: PresentState, } @@ -579,6 +600,7 @@ impl PresentToken { Ok(stream) => stream, Err(err) => return Poll::Ready(err), }; + stream.reader.track(&self.pressure, Rank::Control); let res = stream .writer .buffer(&super::ControlType::Auth) diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index cd9948f28f6..f906533359a 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10,6 +10,7 @@ use crate::{ AsPath, Error, Path, PathOwned, Timescale, Timestamp, bandwidth, coding::{Reader, Stream}, lite, + pressure::{Pressure, Rank}, track::{Position, Subscription}, }; @@ -88,6 +89,8 @@ pub(super) struct Subscriber { frames: frame::Budget, /// Broadcasts the peer may have announced at once (`session::Limits::announces`). pub(super) announces: crate::session::Slots, + /// Every stream this half reads, cancelled by rank when held receive credit fills. + pub(super) pressure: Pressure, } #[derive(Clone)] @@ -98,6 +101,27 @@ struct TrackEntry { timescale: Option, /// The groups received so far, so the subscription's end can wait for the ones owed. tail: kio::Producer, + /// Ranks the subscription's group streams when held receive credit fills. + rank: GroupRank, +} + +/// What ranks a subscription's group streams for receive-side recovery. +#[derive(Clone, Default)] +struct GroupRank { + broadcast: Arc, + priority: u8, + order: track::Order, +} + +impl GroupRank { + fn of(&self, subscribe: u64, group: &lite::Group) -> Rank { + Rank::Media { + broadcast: self.broadcast.clone(), + priority: crate::priority::Priority::new(self.priority, subscribe, group.sequence) + .with_order(self.order) + .with_start(group.frame_start), + } + } } impl Subscriber { @@ -125,6 +149,7 @@ impl Subscriber { auth: config.auth, frames: Default::default(), announces: Default::default(), + pressure: Default::default(), } } @@ -681,7 +706,8 @@ impl UniAccept { fn new(subscriber: Subscriber, early: Vec>) -> Self { let accept = subscriber.session.clone(); let mut children = kio::Tasks::new(); - for reader in early { + for mut reader in early { + reader.track(&subscriber.pressure, Rank::Unidentified); children.push(UniServe { subscriber: subscriber.clone(), state: UniState::Start { reader }, @@ -701,11 +727,11 @@ impl UniAccept { loop { match self.accept.poll_accept_uni(&mut cx) { Poll::Ready(Ok(stream)) => { + let mut reader = Reader::new(stream, self.subscriber.version); + reader.track(&self.subscriber.pressure, Rank::Unidentified); self.children.push(UniServe { subscriber: self.subscriber.clone(), - state: UniState::Start { - reader: Reader::new(stream, self.subscriber.version), - }, + state: UniState::Start { reader }, }); } Poll::Ready(Err(err)) => return Poll::Ready(Err(crate::lite::error::from_transport(err))), @@ -786,7 +812,10 @@ impl UniServe { }; self.state = match kind { lite::DataType::Group => UniState::Group(GroupRecv::new(self.subscriber.clone(), reader)), - lite::DataType::Setup => UniState::Setup { reader }, + lite::DataType::Setup => { + reader.rank(Rank::Control); + UniState::Setup { reader } + } }; } UniState::Setup { reader } => { @@ -870,6 +899,7 @@ impl GroupRecv { let (group, track, timescale, reading) = { let mut subs = self.subscriber.subscribes.lock(); let entry = subs.get_mut(&hdr.subscribe).ok_or(Error::Cancel)?; + self.reader.rank(entry.rank.of(hdr.subscribe, &hdr)); let group_info = group::Info { sequence: hdr.sequence }; // Stats (groups/frames/bytes) are counted in the model as the group @@ -1254,6 +1284,7 @@ impl ProbeStream { return Poll::Ready(Ok(())); } let mut stream = ready!(Stream::poll_open(&mut self.session, self.subscriber.version, &mut cx))?; + stream.reader.track(&self.subscriber.pressure, Rank::Control); stream.writer.buffer(&lite::ControlType::Probe)?; self.state = ProbeState::Send { stream }; } @@ -1348,6 +1379,7 @@ impl AnnouncePrefix { self.subscriber.version, &mut cx ))?; + stream.reader.track(&self.subscriber.pressure, Rank::Control); stream.writer.buffer(&lite::ControlType::Announce)?; // Lite04/05: ask the peer to filter out announces that already passed @@ -2142,6 +2174,7 @@ mod tests { producer: track, timescale: Some(Timescale::MILLI), tail: Default::default(), + rank: Default::default(), }, ); let mut accept = UniAccept::new(subscriber, early); @@ -2198,6 +2231,7 @@ mod tests { producer: track.clone(), timescale: Some(Timescale::MILLI), tail: Default::default(), + rank: Default::default(), }, ); @@ -2266,6 +2300,7 @@ mod tests { producer: track.clone(), timescale: Some(Timescale::MILLI), tail: tail.clone(), + rank: Default::default(), }, ); @@ -2484,6 +2519,7 @@ mod tests { producer, timescale: Some(Timescale::MILLI), tail: Default::default(), + rank: Default::default(), }, ); @@ -4598,6 +4634,11 @@ impl TrackServe { let start_moved = active.start != subscription.start; active.priority = subscription.priority; active.order = subscription.order; + // Streams already open keep the rank they were read at. + if let Some(entry) = self.subscriber.subscribes.lock().get_mut(&active.id) { + entry.rank.priority = subscription.priority; + entry.rank.order = subscription.order; + } active.max_delay = subscription.max_delay; if let Ok(mut tail) = active.tail.write() { tail.set_grace(tail::grace(subscription.max_delay)); @@ -4684,6 +4725,11 @@ impl TrackServe { producer: producer.clone(), timescale, tail: tail.clone(), + rank: GroupRank { + broadcast: Arc::new(self.path.clone()), + priority: subscription.priority, + order: subscription.order, + }, }, ); @@ -4826,6 +4872,7 @@ impl Establish { self.serve.subscriber.version, &mut cx ))?; + stream.reader.track(&self.serve.subscriber.pressure, Rank::Control); let bounds = WireBounds::new(self.subscription.start, self.subscription.end); let msg = lite::Subscribe { @@ -5124,6 +5171,7 @@ impl TrackInfoFetch { match &mut self.state { TrackInfoState::Open => { let mut stream = ready!(Stream::poll_open(&mut self.session, serve.subscriber.version, &mut cx))?; + stream.reader.track(&serve.subscriber.pressure, Rank::Control); stream.writer.buffer(&lite::ControlType::Track)?; stream.writer.buffer(&lite::Track { epoch: serve.epoch.clone(), @@ -5844,6 +5892,14 @@ impl kio::Task for FetchServeRun { tracing::info!(broadcast = %self.serve.subscriber.log_path(&self.serve.path), track = %self.serve.name, group = self.group, "fetch started"); let request = request.take().expect("request pending"); + // The response is the group's frames, ranked like a group stream of the track. + stream.reader.track( + &self.serve.subscriber.pressure, + Rank::Media { + broadcast: Arc::new(self.serve.path.clone()), + priority: crate::priority::Priority::new(request.priority(), u64::MAX, self.group), + }, + ); // A peer that predates lite-06 addresses whole groups only, so ask for // the whole group and number the response from 0. The wider group still diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 9597c0557a1..2fbebb7e0b9 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -39,8 +39,9 @@ pub(crate) enum Rank { Unidentified, /// A media stream of one broadcast, ranked within it. Media { - /// The broadcast, the domain inside which priorities compare. - broadcast: PathOwned, + /// The broadcast, the domain inside which priorities compare. Shared, since every + /// group stream of a subscription names the same one. + broadcast: Arc, /// The stream's rank within its broadcast. priority: Priority, }, @@ -132,7 +133,7 @@ fn victim(entries: &Slab) -> Option { }; // Media: the broadcast holding the most, then its least useful stream. - let mut domains: Vec<(&PathOwned, u64)> = Vec::new(); + let mut domains: Vec<(&Arc, u64)> = Vec::new(); for (_, entry) in holding() { if let Rank::Media { broadcast, .. } = &entry.rank { match domains.iter_mut().find(|(path, _)| *path == broadcast) { @@ -187,6 +188,11 @@ impl Ticket { self.slot.held.store(bytes, Ordering::Relaxed); } + /// Whether the monitor chose this stream, without registering to hear when it does. + pub fn is_evicted(&self) -> bool { + self.slot.evicted.load(Ordering::Acquire) + } + /// Whether the monitor chose this stream, registering `cx` to hear when it does. pub fn poll_evicted(&self, cx: &Context<'_>) -> bool { if self.slot.evicted.load(Ordering::Acquire) { From 8e780f42246393d8f94a05fc6d57e618d0285e0f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:43:44 -0700 Subject: [PATCH 23/51] test(moq-net): receive-credit recovery over a finite memory window Session tests fail without the monitor: partial frames, unidentified headers, and partial control requests each stall the sender for good. The memory transport reports fills through its own signal, so a monitor is not woken by every read and write. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/transport/adapter.rs | 7 +- rs/moq-net/benches/auth.rs | 2 +- rs/moq-net/src/client.rs | 12 +- rs/moq-net/src/coding/reader.rs | 183 +++++++++++++++++- rs/moq-net/src/fuzz.rs | 1 - rs/moq-net/src/ietf/adapter.rs | 2 - rs/moq-net/src/ietf/auth.rs | 1 + rs/moq-net/src/lite/subscriber.rs | 2 + rs/moq-net/src/pressure.rs | 133 ++++++++++++- rs/moq-net/src/transport.rs | 20 +- rs/moq-net/src/transport/memory.rs | 143 ++++++++++++-- rs/moq-net/tests/auth.rs | 2 +- rs/moq-net/tests/dial_split_horizon.rs | 2 +- rs/moq-net/tests/goaway.rs | 2 +- rs/moq-net/tests/receive_credit.rs | 258 +++++++++++++++++++++++++ rs/moq-net/tests/session_panic.rs | 2 +- rs/moq-net/tests/support/harness.rs | 5 +- rs/moq-net/tests/support/mock.rs | 11 +- rs/moq-uring/src/transport/adapter.rs | 7 +- rs/moq-wasm/src/transport/adapter.rs | 7 +- 20 files changed, 759 insertions(+), 43 deletions(-) create mode 100644 rs/moq-net/tests/receive_credit.rs diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 4b176d13e34..cefedf2a597 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -155,7 +155,12 @@ impl poll::RecvStream for RecvStream { ) -> Poll, Self::Error>> { backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) .map_err(Error) - .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) + .map_ok(|chunk| { + chunk.map(|b| { + let n = b.len() as u64; + (b, moq::Unheld(n)) + }) + }) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) diff --git a/rs/moq-net/benches/auth.rs b/rs/moq-net/benches/auth.rs index 044c88254ac..da325002c07 100644 --- a/rs/moq-net/benches/auth.rs +++ b/rs/moq-net/benches/auth.rs @@ -21,7 +21,7 @@ struct Pair { } async fn pair(version: Version) -> Pair { - let (client, server) = create_mock_session_pair(Some(version.alpn())); + let (client, server) = create_mock_session_pair(Some(version.alpn()), None); let client = async { let (session, driver) = moq_net::Client::new() .with_versions(version.into()) diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 91eefbd3f76..9458d8aee7f 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -709,7 +709,11 @@ mod tests { type Error = FakeError; type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read( + &mut self, + _cx: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>> { if self.data.is_empty() { return Poll::Ready(Ok(None)); } @@ -1075,7 +1079,11 @@ mod tests { type Error = FakeError; type Credit = crate::transport::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read( + &mut self, + cx: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>> { self.inner.poll_read(cx, dst) } diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 58d1cb7a92a..fdace948873 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -608,7 +608,11 @@ mod tests { type Error = crate::lite::test_transport::SinkError; type Credit = crate::transport::Unheld; - fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read( + &mut self, + _cx: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>> { let Some(chunk) = self.0.pop_front() else { return Poll::Pending; }; @@ -810,7 +814,11 @@ mod tests { type Error = crate::lite::test_transport::SinkError; type Credit = crate::transport::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + fn poll_read( + &mut self, + cx: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>> { // Start each turn with none of the last turn's registrations left. self.waiter = kio::Waiter::new(self.waiter.waker().clone()); while self.payload.poll_read_chunk(&self.waiter).is_ready() {} @@ -850,4 +858,175 @@ mod tests { // Two budgets' worth, and the boundary is owed nothing by the time it arrives. assert_eq!(wakes.count(), 2, "the burst was withheld past its budget"); } + + mod credit { + use std::task::Waker; + + use super::*; + use crate::lite::Version; + use crate::transport::{ + memory, + poll::{SendStream as _, Session as _}, + }; + + fn noop() -> Context<'static> { + Context::from_waker(Waker::noop()) + } + + /// A memory stream whose writer stays open, read by a reader the returned pressure + /// tracks. `window` bounds what the reader's side accepts. + fn tracked( + window: Option, + ) -> ( + memory::SendStream, + Reader, + Pressure, + (memory::Session, memory::Session), + ) { + let (mut a, mut b) = memory::pair(memory::Config { + receive_window: window, + ..Default::default() + }); + let Poll::Ready(Ok(send)) = a.poll_open_uni(&mut noop()) else { + panic!("open") + }; + let Poll::Ready(Ok(recv)) = b.poll_accept_uni(&mut noop()) else { + panic!("accept") + }; + let pressure = Pressure::default(); + let mut reader = Reader::new(recv, Version::Lite05); + reader.track(&pressure, Rank::Unidentified); + (send, reader, pressure, (a, b)) + } + + fn varint(poll: Poll>) -> u64 { + match poll { + Poll::Ready(Ok(value)) => value, + other => panic!("{other:?}"), + } + } + + fn write(send: &mut memory::SendStream, bytes: &'static [u8]) -> Poll { + send.poll_write(&mut noop(), bytes).map(|res| res.expect("write")) + } + + /// An incomplete unit keeps its credit, and the bytes read past a unit stay held + /// for the next one. + #[test] + fn credit_returns_per_complete_unit() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + + // The first byte of a two-byte varint. + assert_eq!(write(&mut send, &[0x40]), Poll::Ready(1)); + assert!(reader.poll_varint(&mut noop()).is_pending()); + assert_eq!(pressure.held(), 1); + + // The rest of it, plus a second varint read ahead in the same chunk. + assert_eq!(write(&mut send, &[0x05, 0x07]), Poll::Ready(2)); + assert_eq!(varint(reader.poll_varint(&mut noop())), 5); + assert_eq!(pressure.held(), 1); + assert_eq!(varint(reader.poll_varint(&mut noop())), 7); + assert_eq!(pressure.held(), 0); + } + + /// A peek holds what it looked at until the unit is consumed. + #[test] + fn a_peek_keeps_the_credit() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + assert_eq!(write(&mut send, &[0x07, 0x00, 0x01]), Poll::Ready(3)); + assert_eq!(varint(reader.poll_varint_peek(&mut noop())), 7); + assert_eq!(pressure.held(), 3); + let bytes = reader.poll_read_exact(&mut noop(), 2); + assert!(matches!(bytes, Poll::Ready(Ok(bytes)) if bytes.as_ref() == [0x07, 0x00])); + assert_eq!(pressure.held(), 1); + } + + /// Bytes buffered before tracking began already returned their credit. + #[test] + fn tracking_starts_from_what_is_buffered() { + let (mut a, mut b) = memory::pair(memory::Config::default()); + let Poll::Ready(Ok(mut send)) = a.poll_open_uni(&mut noop()) else { + panic!("open") + }; + let Poll::Ready(Ok(recv)) = b.poll_accept_uni(&mut noop()) else { + panic!("accept") + }; + let mut reader = Reader::new(recv, Version::Lite05); + let _sessions = (a, b); + + assert_eq!(write(&mut send, &[0x07, 0x40]), Poll::Ready(2)); + assert_eq!(varint(reader.poll_varint(&mut noop())), 7); + let pressure = Pressure::default(); + reader.track(&pressure, Rank::Unidentified); + assert_eq!(pressure.held(), 0); + + assert_eq!(write(&mut send, &[0x09, 0x01]), Poll::Ready(2)); + assert_eq!(varint(reader.poll_varint(&mut noop())), 9); + assert_eq!(pressure.held(), 1); + } + + /// An untracked reader returns credit as it reads, so an incomplete unit never + /// stalls the peer. + #[test] + fn an_untracked_reader_holds_nothing() { + let (mut a, mut b) = memory::pair(memory::Config { + receive_window: Some(2), + ..Default::default() + }); + let Poll::Ready(Ok(mut send)) = a.poll_open_uni(&mut noop()) else { + panic!("open") + }; + let Poll::Ready(Ok(recv)) = b.poll_accept_uni(&mut noop()) else { + panic!("accept") + }; + let mut reader = Reader::new(recv, Version::Lite05); + let _sessions = (a, b); + + // An eight-byte varint, two bytes at a time. + assert_eq!(write(&mut send, &[0xc0, 0x00]), Poll::Ready(2)); + assert!(reader.poll_varint(&mut noop()).is_pending()); + assert_eq!(write(&mut send, &[0x00, 0x00]), Poll::Ready(2)); + } + + /// A tracked reader's incomplete unit spends the window until it completes. + #[test] + fn a_tracked_reader_holds_the_window() { + let (mut send, mut reader, _pressure, _sessions) = tracked(Some(2)); + assert_eq!(write(&mut send, &[0xc0, 0x00]), Poll::Ready(2)); + assert!(reader.poll_varint(&mut noop()).is_pending()); + assert!(write(&mut send, &[0x00]).is_pending()); + } + + /// Aborting lets go of the credit, and what was buffered stays readable. + #[test] + fn abort_returns_the_credit() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + assert_eq!(write(&mut send, &[0x07, 0x40]), Poll::Ready(2)); + assert!(reader.poll_has_more(&mut noop()).is_ready()); + assert_eq!(pressure.held(), 2); + reader.abort(&Error::Cancel); + assert_eq!(pressure.held(), 0); + assert_eq!(varint(reader.poll_varint(&mut noop())), 7); + assert_eq!(pressure.held(), 0); + } + + /// An evicted reader stops its stream and fails its next read. + #[test] + fn eviction_stops_the_stream() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + assert_eq!(write(&mut send, &[0x40]), Poll::Ready(1)); + assert!(reader.poll_varint(&mut noop()).is_pending()); + + assert!(pressure.evict()); + assert!(matches!( + reader.poll_varint(&mut noop()), + Poll::Ready(Err(Error::Evicted)) + )); + assert_eq!(pressure.held(), 0); + assert!(matches!( + send.poll_write(&mut noop(), &[0x05]), + Poll::Ready(Err(memory::Error::Stopped(_))) + )); + } + } } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index cc7ae0fac20..c6be95db885 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1193,7 +1193,6 @@ impl crate::transport::poll::RecvStream for Chunks { } let n = self.chunk.min(max).min(self.payload.len()); std::task::Poll::Ready(Ok(Some((self.payload.split_to(n), crate::transport::Unheld(n as u64))))) - } fn stop(&mut self, _code: u32) {} diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 9f11276b241..484b05860a7 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -13,7 +13,6 @@ use crate::{ transport::{Credit, Unheld}, }; - use super::{Control, Message, Version, control::RequestPermit}; // === Message Queues === @@ -538,7 +537,6 @@ impl crate::transport::poll::RecvStream for } } - fn stop(&mut self, code: u32) { match self { Self::Real(s) => s.stop(code), diff --git a/rs/moq-net/src/ietf/auth.rs b/rs/moq-net/src/ietf/auth.rs index 9306244a0e6..59efa7ab5b1 100644 --- a/rs/moq-net/src/ietf/auth.rs +++ b/rs/moq-net/src/ietf/auth.rs @@ -233,6 +233,7 @@ pub(super) fn peer_grant( /// says it negotiated the extension. Never returns: a session without it just fails /// the tokens as unsupported. #[cfg(feature = "async")] +#[allow(clippy::too_many_arguments)] pub(super) async fn run_present( runtime: crate::time::Clock, session: S, diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index f906533359a..9be714605f1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -4201,6 +4201,8 @@ impl SubStream { } } +// One per subscribed track, and the big variant is the working state. +#[allow(clippy::large_enum_variant)] enum Sub { None, Active(SubStream), diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 2fbebb7e0b9..fb368f0f49f 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -119,7 +119,11 @@ impl Pressure { #[cfg(test)] pub fn held(&self) -> u64 { let state = self.state.lock(); - state.entries.iter().map(|(_, e)| e.slot.held.load(Ordering::Relaxed)).sum() + state + .entries + .iter() + .map(|(_, e)| e.slot.held.load(Ordering::Relaxed)) + .sum() } } @@ -240,3 +244,130 @@ impl Monitor { Poll::Pending } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Path, track::Order}; + + fn media(broadcast: &str, track: u8, group: u64) -> Rank { + Rank::Media { + broadcast: Arc::new(Path::new(broadcast).to_owned()), + priority: Priority::new(track, 0, group), + } + } + + /// Track a receiver holding `held` bytes. + fn hold(pressure: &Pressure, rank: Rank, held: u64) -> Ticket { + let ticket = pressure.track(rank); + ticket.held(held); + ticket + } + + /// Evict once, returning every ticket chosen so far. + fn evict(pressure: &Pressure, tickets: &[&Ticket]) -> Vec { + assert!(pressure.evict()); + tickets + .iter() + .enumerate() + .filter(|(_, ticket)| ticket.is_evicted()) + .map(|(i, _)| i) + .collect() + } + + #[test] + fn media_goes_before_unidentified_before_control() { + let pressure = Pressure::default(); + let control = hold(&pressure, Rank::Control, 100); + let unidentified = hold(&pressure, Rank::Unidentified, 50); + let media = hold(&pressure, media("room", 255, 0), 10); + let all = [&control, &unidentified, &media]; + + assert_eq!(evict(&pressure, &all), [2]); + assert_eq!(evict(&pressure, &all), [1, 2]); + assert_eq!(evict(&pressure, &all), [0, 1, 2]); + // Everything holding was chosen already. + assert!(!pressure.evict()); + } + + #[test] + fn the_least_urgent_stream_of_the_fullest_broadcast_goes() { + let pressure = Pressure::default(); + // `small` holds less in total, so its less urgent track is not compared. + let small = hold(&pressure, media("small", 0, 0), 30); + let urgent = hold(&pressure, media("big", 200, 0), 20); + let lax = hold(&pressure, media("big", 100, 0), 20); + let all = [&small, &urgent, &lax]; + + assert_eq!(evict(&pressure, &all), [2]); + // `big` is left with 20, now less than `small`. + assert_eq!(evict(&pressure, &all), [0, 2]); + } + + #[test] + fn group_order_ranks_within_a_subscription() { + let pressure = Pressure::default(); + let old = hold(&pressure, media("room", 1, 1), 10); + let new = hold(&pressure, media("room", 1, 2), 10); + // Newest first by default, so the older group is the less useful one. + assert_eq!(evict(&pressure, &[&old, &new]), [0]); + + let ascending = |group| Rank::Media { + broadcast: Arc::new(Path::new("room").to_owned()), + priority: Priority::new(1, 0, group).with_order(Order::Ascending), + }; + let pressure = Pressure::default(); + let old = hold(&pressure, ascending(1), 10); + let new = hold(&pressure, ascending(2), 10); + assert_eq!(evict(&pressure, &[&old, &new]), [1]); + } + + #[test] + fn ties_go_to_the_larger_then_the_newer() { + let pressure = Pressure::default(); + let first = hold(&pressure, Rank::Unidentified, 10); + let larger = hold(&pressure, Rank::Unidentified, 20); + let newest = hold(&pressure, Rank::Unidentified, 10); + let all = [&first, &larger, &newest]; + assert_eq!(evict(&pressure, &all), [1]); + assert_eq!(evict(&pressure, &all), [1, 2]); + } + + #[test] + fn empty_and_dropped_receivers_are_never_chosen() { + let pressure = Pressure::default(); + let idle = hold(&pressure, media("room", 0, 0), 0); + drop(hold(&pressure, media("room", 0, 0), 10)); + assert!(!pressure.evict()); + assert!(!idle.is_evicted()); + } + + #[test] + fn rerank_once_identified() { + let pressure = Pressure::default(); + let header = hold(&pressure, Rank::Unidentified, 10); + let control = hold(&pressure, Rank::Control, 10); + header.rank(media("room", 0, 0)); + assert_eq!(evict(&pressure, &[&header, &control]), [0]); + } + + #[test] + fn eviction_wakes_the_reader() { + #[derive(Default)] + struct Count(std::sync::atomic::AtomicUsize); + impl std::task::Wake for Count { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + let pressure = Pressure::default(); + let ticket = hold(&pressure, Rank::Unidentified, 10); + let count = Arc::new(Count::default()); + let waker = Waker::from(count.clone()); + assert!(!ticket.poll_evicted(&Context::from_waker(&waker))); + assert!(pressure.evict()); + assert_eq!(count.0.load(Ordering::Relaxed), 1); + assert!(ticket.poll_evicted(&Context::from_waker(&waker))); + } +} diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index cdbf4ec1419..253696211d2 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -351,15 +351,21 @@ pub mod poll { /// boxed drivers (`Send` boxes on native): what the moq-transport path /// requires until it too becomes named machines. Implemented automatically. pub trait Boxable: - Session, Error: MaybeSend> - + MaybeSend + Session< + SendStream: MaybeSend, + RecvStream: MaybeSend + RecvStream, + Error: MaybeSend, + > + MaybeSend + MaybeSync { } impl Boxable for S where - S: Session, Error: MaybeSend> - + MaybeSend + S: Session< + SendStream: MaybeSend, + RecvStream: MaybeSend + RecvStream, + Error: MaybeSend, + > + MaybeSend + MaybeSync { } @@ -527,7 +533,11 @@ pub mod poll { /// Returns the credit for the bytes read ([`Credit::len`] counts them), or /// `None` once the peer has finished the stream. An empty `dst` reads nothing /// and returns an empty credit: asking for no bytes is not end of stream. - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>>; + fn poll_read( + &mut self, + cx: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>>; /// Poll to read some data into the provided buffer, advancing it by the bytes /// read, which the returned credit covers. diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 4d4dc18d574..12cef7337e6 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -67,16 +67,19 @@ impl Default for Config { /// Create two connected sessions: streams one opens, the other accepts. pub fn pair(config: Config) -> (Session, Session) { - let window = || { + let fills = [Arc::new(Fills::default()), Arc::new(Fills::default())]; + let window = |fills: &Arc| { kio::Shared::new(Window { limit: config.receive_window, + fills: fills.clone(), ..Default::default() }) }; let conn = Arc::new(Conn { close: Default::default(), incoming: [Incoming::default(), Incoming::default()], - windows: [window(), window()], + windows: [window(&fills[0]), window(&fills[1])], + fills, config, }); @@ -89,7 +92,6 @@ pub fn pair(config: Config) -> (Session, Session) { accept_bi: Default::default(), datagram: Default::default(), closed: Default::default(), - exhausted: Default::default(), }; (side(0), side(1)) @@ -164,6 +166,8 @@ struct Conn { incoming: [Incoming; 2], /// Each side's receive allowance. windows: [kio::Shared; 2], + /// Each side's fills of that allowance, not yet reported. + fills: [Arc; 2], } /// One side's receive allowance, shared by every stream it reads. @@ -177,8 +181,9 @@ struct Window { held: usize, /// Held credit fills the limit. exhausted: bool, - /// A fill not yet reported to [`poll::Session::poll_credit_exhausted`]. - unreported: bool, + /// Where a new fill is reported. Its own lock and waker, so the session's monitor + /// is not woken by every read and write that changes the window. + fills: Arc, } impl Window { @@ -194,12 +199,43 @@ impl Window { fn check(&mut self) { let full = self.limit.is_some_and(|limit| self.held > 0 && self.held >= limit); if full && !self.exhausted { - self.unreported = true; + self.fills.report(); } self.exhausted = full; } } +/// Fills of one side's receive allowance, waiting for +/// [`poll::Session::poll_credit_exhausted`]. +#[derive(Default)] +struct Fills { + unreported: std::sync::atomic::AtomicBool, + waker: std::sync::Mutex>, +} + +impl Fills { + fn report(&self) { + self.unreported.store(true, std::sync::atomic::Ordering::Release); + let waker = self.waker.lock().unwrap_or_else(|err| err.into_inner()).take(); + if let Some(waker) = waker { + waker.wake(); + } + } + + fn poll(&self, cx: &Context<'_>) -> Poll<()> { + use std::sync::atomic::Ordering; + if self.unreported.swap(false, Ordering::AcqRel) { + return Poll::Ready(()); + } + *self.waker.lock().unwrap_or_else(|err| err.into_inner()) = Some(cx.waker().clone()); + // Checked again: a fill between the swap and the store took no waker. + match self.unreported.swap(false, Ordering::AcqRel) { + true => Poll::Ready(()), + false => Poll::Pending, + } + } +} + /// The receive allowance taken by bytes read from an in-memory stream, returned on drop. pub struct Credit { window: kio::Shared, @@ -349,7 +385,6 @@ pub struct Session { accept_bi: kio::Park, datagram: kio::Park, closed: kio::Park, - exhausted: kio::Park, } impl Session { @@ -444,14 +479,7 @@ impl poll::Session for Session { } fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { - let waiter = self.exhausted.hold(cx); - let window = &self.side.conn.windows[self.side.local]; - let mut window = ready!(window.poll(waiter, |window| match window.unreported { - true => Poll::Ready(()), - false => Poll::Pending, - })); - window.unreported = false; - Poll::Ready(()) + self.side.conn.fills[self.side.local].poll(cx) } fn stats(&self) -> impl super::Stats { @@ -712,7 +740,6 @@ impl poll::RecvStream for RecvStream { } } - /// Ready once the writer finished or reset the stream, or this side stopped it. fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { if self.stopped { @@ -749,6 +776,7 @@ mod tests { }; use super::*; + use crate::transport::Credit as _; use poll::{RecvStream as _, SendStream as _, Session as _}; /// Counts its wakes. @@ -795,7 +823,6 @@ mod tests { .map_ok(|chunk| chunk.map(|(chunk, _)| chunk)) } - fn text(chunk: Poll, Error>>) -> Bytes { ready(chunk).unwrap().unwrap() } @@ -1033,4 +1060,86 @@ mod tests { } assert!(b.poll_recv_datagram(&mut noop()).is_pending()); } + + fn window(receive_window: usize) -> Config { + Config { + receive_window: Some(receive_window), + ..Default::default() + } + } + + /// Queued bytes and held credit both spend the window; dropping the credit returns it. + #[test] + fn the_window_bounds_queued_and_held_bytes() { + let (mut a, mut b) = pair(window(10)); + let (mut send, mut recv) = uni(&mut a, &mut b); + let (mut other, _other_recv) = uni(&mut a, &mut b); + + // A write takes only what fits, and the window is shared by every stream. + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 16]))).unwrap(), 10); + assert!(write(&mut other, &Bytes::from_static(b"x")).is_pending()); + + // Reading moves bytes from queued to held, which still spends the window. + let (chunk, credit) = ready(recv.poll_read_chunk(&mut noop(), 4)).unwrap().unwrap(); + assert_eq!(chunk.len(), 4); + assert!(write(&mut other, &Bytes::from_static(b"x")).is_pending()); + + drop(credit); + assert_eq!(ready(write(&mut other, &Bytes::from(vec![2u8; 8]))).unwrap(), 4); + } + + /// A fill is reported once, and again only after some credit returned and the window + /// filled again. + #[test] + fn exhaustion_is_reported_once_per_fill() { + let (mut a, mut b) = pair(window(8)); + let (mut send, mut recv) = uni(&mut a, &mut b); + + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 8]))).unwrap(), 8); + // Unread bytes are still on their way to the application, so nothing is stuck. + assert!(b.poll_credit_exhausted(&mut noop()).is_pending()); + + let (_, mut held) = ready(recv.poll_read_chunk(&mut noop(), 8)).unwrap().unwrap(); + ready(b.poll_credit_exhausted(&mut noop())); + assert!(b.poll_credit_exhausted(&mut noop()).is_pending()); + + // Returning some credit clears the fill; spending it again is a new one. + drop(held.split(2)); + assert!(b.poll_credit_exhausted(&mut noop()).is_pending()); + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 8]))).unwrap(), 2); + let (_, more) = ready(recv.poll_read_chunk(&mut noop(), 8)).unwrap().unwrap(); + ready(b.poll_credit_exhausted(&mut noop())); + + // The other direction has its own window. + assert!(a.poll_credit_exhausted(&mut noop()).is_pending()); + drop((held, more)); + } + + /// The peer waiting on a fill wakes when it happens. + #[test] + fn exhaustion_wakes_the_monitor() { + let (mut a, mut b) = pair(window(4)); + let (mut send, mut recv) = uni(&mut a, &mut b); + + let count = Arc::new(Count::default()); + let waker = Waker::from(count.clone()); + assert!(b.poll_credit_exhausted(&mut Context::from_waker(&waker)).is_pending()); + + ready(write(&mut send, &Bytes::from(vec![1u8; 4]))).unwrap(); + let _held = ready(recv.poll_read_chunk(&mut noop(), 4)).unwrap().unwrap(); + assert!(count.0.load(Ordering::Relaxed) > 0); + } + + /// Stopping a stream returns the window its queued bytes took. + #[test] + fn stop_returns_queued_room() { + let (mut a, mut b) = pair(window(8)); + let (mut send, mut recv) = uni(&mut a, &mut b); + let (mut other, _other_recv) = uni(&mut a, &mut b); + + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 8]))).unwrap(), 8); + assert!(write(&mut other, &Bytes::from_static(b"x")).is_pending()); + recv.stop(1); + assert_eq!(ready(write(&mut other, &Bytes::from(vec![2u8; 8]))).unwrap(), 8); + } } diff --git a/rs/moq-net/tests/auth.rs b/rs/moq-net/tests/auth.rs index ee2f9e96a9b..0f12f3bff09 100644 --- a/rs/moq-net/tests/auth.rs +++ b/rs/moq-net/tests/auth.rs @@ -221,7 +221,7 @@ struct Pair { async fn connect(opts: Options) -> Pair { let version: Version = opts.version.unwrap_or(LITE_07).parse().unwrap(); - let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn())); + let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn()), None); let mut client = Client::new().with_versions(version.into()); if let Some(publish) = &opts.client_publish { diff --git a/rs/moq-net/tests/dial_split_horizon.rs b/rs/moq-net/tests/dial_split_horizon.rs index 93715edbe57..6fbd9d466b0 100644 --- a/rs/moq-net/tests/dial_split_horizon.rs +++ b/rs/moq-net/tests/dial_split_horizon.rs @@ -44,7 +44,7 @@ async fn dial(dialer: &origin::Producer, edge: &origin::Producer) -> Link { let version = version(); let registry = stats::Registry::new(stats::Config::new()); let session_stats = registry.tier(stats::Tier::default()).session(""); - let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn())); + let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn()), None); // Cluster dials publish and subscribe through a peer handle, which is what // records the announcing session on the route. diff --git a/rs/moq-net/tests/goaway.rs b/rs/moq-net/tests/goaway.rs index 018f1bfa270..806f3e67076 100644 --- a/rs/moq-net/tests/goaway.rs +++ b/rs/moq-net/tests/goaway.rs @@ -274,7 +274,7 @@ async fn duplicate_goaway_keeps_first_payload_moq_lite_04() { moq_net_sim::timeout(TEST_TIMEOUT, async { let version: Version = "moq-lite-04".parse().unwrap(); - let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn())); + let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn()), None); let server_raw = server_transport.clone(); let client = moq_net::Client::new().with_versions(version.into()); diff --git a/rs/moq-net/tests/receive_credit.rs b/rs/moq-net/tests/receive_credit.rs new file mode 100644 index 00000000000..ae22a9dac0e --- /dev/null +++ b/rs/moq-net/tests/receive_credit.rs @@ -0,0 +1,258 @@ +#![cfg(feature = "async")] + +//! Incomplete messages hold the receiver's connection credit, and when they fill the +//! allowance the receiver cancels the least useful stream and keeps going. +//! +//! Deterministic: paused time over the mock transport, whose finite receive allowance +//! stands in for QUIC connection flow control. Without recovery each of these stalls: +//! the sender has spent every byte of credit on messages that can never complete. + +mod support; + +use std::time::Duration; + +use bytes::Bytes; +use moq_net::transport::poll::{SendStream as _, Session as _}; +use moq_net::{Hop, Timestamp, Version, frame, origin, track::Subscription}; +use support::harness::{MockConnectOptions, MockPair, connect_mock, subscribed}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Each side's receive allowance. +const WINDOW: usize = 64 * 1024; + +const VERSIONS: &[&str] = &[ + "moq-lite-03", + "moq-lite-06", + "moq-transport-14", + "moq-transport-16", + "moq-transport-17", + "moq-transport-22", +]; + +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + support::harness::spawn(driver); + producer +} + +/// A client publishing broadcast `room` to a server that subscribes to it, over links +/// that each accept [`WINDOW`] bytes of credit. +struct Setup { + pair: MockPair, + broadcast: moq_net::broadcast::Producer, + remote: moq_net::broadcast::Consumer, +} + +async fn setup(version: Version) -> Setup { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("room").unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let subscriber = produce_origin(2); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(subscriber.clone()); + options.receive_window = Some(WINDOW); + let pair = connect_mock(options).await; + + let consumer = subscriber.consume(); + moq_net_sim::timeout(TIMEOUT, consumer.routed("room")) + .await + .expect("announce timeout") + .expect("routed"); + let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("room", None)) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + Setup { + pair, + broadcast, + remote, + } +} + +impl Setup { + /// Subscribe to a new track at `priority`, once the publisher sees the demand. + async fn subscribe(&self, name: &str, priority: u8) -> (moq_net::track::Producer, moq_net::track::Subscriber) { + let track = self.broadcast.create_track(name, timed()).unwrap(); + let sub = moq_net_sim::timeout( + TIMEOUT, + self.remote + .track(name) + .unwrap() + .subscribe(Subscription::default().with_priority(priority)), + ) + .await + .expect("subscribe timeout") + .expect("subscribe"); + subscribed(&track).await; + (track, sub) + } +} + +/// The next frame's payload, which must arrive in time. +async fn next_frame(sub: &mut moq_net::track::Subscriber) -> moq_net::Result { + let mut group = moq_net_sim::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("group timed out")? + .expect("track ended"); + let frame = moq_net_sim::timeout(TIMEOUT, group.read_frame()) + .await + .expect("frame timed out")? + .expect("group ended"); + Ok(frame.payload) +} + +/// Two partial frames that together exceed the allowance: the lower-priority one is +/// cancelled, and the higher-priority one completes on the credit it returns. +#[moq_net_sim::test] +async fn the_less_urgent_partial_frame_gives_way() { + for version in VERSIONS { + let setup = setup(version.parse().unwrap()).await; + let (low, mut low_sub) = setup.subscribe("low", 1).await; + let (high, mut high_sub) = setup.subscribe("high", 200).await; + + // Three quarters of the allowance, held by a frame that never finishes. + let mut low_group = low.append_group().unwrap(); + let mut low_frame = low_group + .create_frame(frame::Info { + size: WINDOW as u64, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + low_frame.write(vec![1u8; WINDOW * 3 / 4]).unwrap(); + + // Half the allowance, which only fits once the low frame lets go. + let payload = Bytes::from(vec![2u8; WINDOW / 2]); + let mut high_group = high.append_group().unwrap(); + high_group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + high_group.finish().unwrap(); + + let got = next_frame(&mut high_sub).await.expect("high frame"); + assert_eq!(got, payload, "{version}"); + + // The session survives, and the low track keeps delivering what fits. Its + // cancelled group never completes, so the next frame the subscriber reads is this. + drop(low_frame); + drop(low_group); + let mut group = low.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, &b"after"[..]).unwrap(); + group.finish().unwrap(); + let got = next_frame(&mut low_sub).await.expect("low frame"); + assert_eq!(got, &b"after"[..], "{version}"); + drop(setup.pair); + } +} + +/// Write `bytes` on a new uni stream from the client, in the background: it pends once +/// the allowance runs out. +fn raw_uni(pair: &MockPair, bytes: Vec) { + let mut session = pair.client_transport.clone(); + moq_net_sim::spawn(async move { + let Ok(mut send) = session.open_uni().await else { return }; + let _ = send.write_chunk(Bytes::from(bytes)).await; + // Hold the stream open, so only the receiver can end it. + std::future::pending::<()>().await; + }); +} + +/// The prefix of a lite GROUP stream whose header claims `size` bytes and carries `body` +/// of them, so it never identifies itself. +fn partial_group_header(size: u32, body: usize) -> Vec { + assert!(body < size as usize); + let mut bytes = vec![0x00]; // GROUP + bytes.extend_from_slice(&(0x8000_0000 | size).to_be_bytes()); // A 4-byte varint size. + bytes.resize(bytes.len() + body, 0); + bytes +} + +/// Headers that never complete fill the allowance: one is cancelled, and a subscription +/// that arrives afterwards still delivers. +#[moq_net_sim::test] +async fn unidentified_headers_are_cancelled() { + let setup = setup("moq-lite-06".parse().unwrap()).await; + + // Together more than the allowance, and each short of its declared size. + raw_uni(&setup.pair, partial_group_header(u16::MAX as u32, 40_000)); + raw_uni(&setup.pair, partial_group_header(u16::MAX as u32, 40_000)); + + let (track, mut sub) = setup.subscribe("video", 1).await; + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, vec![3u8; 4096]).unwrap(); + group.finish().unwrap(); + assert_eq!(next_frame(&mut sub).await.expect("frame").len(), 4096); +} + +/// Control requests that never complete fill the allowance: one is cancelled with its +/// stream, and the session carries on. +#[moq_net_sim::test] +async fn partial_control_requests_are_cancelled() { + let setup = setup("moq-lite-06".parse().unwrap()).await; + + for _ in 0..2 { + let mut session = setup.pair.client_transport.clone(); + moq_net_sim::spawn(async move { + let Ok((mut send, _recv)) = session.open_bi().await else { + return; + }; + // ANNOUNCE_PLEASE, with a body that claims more than arrives. + let mut bytes = vec![0x01]; + bytes.extend_from_slice(&(0x8000_0000u32 | u16::MAX as u32).to_be_bytes()); + bytes.resize(bytes.len() + 40_000, 0); + let _ = send.write_chunk(Bytes::from(bytes)).await; + // Hold the stream open, so only the receiver can end it. + std::future::pending::<()>().await; + }); + } + + let (track, mut sub) = setup.subscribe("video", 1).await; + let mut group = track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, vec![3u8; 4096]).unwrap(); + group.finish().unwrap(); + assert_eq!(next_frame(&mut sub).await.expect("frame").len(), 4096); +} + +/// A frame as large as the allowance allows completes while partial frames of a less +/// urgent track keep arriving, each cancelled as it gets in the way. +#[moq_net_sim::test] +async fn the_largest_unit_completes_under_pressure() { + for version in VERSIONS { + let setup = setup(version.parse().unwrap()).await; + let (low, _low_sub) = setup.subscribe("low", 1).await; + let (high, mut high_sub) = setup.subscribe("high", 200).await; + + // Several groups' worth of frames that never finish, opened before and after. + let mut partial = Vec::new(); + let mut open_partial = || { + // Leaked, so the frame can borrow its group for the rest of the test. + let group = Box::leak(Box::new(low.append_group().unwrap())); + let mut frame = group + .create_frame(frame::Info { + size: WINDOW as u64, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + frame.write(vec![1u8; WINDOW / 4]).unwrap(); + partial.push(frame); + }; + open_partial(); + open_partial(); + + // Room for the stream's headers and nothing else. + let payload = Bytes::from(vec![2u8; WINDOW - 64]); + let mut group = high.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.finish().unwrap(); + open_partial(); + + let got = next_frame(&mut high_sub).await.expect("high frame"); + assert_eq!(got.len(), payload.len(), "{version}"); + drop(setup.pair); + } +} diff --git a/rs/moq-net/tests/session_panic.rs b/rs/moq-net/tests/session_panic.rs index 8ff80b487bf..7eaa2d5e319 100644 --- a/rs/moq-net/tests/session_panic.rs +++ b/rs/moq-net/tests/session_panic.rs @@ -42,7 +42,7 @@ async fn publish(version: Version, relay: &origin::Producer, hop: u64, name: &st let broadcast = origin.create_broadcast(name).unwrap(); broadcast.announce(Default::default()).unwrap(); - let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn())); + let (client_transport, server_transport) = create_mock_session_pair(Some(version.alpn()), None); let client = Client::new() .with_versions(version.into()) .with_publisher(origin.consume()); diff --git a/rs/moq-net/tests/support/harness.rs b/rs/moq-net/tests/support/harness.rs index 3d9ad90e36a..f12c206b6f3 100644 --- a/rs/moq-net/tests/support/harness.rs +++ b/rs/moq-net/tests/support/harness.rs @@ -84,6 +84,8 @@ pub struct MockConnectOptions { pub stats: MockStats, /// Count server driver polls into this counter when supplied. pub server_polls: Option>, + /// Each side's connection receive allowance, unbounded when `None`. + pub receive_window: Option, } impl MockConnectOptions { @@ -102,6 +104,7 @@ impl MockConnectOptions { cost: None, stats: MockStats::default(), server_polls: None, + receive_window: None, } } } @@ -128,7 +131,7 @@ pub struct MockPair { /// Panics if the handshake fails on either side (test-only code). pub async fn connect_mock(opts: MockConnectOptions) -> MockPair { let protocol = opts.version.alpn(); - let (client_transport, server_transport) = create_mock_session_pair(Some(protocol)); + let (client_transport, server_transport) = create_mock_session_pair(Some(protocol), opts.receive_window); client_transport.set_latency(opts.latency); client_transport.set_stats(opts.stats); server_transport.set_stats(opts.stats); diff --git a/rs/moq-net/tests/support/mock.rs b/rs/moq-net/tests/support/mock.rs index 0f7bb67745b..39a40dc8c22 100644 --- a/rs/moq-net/tests/support/mock.rs +++ b/rs/moq-net/tests/support/mock.rs @@ -414,7 +414,6 @@ impl poll::RecvStream for MockRecvStream { Poll::Ready(Ok(Some(chunk))) } - fn stop(&mut self, code: u32) { self.stop_after_latency(code); } @@ -619,8 +618,8 @@ impl poll::Session for MockSession { self.inner.poll_closed(cx) } - fn poll_credit_exhausted(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { - std::task::Poll::Pending + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + self.inner.poll_credit_exhausted(cx) } fn stats(&self) -> impl moq_net::transport::Stats { @@ -799,9 +798,13 @@ impl MockSession { /// Both sides report the given `protocol` from /// [`moq_net::transport::poll::Session::protocol`], matching ALPN negotiation /// behavior. -pub fn create_mock_session_pair(protocol: Option<&'static str>) -> (MockSession, MockSession) { +pub fn create_mock_session_pair( + protocol: Option<&'static str>, + receive_window: Option, +) -> (MockSession, MockSession) { let mut config = memory::Config::default(); config.protocol = protocol.map(str::to_string); + config.receive_window = receive_window; let (client, server) = memory::pair(config); let conn = Arc::new(Conn::default()); diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 6cfc014254e..7b0435d83f7 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -155,7 +155,12 @@ impl poll::RecvStream for RecvStream { ) -> Poll, Self::Error>> { backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) .map_err(Error) - .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) + .map_ok(|chunk| { + chunk.map(|b| { + let n = b.len() as u64; + (b, moq::Unheld(n)) + }) + }) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 4b176d13e34..cefedf2a597 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -155,7 +155,12 @@ impl poll::RecvStream for RecvStream { ) -> Poll, Self::Error>> { backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) .map_err(Error) - .map_ok(|chunk| chunk.map(|b| { let n = b.len() as u64; (b, moq::Unheld(n)) })) + .map_ok(|chunk| { + chunk.map(|b| { + let n = b.len() as u64; + (b, moq::Unheld(n)) + }) + }) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) From b2f62b34aad8a578fc7b6ab2d58585bca4668cfc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 09:48:39 -0700 Subject: [PATCH 24/51] feat(moq-quic): report when held credit is all the peer may send CreditStats gains unread and exhausted(): held credit fills the window and nothing received is left to read, so only a release makes progress. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/stats.rs | 11 ++++ rs/moq-quic/src/connection/streams/state.rs | 66 +++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/rs/moq-quic/src/connection/stats.rs b/rs/moq-quic/src/connection/stats.rs index 1a6dab4ba3b..bf3f424661f 100644 --- a/rs/moq-quic/src/connection/stats.rs +++ b/rs/moq-quic/src/connection/stats.rs @@ -192,4 +192,15 @@ pub struct CreditStats { /// Bytes the peer may still send before it needs more credit, including credit not yet /// advertised pub available: u64, + /// Bytes received, or owed for a gap below a received offset, that the application has not + /// read yet + pub unread: u64, +} + +impl CreditStats { + /// Whether held credit is all the peer may send: nothing more can arrive and nothing + /// received is left to read, so only releasing [`held`](Self::held) credit makes progress. + pub fn exhausted(&self) -> bool { + self.held > 0 && self.available == 0 && self.unread == 0 + } } diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 0b792695d9a..e2dd53d3df4 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -124,6 +124,8 @@ pub struct StreamsState { /// Bytes read with [`Chunks::hold`](super::Chunks::hold) whose connection credit the /// application has not released yet pub(super) held: u64, + /// Connection credit returned for received bytes so far: read, released, or discarded + credited: u64, /// Total quantity of outgoing data retained in send buffers, including acknowledged data pub(super) buffered_data: u64, /// Configured upper bound for `buffered_data`. @@ -184,6 +186,7 @@ impl StreamsState { data_sent: 0, data_recvd: 0, held: 0, + credited: 0, buffered_data: 0, send_window, stream_receive_window: stream_receive_window.into(), @@ -989,6 +992,7 @@ impl StreamsState { /// suppress sending further updates until the window increases significantly /// again. pub(super) fn add_read_credits(&mut self, credits: u64) -> ShouldTransmit { + self.credited += credits; if credits > self.receive_window_shrink_debt { let net_credits = credits - self.receive_window_shrink_debt; self.local_max_data = self.local_max_data.saturating_add(net_credits); @@ -1032,6 +1036,7 @@ impl StreamsState { CreditStats { held: self.held, available: self.local_max_data.saturating_sub(self.data_recvd), + unread: self.data_recvd - self.credited - self.held, } } @@ -2707,6 +2712,67 @@ mod tests { assert_eq!(client.local_max_data, initial + 16); } + /// Exhausted only once every byte the peer may send arrived and is held: unread bytes, gaps + /// still owed, and unadvertised credit all mean something can still move. + #[test] + fn exhausted_when_every_byte_is_held() { + let mut client = StreamsState::new( + Side::Client, + 128u32.into(), + 128u32.into(), + 1024 * 1024, + 64u32.into(), + (1024 * 1024u32).into(), + ); + let id = StreamId::new(Side::Server, Dir::Uni, 0); + assert!(!client.credit_stats().exhausted(), "nothing held"); + + // A readable prefix, then a gap below the last byte: the gap is still owed. + data(&mut client, id, 0, 32, false); + data(&mut client, id, 48, 16, false); + let prefix = read_held(&mut client, id); + let stats = client.credit_stats(); + assert_eq!((stats.held, stats.available, stats.unread), (32, 0, 32)); + assert!(!stats.exhausted(), "the gap can still arrive"); + + // The gap fills, so everything is readable, but not read yet. + data(&mut client, id, 32, 16, false); + assert!(!client.credit_stats().exhausted(), "unread bytes can still be read"); + + let rest = read_held(&mut client, id); + let stats = client.credit_stats(); + assert_eq!((stats.held, stats.available, stats.unread), (64, 0, 0)); + assert!(stats.exhausted()); + + // Releasing any of it lets the peer send again. + let _ = client.release(prefix); + assert!(!client.credit_stats().exhausted()); + let _ = client.release(rest); + assert_eq!(client.credit_stats().unread, 0); + } + + /// Bytes read without holding return their credit, so they never count as held or unread. + #[test] + fn plain_reads_are_neither_held_nor_unread() { + let mut client = make(Side::Client); + let id = StreamId::new(Side::Server, Dir::Uni, 0); + data(&mut client, id, 0, 32, false); + assert_eq!(client.credit_stats().unread, 32); + + let mut pending = Retransmits::default(); + let mut recv = RecvStream { + id, + state: &mut client, + pending: &mut pending, + }; + let mut chunks = recv.read(true).unwrap(); + while let Ok(Some(_)) = chunks.next(usize::MAX) {} + let _ = chunks.finalize(); + + let stats = client.credit_stats(); + assert_eq!((stats.held, stats.unread), (0, 0)); + } + #[test] #[should_panic(expected = "only 0 held")] fn release_from_another_connection() { From 8528176ee0a817b2a099c007e1eedaef93255c38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:10:52 -0700 Subject: [PATCH 25/51] feat(moq-tokio): hold receive credit through native QUIC The tokio driver reads held chunks and reports, once per stall, when held credit fills the connection window. The WebTransport bridge carries that credit through its read-ahead into moq-net, so incomplete units hold the window and the session evicts the least urgent stream when they fill it. The connection window now has to fit the largest message moq-net assembles (moq_net::transport::MIN_RECEIVE_WINDOW, just over 64 MiB). It defaults to 128 MiB, and an unset stream window follows it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/transport.rs | 12 + rs/moq-quic/src/connection/mod.rs | 9 + rs/moq-quic/src/connection/streams/state.rs | 38 +++ rs/moq-relay/tests/drills.rs | 16 +- rs/moq-tokio/src/error.rs | 6 + rs/moq-tokio/src/iroh.rs | 11 +- rs/moq-tokio/src/quic.rs | 65 ++++-- rs/moq-tokio/src/quic/backend.rs | 15 +- rs/moq-tokio/src/quic/driver/connection.rs | 43 +++- rs/moq-tokio/src/quic/driver/recv_stream.rs | 31 +++ rs/moq-tokio/src/quic/driver/tests.rs | 45 ++++ rs/moq-tokio/src/quic/web/recv.rs | 12 + rs/moq-tokio/src/server.rs | 4 +- rs/moq-tokio/src/transport.rs | 176 +++++++++----- rs/moq-tokio/src/transport/held.rs | 80 +++++++ rs/moq-tokio/src/transport/owned.rs | 41 ++-- rs/moq-tokio/tests/backend.rs | 9 +- rs/moq-tokio/tests/receive_credit.rs | 246 ++++++++++++++++++++ 18 files changed, 736 insertions(+), 123 deletions(-) create mode 100644 rs/moq-tokio/src/transport/held.rs create mode 100644 rs/moq-tokio/tests/receive_credit.rs diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 253696211d2..85c89646216 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -97,6 +97,18 @@ pub trait Stats { pub struct StatsUnavailable; impl Stats for StatsUnavailable {} +/// The smallest connection receive window a transport that holds [`Credit`] may offer. +/// +/// A session holds the credit of each unit it is still assembling, so the window has to +/// fit the largest one with its framing: a legacy announcement set, past any frame payload. +pub const MIN_RECEIVE_WINDOW: u64 = { + // A type and a size prefix, each a varint of at most 8 bytes. + const PREFIXES: u64 = 16; + let message = as crate::lite::Message>::MAX_SIZE as u64 + PREFIXES; + let frame = crate::group::MAX_CACHE_BYTES; + if message > frame { message } else { frame } +}; + /// Connection flow-control credit for bytes read from a [`poll::RecvStream`]. /// /// A transport that bounds receive memory keeps the connection window those bytes diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 50ac930e1b7..d0d7fd79fa5 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -1377,6 +1377,15 @@ impl Connection { transmit } + /// Whether held credit came to fill the receive window since the last call. + /// + /// True when [`CreditStats::exhausted()`] holds now and some read, release, or discarded + /// stream since the last call left it so: nothing more can arrive until held credit is + /// released. Check after holding, releasing, stopping, or handling incoming data. + pub fn poll_credit_exhausted(&mut self) -> bool { + mem::take(&mut self.streams.stalled) && self.streams.credit_stats().exhausted() + } + /// Ping the remote endpoint /// /// Causes an ACK-eliciting packet to be transmitted. diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index e2dd53d3df4..f460d2c7c6c 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -126,6 +126,8 @@ pub struct StreamsState { pub(super) held: u64, /// Connection credit returned for received bytes so far: read, released, or discarded credited: u64, + /// Held credit was left filling the window since the application last asked + pub(crate) stalled: bool, /// Total quantity of outgoing data retained in send buffers, including acknowledged data pub(super) buffered_data: u64, /// Configured upper bound for `buffered_data`. @@ -187,6 +189,7 @@ impl StreamsState { data_recvd: 0, held: 0, credited: 0, + stalled: false, buffered_data: 0, send_window, stream_receive_window: stream_receive_window.into(), @@ -1001,6 +1004,10 @@ impl StreamsState { self.receive_window_shrink_debt -= credits; } + // Every change to held or returned credit ends here, a hold included (its `finalize` + // returns nothing), so this sees each moment that leaves the window stalled. + self.stalled |= self.held > 0 && self.credit_stats().exhausted(); + ShouldTransmit(self.max_data_due()) } @@ -2751,6 +2758,37 @@ mod tests { assert_eq!(client.credit_stats().unread, 0); } + /// A stall reports once, and again whenever credit moves without relieving it. + #[test] + fn stall_reports_once_per_change() { + let mut client = StreamsState::new( + Side::Client, + 128u32.into(), + 128u32.into(), + 1024 * 1024, + 64u32.into(), + (1024 * 1024u32).into(), + ); + let take = |state: &mut StreamsState| { + mem::take(&mut state.stalled) && state.credit_stats().exhausted() + }; + let id = StreamId::new(Side::Server, Dir::Uni, 0); + data(&mut client, id, 0, 64, false); + let mut held = read_held(&mut client, id); + assert!(take(&mut client)); + assert!(!take(&mut client), "reported already"); + + // Shrinking the window absorbs a release, so the peer still may send nothing. + client.set_receive_window(32u32.into()); + let _ = client.release(held.split(16)); + assert!(take(&mut client), "still stalled after a release"); + + // A release past the debt relieves it. + let _ = client.release(held.split(32)); + assert!(!take(&mut client)); + let _ = client.release(held); + } + /// Bytes read without holding return their credit, so they never count as held or unread. #[test] fn plain_reads_are_neither_held_nor_unread() { diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index bce5ce6144f..3e09794d4b0 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -794,7 +794,7 @@ const SETTLE: Duration = Duration::from_secs(10); /// The cross-relay drill's link: the drills' impairment behind a bottleneck a /// burst overruns, so its tail waits in the queue or is dropped from it. It is -/// slower than [`credit`]'s receive window lets a sender go. The first burst +/// slower than [`credit`]'s stream window lets a sender go. The first burst /// is collected at ingress after setup, so scheduling cannot drain the bucket /// between its datagrams; its surviving bytes exceed the burst and 50ms queue /// allowance before they are accounted at one instant. @@ -811,18 +811,16 @@ fn bursty() -> moq_shaper::Profile { /// QUIC credit tighter than a burst on the impaired lane: fewer concurrent /// streams than a burst has groups (each group is a stream, each FETCH another), -/// and receive windows of a few groups, so senders wait on MAX_STREAMS and -/// MAX_DATA mid-burst, over a path that loses the frames granting them. +/// and stream windows of a few groups, so senders wait on MAX_STREAMS and +/// MAX_STREAM_DATA mid-burst, over a path that loses the frames granting them. +/// The connection window keeps its default, since it has to fit the largest +/// message a session assembles. fn credit(lane: Lane, mut quic: moq_tokio::quic::Config) -> moq_tokio::quic::Config { if let Lane::Impaired = lane { quic.max_streams = Some(16); // The one-shot ingress burst must fit inside transport credit before - // anything is released. Keep two bucket allowances, rounded to groups, - // while still forcing a full application burst to wait for more credit. - let receive_window = ingress_bytes(); - assert!(receive_window < BURST_GROUPS * PAYLOAD as u64); - quic.receive_window = Some(receive_window); - quic.stream_receive_window = Some(receive_window); + // anything is released. Keep two bucket allowances, rounded to groups. + quic.stream_receive_window = Some(ingress_bytes()); } quic } diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index 21aff0f4dbe..212601cf73f 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -105,6 +105,11 @@ pub enum Error { #[error("quic {0} must be non-zero and fit a QUIC varint")] WindowRange(&'static str), + /// The QUIC connection receive window can't fit the largest message a session + /// assembles. The payload is the minimum, in bytes. + #[error("quic receive_window must be at least {0} bytes to fit the largest message")] + ReceiveWindowTooSmall(u64), + /// Every backend we tried gave up without reporting why. #[error("failed to connect to server")] ConnectFailed, @@ -250,6 +255,7 @@ impl Error { | Self::Deprecated(_) | Self::IdleTimeoutRange | Self::WindowRange(_) + | Self::ReceiveWindowTooSmall(_) | Self::QlogUnsupported | Self::Connect(_) => true, #[cfg(feature = "quic")] diff --git a/rs/moq-tokio/src/iroh.rs b/rs/moq-tokio/src/iroh.rs index 058f33ca0c9..6e563179cd1 100644 --- a/rs/moq-tokio/src/iroh.rs +++ b/rs/moq-tokio/src/iroh.rs @@ -288,14 +288,14 @@ fn transport_config(quic: &crate::quic::Resolved) -> iroh::endpoint::QuicTranspo if !quic.mtu_discovery { transport = transport.mtu_discovery_config(None); } - // iroh's connection window is unlimited, so apply ours when unset. + // iroh's connection window is unlimited, so apply ours when unset, and the stream + // window follows it as on the native backend. let window = quic.receive_window.unwrap_or(crate::quic::DEFAULT_RECEIVE_WINDOW); + let stream = quic.stream_receive_window.unwrap_or(window); let window = iroh::endpoint::VarInt::from_u64(window).unwrap_or(iroh::endpoint::VarInt::MAX); + let stream = iroh::endpoint::VarInt::from_u64(stream).unwrap_or(iroh::endpoint::VarInt::MAX); transport = transport.receive_window(window); - if let Some(window) = quic.stream_receive_window { - let window = iroh::endpoint::VarInt::from_u64(window).unwrap_or(iroh::endpoint::VarInt::MAX); - transport = transport.stream_receive_window(window); - } + transport = transport.stream_receive_window(stream); if let Some(window) = quic.send_window { transport = transport.send_window(window); } @@ -500,7 +500,6 @@ mod tests { async fn bind_rejects_invalid_windows_before_reading_secret() { let dir = tempfile::tempdir().unwrap(); for (name, value) in [ - ("receive_window", 0), ("stream_receive_window", 0), ("send_window", 0), ("receive_window", 1 << 62), diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 4b28ac64ef4..5ed880deb53 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -107,11 +107,14 @@ pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Default connection-wide receive window, used when [`Config::receive_window`] is unset. /// -/// Bounds how much unread data one peer can make us buffer across all of its streams, -/// which the backends otherwise leave unlimited. Sized so a relay-to-relay session, which -/// shares one connection, can still sustain several Gbps at intercontinental RTTs. +/// Bounds how much unread data and incomplete messages one peer can make us buffer across +/// all of its streams, which the backends otherwise leave unlimited. Sized to fit the +/// largest message with room to spare, so a relay-to-relay session, which shares one +/// connection, can still sustain several Gbps at intercontinental RTTs. #[cfg(any(feature = "quic", feature = "iroh"))] -pub(crate) const DEFAULT_RECEIVE_WINDOW: u64 = 64 << 20; +pub(crate) const DEFAULT_RECEIVE_WINDOW: u64 = 128 << 20; +#[cfg(any(feature = "quic", feature = "iroh"))] +const _: () = assert!(DEFAULT_RECEIVE_WINDOW >= moq_net::transport::MIN_RECEIVE_WINDOW); /// Default idle timeout before an inactive connection is dropped. /// @@ -231,7 +234,8 @@ pub struct Config { pub congestion_control: Option, /// Connection-wide receive window, in bytes: how much data the peer may have - /// in flight across every stream at once. Defaults to 64 MiB. + /// in flight or in incomplete messages across every stream at once. Defaults to + /// 128 MiB, and must fit the largest message a session assembles, just over 64 MiB. /// /// Raise it when a fat, long path is idling below the link rate: the window /// has to cover the bandwidth-delay product or the peer stalls waiting for @@ -245,10 +249,13 @@ pub struct Config { )] pub receive_window: Option, - /// Per-stream receive window, in bytes. Unset leaves the backend default. + /// Per-stream receive window, in bytes. Unset follows + /// [`receive_window`](Self::receive_window). /// - /// Keep it below [`receive_window`](Self::receive_window) so one slow group - /// can't starve the rest of the connection. + /// Stream credit returns as bytes are read, whether or not their message is + /// complete, so a smaller window only adds round trips to a large frame. The + /// connection window bounds what incomplete messages hold, and when they fill + /// it the least urgent stream is cancelled. #[serde(skip_serializing_if = "Option::is_none")] #[usage( name = "quic-stream-receive-window", @@ -607,9 +614,15 @@ const MAX_VARINT: u64 = (1 << 62) - 1; /// Zero is rejected everywhere: a window of zero credits nothing, so it would wedge /// the connection rather than tighten it. The receive windows additionally have to /// fit a varint, since they are sent as transport parameters; the send window is -/// local bookkeeping and only has to be non-zero. +/// local bookkeeping and only has to be non-zero. The connection receive window has +/// to fit the largest message, which holds its credit until it is complete. #[cfg(feature = "_transport")] fn validate_windows(quic: &Config) -> crate::Result<()> { + if let Some(window) = quic.receive_window + && window < moq_net::transport::MIN_RECEIVE_WINDOW + { + return Err(crate::Error::ReceiveWindowTooSmall(moq_net::transport::MIN_RECEIVE_WINDOW)); + } for (name, window, wire) in [ ("receive_window", quic.receive_window, true), ("stream_receive_window", quic.stream_receive_window, true), @@ -655,9 +668,9 @@ pub struct Resolved { pub mtu_discovery: bool, /// Congestion control override, or `None` for the default. pub congestion_control: Option, - /// Connection-wide receive window in bytes, or `None` for the 64 MiB default. + /// Connection-wide receive window in bytes, or `None` for the 128 MiB default. pub receive_window: Option, - /// Per-stream receive window in bytes, or `None` for the backend default. + /// Per-stream receive window in bytes, or `None` to follow the connection window. pub stream_receive_window: Option, /// Cap on unacknowledged outgoing data in bytes, or `None` for the backend default. pub send_window: Option, @@ -922,13 +935,6 @@ mod tests { #[test] fn zero_windows_are_rejected() { for (name, config) in [ - ( - "receive_window", - Config { - receive_window: Some(0), - ..Default::default() - }, - ), ( "stream_receive_window", Config { @@ -981,6 +987,29 @@ mod tests { assert!(at_limit.validate().is_ok()); } + /// Incomplete messages hold connection credit, so a connection window that can't fit + /// the largest one could stall it forever. + #[cfg(feature = "_transport")] + #[test] + fn the_receive_window_fits_the_largest_message() { + let min = moq_net::transport::MIN_RECEIVE_WINDOW; + for window in [0, min - 1] { + let config = Config { + receive_window: Some(window), + ..Default::default() + }; + assert!(matches!( + config.validate(), + Err(crate::Error::ReceiveWindowTooSmall(got)) if got == min + )); + } + let config = Config { + receive_window: Some(min), + ..Default::default() + }; + assert!(config.validate().is_ok()); + } + #[test] fn window_flags_parse() { let quic = parse(&[ diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index e743af6ea7c..b3876d299f0 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -74,15 +74,15 @@ fn apply_transport(transport: &mut driver::TransportConfig, quic: &Resolved) { } /// Apply the flow-control windows. The connection window defaults to ours, since the -/// backend's is unlimited; the others stay at the backend default when unset. +/// backend's is unlimited, and the stream window follows it; the send window stays at +/// the backend default when unset. fn apply_windows(transport: &mut driver::TransportConfig, quic: &Resolved) { // Saturating rather than erroring: `Config::validate` already rejects a window // past the varint, so this only bites a `Resolved` built without it. let window = quic.receive_window.unwrap_or(crate::quic::DEFAULT_RECEIVE_WINDOW); + let stream = quic.stream_receive_window.unwrap_or(window); transport.receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); - if let Some(window) = quic.stream_receive_window { - transport.stream_receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); - } + transport.stream_receive_window(driver::VarInt::from_u64(stream).unwrap_or(driver::VarInt::MAX)); if let Some(window) = quic.send_window { transport.send_window(window); } @@ -1081,7 +1081,8 @@ mod tests { /// moq-quic exposes no getters for the flow-control windows, but its `Debug` prints /// them, which is enough to prove each one reached the transport config, that an /// unset connection window gets our finite default instead of moq-quic's unlimited one, - /// and that the other unset knobs leave moq-quic's own defaults in place. + /// that an unset stream window follows it, and that an unset send window leaves + /// moq-quic's own default in place. #[test] fn apply_windows_writes_each_field() { let quic = crate::quic::Config { @@ -1106,7 +1107,9 @@ mod tests { let mut untouched = driver::TransportConfig::default(); apply_windows(&mut untouched, &crate::quic::Config::default().resolve()); let mut expected = driver::TransportConfig::default(); - expected.receive_window(driver::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); + let window = driver::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap(); + expected.receive_window(window); + expected.stream_receive_window(window); assert_eq!(format!("{:?}", untouched), format!("{:?}", expected)); assert_ne!( format!("{:?}", untouched), diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 44f79933e7c..37e2b1f6d68 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -2,7 +2,7 @@ use std::{ any::Any, fmt, future::Future, - io, + io, mem, net::{IpAddr, SocketAddr}, pin::Pin, sync::{ @@ -629,6 +629,25 @@ impl Connection { self.0.state.lock("stats").inner.stats() } + /// Poll until held [`Credit`](crate::quic::driver::Credit) fills the receive window: the peer + /// may send nothing more and every byte it sent is held, so no stream makes progress until + /// some credit is dropped. + /// + /// Resolves once per stall, and again only after more credit is held or credit drops without + /// relieving it. Only the most recent task to poll is woken. + pub fn poll_credit_exhausted(&self, cx: &mut Context<'_>) -> Poll<()> { + let mut conn = self.0.state.lock("poll_credit_exhausted"); + // Credit may have dropped since the stall was noted. + if mem::take(&mut conn.credit_exhausted) && conn.inner.stats().credit.exhausted() { + return Poll::Ready(()); + } + match conn.credit_waker.as_mut() { + Some(waker) => waker.clone_from(cx.waker()), + None => conn.credit_waker = Some(cx.waker().clone()), + } + Poll::Pending + } + /// Current state of the congestion control algorithm, for debugging purposes pub fn congestion_state(&self) -> Box { self.0 @@ -1091,6 +1110,11 @@ pub(crate) struct State { pub(crate) blocked_writers: FxHashMap, pub(crate) blocked_readers: FxHashMap, pub(crate) stopped: FxHashMap>, + /// Held credit filled the receive window since [`Connection::poll_credit_exhausted`] last + /// resolved + credit_exhausted: bool, + /// The task watching for held credit to fill the receive window + credit_waker: Option, /// Always set to Some before the connection becomes drained pub(crate) error: Option, sender: Pin>, @@ -1126,6 +1150,8 @@ impl State { blocked_writers: FxHashMap::default(), blocked_readers: FxHashMap::default(), stopped: FxHashMap::default(), + credit_exhausted: false, + credit_waker: None, error: None, sender, send_buffer: Vec::new(), @@ -1290,6 +1316,9 @@ impl State { _ => {} } } + // Data for a stopped stream or a reset returns its credit, possibly without relieving + // a stall. + self.check_credit(); } fn drive_timer(&mut self, cx: &mut Context<'_>) -> bool { @@ -1331,6 +1360,18 @@ impl State { true } + /// Note when held credit comes to fill the receive window, waking the task watching for it. + /// + /// Call after anything that holds, returns, or discards connection credit. + pub(crate) fn check_credit(&mut self) { + if self.inner.poll_credit_exhausted() { + self.credit_exhausted = true; + if let Some(waker) = self.credit_waker.take() { + waker.wake(); + } + } + } + /// Wake up a blocked `Driver` task to process I/O pub(crate) fn wake(&mut self) { if let Some(x) = self.driver.take() { diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index bde840c96c6..d66bd1f3690 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -235,6 +235,34 @@ impl RecvStream { }) } + /// Read the next segment of data in order, keeping its connection flow control credit. + /// + /// Like [`read_chunk()`] with `ordered`, but yields the [`Credit`] for exactly the chunk's + /// bytes. See [`read_held()`] for how held credit behaves. + /// + /// This operation is cancel-safe. + /// + /// [`read_chunk()`]: RecvStream::read_chunk + /// [`read_held()`]: RecvStream::read_held + pub async fn read_chunk_held( + &mut self, + max_length: usize, + ) -> Result, ReadError> { + poll_fn(|cx| { + self.poll_read_generic(cx, true, |chunks| match chunks.next(max_length) { + Ok(Some(chunk)) => ReadStatus::Readable((chunk.bytes, chunks.hold())), + res => (None, res.err()).into(), + }) + .map_ok(|read| { + read.map(|(bytes, inner)| { + let conn = self.conn.0.clone(); + (bytes, Credit { conn, inner }) + }) + }) + }) + .await + } + /// Read the next segments of data /// /// Fills `bufs` with the segments of data beginning immediately after the @@ -308,6 +336,7 @@ impl RecvStream { return Ok(()); } conn.inner.recv_stream(self.stream).stop(error_code)?; + conn.check_credit(); conn.wake(); self.all_data_read = true; // Clean up shared state that might be left over from a cancalled read @@ -416,6 +445,7 @@ impl RecvStream { if chunks.finalize().should_transmit() { conn.wake(); } + conn.check_credit(); status } }; @@ -538,6 +568,7 @@ impl Drop for Credit { { conn.wake(); } + conn.check_credit(); } } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 4b63539caf3..2cd9914ba17 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1902,6 +1902,51 @@ async fn held_reads_spend_the_connection_window() { writer.await.unwrap(); } +/// Held chunks filling the window resolve `poll_credit_exhausted` once, and plain reads after the +/// credit drops never do. +#[tokio::test] +async fn filling_the_window_with_held_credit_is_reported_once() { + const WINDOW: usize = 32 * 1024; + let _guard = subscribe(); + let mut transport = TransportConfig::default(); + transport.receive_window((WINDOW as u32).into()); + let endpoint = endpoint_with_config(transport); + + let (client, server) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let mut send = client.open_uni().await.unwrap(); + let writer = tokio::spawn(async move { + // The receiver walks away before this finishes. + let _ = send.write_all(&[7; 4 * WINDOW]).await; + }); + + let (waker, counter) = new_count_waker(); + let mut cx = Context::from_waker(&waker); + assert!(server.poll_credit_exhausted(&mut cx).is_pending()); + + let mut recv = server.accept_uni().await.unwrap(); + let mut held = recv.read_chunk_held(usize::MAX).await.unwrap().unwrap().1; + while (held.len() as usize) < WINDOW { + held.merge(recv.read_chunk_held(usize::MAX).await.unwrap().unwrap().1); + } + assert_eq!(counter.wakes(), 1); + assert!(server.poll_credit_exhausted(&mut cx).is_ready()); + assert!(server.poll_credit_exhausted(&mut cx).is_pending()); + + drop(held); + let mut buf = vec![0; WINDOW]; + recv.read_exact(&mut buf).await.unwrap(); + assert!(server.poll_credit_exhausted(&mut cx).is_pending()); + assert_eq!(counter.wakes(), 1); + drop(recv); + writer.await.unwrap(); +} + /// Fill `buf` with held reads, returning the credit for all of it. async fn read_held_exact(recv: &mut RecvStream, buf: &mut [u8]) -> Credit { let mut held = recv.read_held(buf).await.unwrap().unwrap(); diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 1e46bac784b..c1fdeefeff6 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -67,6 +67,18 @@ impl RecvStream { .map_err(|e| self.map_error(e)) } + /// Read a chunk of data, keeping its connection flow control credit until the credit is + /// dropped. See [`crate::quic::driver::RecvStream::read_chunk_held`]. + pub async fn read_chunk_held( + &mut self, + max_length: usize, + ) -> Result, ReadError> { + self.inner + .read_chunk_held(max_length) + .await + .map_err(|e| self.map_error(e)) + } + /// Read until the end of the stream or the limit is hit. See [`crate::quic::driver::RecvStream::read_to_end`]. pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { self.inner.read_to_end(size_limit).await.map_err(|e| match e { diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 370e5d817f2..43b91e3a545 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -991,7 +991,7 @@ async fn accept_request( deadline: Option, ) -> crate::Result<(PendingRequest>, Option)> where - S: web_transport_trait::Session, + S: crate::transport::HeldSession, crate::transport::Session: moq_net::transport::poll::Boxable, as moq_net::transport::poll::Session>::SendStream: moq_net::transport::MaybeSync, as moq_net::transport::poll::Session>::RecvStream: moq_net::transport::MaybeSync, @@ -2231,6 +2231,8 @@ mod tests { } #[cfg(feature = "websocket")] + crate::transport::held::unheld!(Scripted, ScriptedRecv); + impl web_transport_trait::Session for Scripted { type SendStream = ScriptedSend; type RecvStream = ScriptedRecv; diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 5ab97f52a9e..81e177180ac 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -14,11 +14,60 @@ use std::task::{Context, Poll, ready}; use bytes::Bytes; use futures::FutureExt; +use moq_net::transport::Credit as _; use web_transport_trait::poll as wt_poll; +pub(crate) mod held; mod owned; pub use owned::Error; +/// A backend receive stream that can keep the connection credit of what it reads. +/// +/// A backend whose reads return their credit straight away reads plain chunks and counts them +/// as [`Unheld`](moq_net::transport::Unheld). +pub trait HeldRecvStream: web_transport_trait::RecvStream { + /// The connection credit a read keeps until it is dropped. + type Credit: moq_net::transport::Credit + Send + Sync; + + /// Read the next chunk, up to `max` bytes, with its credit, or `None` once finished. + fn read_chunk_held( + &mut self, + max: usize, + ) -> impl Future, Self::Error>> + Send; +} + +/// A backend session that reports when held credit fills its receive window. +pub trait HeldSession: web_transport_trait::Session { + /// See [`poll_credit_exhausted`](moq_net::transport::poll::Session::poll_credit_exhausted). + fn poll_credit_exhausted(&self, cx: &mut Context<'_>) -> Poll<()>; +} + +/// The connection credit for bytes read through a [`RecvStream`], none for an empty read. +#[derive(Debug)] +pub struct Credit(Option); + +impl moq_net::transport::Credit for Credit { + fn len(&self) -> u64 { + self.0.as_ref().map_or(0, |credit| credit.len()) + } + + fn split(&mut self, bytes: u64) -> Self { + match &mut self.0 { + _ if bytes == 0 => Self(None), + Some(credit) => Self(Some(credit.split(bytes))), + None => panic!("split {bytes} of 0 bytes"), + } + } + + fn merge(&mut self, other: Self) { + match (&mut self.0, other.0) { + (_, None) => {} + (Some(credit), Some(other)) => credit.merge(other), + (empty, other) => *empty = other, + } + } +} + /// The network transport carrying a MoQ session, on either side of it. #[non_exhaustive] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -157,7 +206,7 @@ impl Clone for Session { impl wt_poll::Session for Session where - S: web_transport_trait::Session, + S: web_transport_trait::Session, S::SendStream: 'static, S::RecvStream: 'static, { @@ -588,16 +637,17 @@ impl Drop for SendStream { /// The receive half produced by [`Session`]: an async-interface stream adapted to /// the poll interface. See [`Session`] for the deferred `stop` behavior. -pub struct RecvStream { +pub struct RecvStream { // See [`SendStream::state`] for why this Mutex exists; it is never locked. state: std::sync::Mutex>>, - /// Bytes already read from the transport but not yet handed to the caller. - buffer: Bytes, + /// A chunk already read from the transport but not yet handed to the caller, never empty, + /// with the credit it keeps. + buffer: Option<(Bytes, S::Credit)>, /// Read-ahead pulled in by the closed-watch while payload sat undelivered, so /// a FIN right behind buffered data still resolves the watch. Drained by the /// read methods before the transport is polled again; bounded by - /// [`READ_AHEAD_CAP`]. - queued: std::collections::VecDeque, + /// [`READ_AHEAD_CAP`]. It keeps its credit until the caller reads it. + queued: std::collections::VecDeque<(Bytes, S::Credit)>, /// Total bytes across `queued`, so the cap check is O(1). queued_len: usize, /// The peer finished the stream. @@ -615,19 +665,22 @@ pub struct RecvStream { /// stream nobody is reading. const READ_AHEAD_CAP: usize = 64 * 1024; -enum RecvState { +/// A chunk read from a backend with its credit, or `None` once finished. +type Held = Option<(Bytes, ::Credit)>; + +enum RecvState { Idle(S), /// A read in flight; a `None` result means it was interrupted by a stop /// (see [`RecvStream::interrupt`]). - Reading(#[allow(clippy::type_complexity)] OpBox<(S, Option, S::Error>>)>), + Reading(#[allow(clippy::type_complexity)] OpBox<(S, Option, S::Error>>)>), } -impl RecvStream { +impl RecvStream { /// Wrap an async-interface receive stream. pub fn new(stream: S) -> Self { Self { state: std::sync::Mutex::new(Some(RecvState::Idle(stream))), - buffer: Bytes::new(), + buffer: None, queued: std::collections::VecDeque::new(), queued_len: 0, fin: false, @@ -636,18 +689,49 @@ impl RecvStream { } } - /// Pop read-ahead into `self.buffer` if the head is empty. - fn unqueue(&mut self) { - if self.buffer.is_empty() - && let Some(next) = self.queued.pop_front() - { - self.queued_len -= next.len(); - self.buffer = next; + /// Take up to `max` buffered bytes with their credit, read-ahead first. + fn take(&mut self, max: usize) -> Option<(Bytes, S::Credit)> { + if self.buffer.is_none() { + let next = self.queued.pop_front()?; + self.queued_len -= next.0.len(); + self.buffer = Some(next); + } + let (bytes, credit) = self.buffer.as_mut().expect("just filled"); + if bytes.len() <= max { + return self.buffer.take(); + } + Some((bytes.split_to(max), credit.split(max as u64))) + } + + /// Read up to `max` bytes with their credit, or `None` once the peer finished the stream. + #[allow(clippy::type_complexity)] + pub(crate) fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll)>, S::Error>> { + if max == 0 { + return Poll::Ready(Ok(Some((Bytes::new(), Credit(None))))); + } + + loop { + if let Some((bytes, credit)) = self.take(max) { + return Poll::Ready(Ok(Some((bytes, Credit(Some(credit)))))); + } + if self.fin { + return Poll::Ready(Ok(None)); + } + // An empty chunk carries nothing worth buffering. + if let Some(chunk) = ready!(self.poll_fill(cx, max))? + && !chunk.0.is_empty() + { + self.buffer = Some(chunk); + } } } /// Read the next chunk from the transport, setting `self.fin` on the FIN. - fn poll_fill(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, S::Error>> { + fn poll_fill(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, S::Error>> { loop { match self.state.get_mut().unwrap().take().expect("in-flight") { RecvState::Idle(mut stream) => { @@ -658,7 +742,7 @@ impl RecvStream { self.interrupt = Some(tx); let fut = async move { let res = { - let mut op = std::pin::pin!(stream.read_chunk(max).fuse()); + let mut op = std::pin::pin!(stream.read_chunk_held(max).fuse()); let mut rx = rx.fuse(); loop { futures::select_biased! { @@ -706,48 +790,20 @@ impl RecvStream { } } -impl wt_poll::RecvStream for RecvStream { +// This view returns connection credit as it reads; moq-net reads through the held one. +impl wt_poll::RecvStream for RecvStream { type Error = S::Error; fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - if dst.is_empty() { - return Poll::Ready(Ok(Some(0))); - } - - loop { - self.unqueue(); - if !self.buffer.is_empty() { - let n = dst.len().min(self.buffer.len()); - dst[..n].copy_from_slice(&self.buffer.split_to(n)); - return Poll::Ready(Ok(Some(n))); - } - if self.fin { - return Poll::Ready(Ok(None)); - } - if let Some(bytes) = ready!(self.poll_fill(cx, dst.len()))? { - self.buffer = bytes; - } - } + let read = ready!(self.poll_read_held(cx, dst.len()))?; + Poll::Ready(Ok(read.map(|(bytes, _)| { + dst[..bytes.len()].copy_from_slice(&bytes); + bytes.len() + }))) } fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); - } - - loop { - self.unqueue(); - if !self.buffer.is_empty() { - let n = max.min(self.buffer.len()); - return Poll::Ready(Ok(Some(self.buffer.split_to(n)))); - } - if self.fin { - return Poll::Ready(Ok(None)); - } - if let Some(bytes) = ready!(self.poll_fill(cx, max))? { - self.buffer = bytes; - } - } + self.poll_read_held(cx, max).map_ok(|read| read.map(|(bytes, _)| bytes)) } fn stop(&mut self, code: u32) { @@ -775,15 +831,15 @@ impl wt_poll::RecvStream for RecvS if self.fin { return Poll::Ready(Ok(())); } - if self.buffer.len() + self.queued_len >= READ_AHEAD_CAP { + if self.buffer.as_ref().map_or(0, |(bytes, _)| bytes.len()) + self.queued_len >= READ_AHEAD_CAP { // The caller's own reads are what drain the backlog, and their // re-poll of this watch is what resumes it. return Poll::Pending; } match ready!(self.poll_fill(cx, 8 * 1024)) { - Ok(Some(bytes)) => { + Ok(Some((bytes, credit))) => { self.queued_len += bytes.len(); - self.queued.push_back(bytes); + self.queued.push_back((bytes, credit)); } Ok(None) => {} // A read error is also a closed stream. @@ -793,7 +849,7 @@ impl wt_poll::RecvStream for RecvS } } -impl Drop for RecvStream { +impl Drop for RecvStream { fn drop(&mut self) { match self.state.get_mut().unwrap().take() { // Apply a deferred stop if the stream is idle. @@ -1282,6 +1338,8 @@ mod tests { #[derive(Clone)] struct FakeSession; + super::held::unheld!(FakeSession, FakeRecv); + impl web_transport_trait::Session for FakeSession { type SendStream = FakeSend; type RecvStream = FakeRecv; diff --git a/rs/moq-tokio/src/transport/held.rs b/rs/moq-tokio/src/transport/held.rs new file mode 100644 index 00000000000..f5e351e8f61 --- /dev/null +++ b/rs/moq-tokio/src/transport/held.rs @@ -0,0 +1,80 @@ +//! How each backend reads with connection credit. + +use bytes::Bytes; +use moq_net::transport::Unheld; + +/// Read a plain chunk, whose connection credit the backend returned as it read. +pub(crate) async fn read_unheld( + stream: &mut S, + max: usize, +) -> Result, S::Error> { + let chunk = stream.read_chunk(max).await?; + Ok(chunk.map(|bytes| { + let len = bytes.len() as u64; + (bytes, Unheld(len)) + })) +} + +/// Give a backend that returns connection credit as it reads, so never fills its window with +/// held credit, the held interface. +macro_rules! unheld { + ($session:ty, $recv:ty) => { + impl $crate::transport::HeldRecvStream for $recv { + type Credit = moq_net::transport::Unheld; + + fn read_chunk_held( + &mut self, + max: usize, + ) -> impl Future, Self::Error>> + Send { + $crate::transport::held::read_unheld(self, max) + } + } + + impl $crate::transport::HeldSession for $session { + fn poll_credit_exhausted(&self, _: &mut std::task::Context<'_>) -> std::task::Poll<()> { + std::task::Poll::Pending + } + } + }; +} +#[cfg(test)] +pub(crate) use unheld; + +#[cfg(any(feature = "tcp", feature = "websocket"))] +unheld!(qmux::Session, qmux::RecvStream); +#[cfg(feature = "iroh")] +unheld!(web_transport_iroh::Session, web_transport_iroh::RecvStream); + +#[cfg(feature = "quic")] +impl moq_net::transport::Credit for crate::quic::driver::Credit { + fn len(&self) -> u64 { + self.len() + } + + fn split(&mut self, bytes: u64) -> Self { + self.split(bytes) + } + + fn merge(&mut self, other: Self) { + self.merge(other) + } +} + +#[cfg(feature = "quic")] +impl super::HeldRecvStream for crate::quic::web::RecvStream { + type Credit = crate::quic::driver::Credit; + + fn read_chunk_held( + &mut self, + max: usize, + ) -> impl Future, Self::Error>> + Send { + crate::quic::web::RecvStream::read_chunk_held(self, max) + } +} + +#[cfg(feature = "quic")] +impl super::HeldSession for crate::quic::web::Session { + fn poll_credit_exhausted(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + crate::quic::driver::Connection::poll_credit_exhausted(self, cx) + } +} diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 225574189e8..bd20cb840db 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -1,6 +1,6 @@ use bytes::{Buf, BufMut, Bytes}; use moq_net::transport::{self as moq, poll}; -use std::task::{Context, Poll}; +use std::task::{Context, Poll, ready}; use web_transport_trait::{self as wt, poll as backend}; /// A backend failure preserving the session and stream code registries. @@ -44,7 +44,7 @@ impl moq::Stats for Stats { self.0.estimated_send_rate() } } -impl> poll::Session for super::Session { +impl> poll::Session for super::Session { type SendStream = super::SendStream; type RecvStream = super::RecvStream; type Error = Error; @@ -81,8 +81,8 @@ impl> poll::Session for fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(self, cx).map(Error) } - fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { - Poll::Pending + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + super::HeldSession::poll_credit_exhausted(&self.session, cx) } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(self)) @@ -109,32 +109,35 @@ impl poll::SendStream for super::SendStream { backend::SendStream::poll_closed(self, cx).map_err(Error) } } -// The WebTransport backend returns connection credit as it reads, so there is none to hold. -impl poll::RecvStream for super::RecvStream { +impl poll::RecvStream for super::RecvStream { type Error = Error; - type Credit = moq::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(self, cx, dst) - .map_err(Error) - .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) + type Credit = super::Credit; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + let read = ready!(self.poll_read_held(cx, dst.len())).map_err(Error)?; + Poll::Ready(Ok(read.map(|(bytes, credit)| { + dst[..bytes.len()].copy_from_slice(&bytes); + credit + }))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(self, cx, buf) - .map_err(Error) - .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) + ) -> Poll, Self::Error>> { + // Hand over the chunk the backend owns, without zeroing a destination first. + let max = buf.remaining_mut().min(64 * 1024); + let read = ready!(self.poll_read_held(cx, max)).map_err(Error)?; + Poll::Ready(Ok(read.map(|(bytes, credit)| { + buf.put(bytes); + credit + }))) } fn poll_read_chunk( &mut self, cx: &mut Context<'_>, max: usize, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(self, cx, max) - .map_err(Error) - .map_ok(|chunk| chunk.map(|b| (b.clone(), moq::Unheld(b.len() as u64)))) + ) -> Poll, Self::Error>> { + self.poll_read_held(cx, max).map_err(Error) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(self, code) diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index e08488a9572..3fc4a193d38 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -1328,16 +1328,17 @@ async fn connect_once( // ── flow control ──────────────────────────────────────────────────── -/// Run a connect with every flow-control window pinned well under the payload, so -/// the frame only arrives if the receiver keeps issuing credit as it drains. +/// Run a connect with the stream and send windows pinned well under the payload, so +/// the frame only arrives if the receiver keeps issuing stream credit as it drains. +/// The connection window sits at the smallest that fits the largest message, which +/// holds its credit until it is complete. /// /// The size is the point: a frame that fits inside one window would pass whether or /// not the setting ever reached the backend. -/// #[cfg(feature = "quic")] async fn window_test(scheme: &str) { let mut quic = moq_tokio::quic::Config::default(); - quic.receive_window = Some(64 * 1024); + quic.receive_window = Some(moq_net::transport::MIN_RECEIVE_WINDOW); quic.stream_receive_window = Some(16 * 1024); quic.send_window = Some(32 * 1024); diff --git a/rs/moq-tokio/tests/receive_credit.rs b/rs/moq-tokio/tests/receive_credit.rs new file mode 100644 index 00000000000..4bd54161e7d --- /dev/null +++ b/rs/moq-tokio/tests/receive_credit.rs @@ -0,0 +1,246 @@ +//! Over native QUIC, incomplete frames hold the receiver's connection credit, and when +//! they fill its window the receiver stops the least urgent stream and keeps going. +//! +//! The server publishes with the stock config. The client dials over the raw driver so +//! its connection window can be far smaller than the stack allows, which keeps the +//! payloads small. Without recovery the high-priority frame never arrives: the window +//! is spent on a low-priority frame that can never complete. + +#![cfg(feature = "quic")] + +use std::{sync::Arc, time::Duration}; + +use bytes::Bytes; +use moq_tokio::moq_net::{self, Timestamp, frame, track::Subscription}; +use moq_tokio::quic::driver; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// The client's connection receive window. +const WINDOW: usize = 256 * 1024; + +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +/// A server publishing broadcast `room`, and a client subscribed to its origin over a +/// connection whose receive window is [`WINDOW`]. +struct Setup { + broadcast: moq_net::broadcast::Producer, + remote: moq_net::broadcast::Consumer, + _client: moq_net::Session, + _endpoint: driver::Endpoint, +} + +async fn setup(version: &str) -> Setup { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let version: moq_net::Version = version.parse().expect("version"); + + let publisher = moq_tokio::origin::spawn(); + let broadcast = publisher.create_broadcast("room").expect("broadcast"); + broadcast.announce(Default::default()).expect("announce"); + + let mut config = moq_tokio::listen::Config::default(); + config.bind = Some("127.0.0.1:0".parse().unwrap()); + config.tls.generate = vec!["localhost".into()]; + config.version = vec![version]; + let mut server = config + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("listen"); + let port = server.local_addr().expect("addr").port(); + tokio::spawn(async move { + let request = server.accept().await.expect("accept"); + let (session, _driver) = request.with_publisher(&publisher).ok().await.expect("session"); + let _ = session.closed().await; + }); + + let mut tls = rustls::ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyCert)) + .with_no_client_auth(); + tls.alpn_protocols = vec![version.alpn().as_bytes().to_vec()]; + let quic = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + let mut transport = driver::TransportConfig::default(); + transport.receive_window((WINDOW as u32).into()); + let mut client = driver::ClientConfig::new(Arc::new(quic)); + client.transport_config(Arc::new(transport)); + + let endpoint = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); + let conn = endpoint + .connect_with(client, ([127, 0, 0, 1], port).into(), "localhost") + .expect("connect") + .await + .expect("handshake"); + + let subscriber = moq_tokio::origin::spawn(); + let session = moq_tokio::transport::Session::new( + moq_tokio::quic::web::Session::raw(conn), + moq_net::transport::Binding::InBand, + ); + let (client, driver) = moq_net::Client::new() + .with_versions(version.into()) + .with_subscriber(subscriber.clone()) + .connect(std::time::Instant::now(), session) + .await + .expect("client handshake"); + tokio::spawn(moq_net::time::run(driver)); + + let consumer = subscriber.consume(); + tokio::time::timeout(TIMEOUT, consumer.routed("room")) + .await + .expect("announce timeout") + .expect("routed"); + let remote = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("room", None)) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + Setup { + broadcast, + remote, + _client: client, + _endpoint: endpoint, + } +} + +impl Setup { + /// Subscribe to a new track at `priority`. + async fn subscribe(&self, name: &str, priority: u8) -> (moq_net::track::Producer, moq_net::track::Subscriber) { + let track = self.broadcast.create_track(name, timed()).unwrap(); + let sub = tokio::time::timeout( + TIMEOUT, + self.remote + .track(name) + .unwrap() + .subscribe(Subscription::default().with_priority(priority)), + ) + .await + .expect("subscribe timeout") + .expect("subscribe"); + (track, sub) + } +} + +/// The next group, which must arrive in time. +async fn next_group(sub: &mut moq_net::track::Subscriber) -> moq_net::group::Consumer { + tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("group timed out") + .expect("group") + .expect("track ended") +} + +/// The next frame's payload, which must arrive in time. +async fn next_frame(sub: &mut moq_net::track::Subscriber) -> Bytes { + let mut group = next_group(sub).await; + tokio::time::timeout(TIMEOUT, group.read_frame()) + .await + .expect("frame timed out") + .expect("frame") + .expect("group ended") + .payload +} + +/// Two partial frames that together exceed the window: the lower-priority one is +/// stopped, and the higher-priority one completes on the credit it returns. +async fn the_less_urgent_partial_frame_gives_way(version: &str) { + let setup = setup(version).await; + let (low, mut low_sub) = setup.subscribe("low", 1).await; + let (high, mut high_sub) = setup.subscribe("high", 200).await; + + // Three quarters of the window, held by a frame that never finishes. + let mut low_group = low.append_group().unwrap(); + let mut low_frame = low_group + .create_frame(frame::Info { + size: WINDOW as u64, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + low_frame.write(vec![1u8; WINDOW * 3 / 4]).unwrap(); + + // Wait until the receiver holds all of it, so nothing else got there first. + let mut partial = next_group(&mut low_sub).await; + let mut partial = tokio::time::timeout(TIMEOUT, partial.next_frame()) + .await + .expect("frame header timed out") + .expect("frame") + .expect("group ended"); + let mut seen = 0; + while seen < WINDOW * 3 / 4 { + let chunk = tokio::time::timeout(TIMEOUT, partial.read_chunk()) + .await + .expect("partial payload timed out") + .expect("chunk") + .expect("frame ended"); + seen += chunk.len(); + } + + // Half the window, which only fits once the low frame lets go. + let payload = Bytes::from(vec![2u8; WINDOW / 2]); + let mut high_group = high.append_group().unwrap(); + high_group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + high_group.finish().unwrap(); + assert_eq!(next_frame(&mut high_sub).await, payload, "{version}"); + + // The session survives, and the low track keeps delivering what fits. + drop(low_frame); + drop(low_group); + let mut group = low.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, &b"after"[..]).unwrap(); + group.finish().unwrap(); + assert_eq!(next_frame(&mut low_sub).await, &b"after"[..], "{version}"); +} + +#[tokio::test] +async fn lite_partial_frame_gives_way() { + the_less_urgent_partial_frame_gives_way("moq-lite-06").await; +} + +#[tokio::test] +async fn ietf_partial_frame_gives_way() { + the_less_urgent_partial_frame_gives_way("moq-transport-16").await; +} + +/// Accepts any server certificate: the test dials its own self-signed server. +#[derive(Debug)] +struct AnyCert; + +impl rustls::client::danger::ServerCertVerifier for AnyCert { + fn verify_server_cert( + &self, + _: &rustls::pki_types::CertificateDer<'_>, + _: &[rustls::pki_types::CertificateDer<'_>], + _: &rustls::pki_types::ServerName<'_>, + _: &[u8], + _: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _: &[u8], + _: &rustls::pki_types::CertificateDer<'_>, + _: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _: &[u8], + _: &rustls::pki_types::CertificateDer<'_>, + _: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(rustls::client::danger::HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} From 5ed1dd31155a68234bab91e9a3aa91568283fa76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:24:07 -0700 Subject: [PATCH 26/51] refactor(moq-tokio): fold held reads into read and read_chunk A loopback benchmark finds handing each read's credit to the caller, which drops it under the connection lock, within noise of returning it inside the read, so the driver keeps one read of each kind. read and read_chunk yield the credit; the convenience reads return it as they go. Also document the new receive window rules for the relay. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 18 ++- rs/moq-relay/src/config.rs | 4 +- rs/moq-tokio/src/quic/driver/recv_stream.rs | 164 +++++--------------- rs/moq-tokio/src/quic/driver/tests.rs | 80 +++++++++- rs/moq-tokio/src/quic/web/recv.rs | 28 ++-- rs/moq-tokio/src/transport/held.rs | 2 +- rs/moq-uring/tests/web.rs | 6 +- sh/gh/nightly.sh | 1 + 8 files changed, 143 insertions(+), 160 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 9bd91a4a5e3..cc17276d0cc 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -61,8 +61,8 @@ idle_timeout = "10s" # Drop a connection after this long with no keep_alive = "3s" # Ping interval; "0s" disables it. iroh still pings at least every 5s. Default. gso = true # UDP segmentation offload. iroh cannot turn it off. mtu_discovery = false # Path MTU discovery. Default. -receive_window = 67108864 # Whole-connection window, in bytes. Default (64 MiB). -stream_receive_window = 8388608 # Per-stream window. Omit for the backend default. +receive_window = 134217728 # Whole-connection window, in bytes. Default (128 MiB). +stream_receive_window = 8388608 # Per-stream window. Omit to follow receive_window. send_window = 33554432 # Unacknowledged outgoing data. Omit for the backend default. qlog = "/var/log/moq/qlog" # Existing directory. Needs the `qlog` build feature. ``` @@ -72,11 +72,15 @@ sessions, and so its [cluster routes](/bin/relay/cluster#failure-detection). QUIC uses the smaller of the two endpoints' values, so this also bounds the other end. Keep `keep_alive` under a third of it. -Raise the receive windows when a fat, long path idles below the link rate. The -64 MiB `receive_window` default carries several Gbps at a 100 ms RTT and bounds -how much unread data one peer can make the relay buffer. Keep -`stream_receive_window` well under it so one slow group cannot starve the -connection. +Raise `receive_window` when a fat, long path idles below the link rate. The +128 MiB default carries several Gbps at a 100 ms RTT and bounds how much unread +data and incomplete messages one peer can make the relay buffer. A message holds +its share until it is complete, so the window must fit the largest one, just over +64 MiB, and the relay refuses to start with less. When incomplete messages fill +it, the relay cancels the least urgent stream and carries on. + +Stream credit returns as data is read, so a `stream_receive_window` under +`receive_window` only adds round trips to a large frame. ## \[session] and \[peer] diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 707ffbc3795..52c41bf1ded 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -920,7 +920,7 @@ congestion_control = "delay" let toml = r#" [quic] -receive_window = 67108864 +receive_window = 134217728 stream_receive_window = 8388608 send_window = 33554432 "#; @@ -932,7 +932,7 @@ send_window = 33554432 let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = Config::parse_and_merge(args).expect("config load"); - assert_eq!(config.quic.receive_window, Some(67108864)); + assert_eq!(config.quic.receive_window, Some(134217728)); assert_eq!(config.quic.stream_receive_window, Some(8388608)); assert_eq!( config.quic.send_window, diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index d66bd1f3690..6b970ea0c16 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -75,15 +75,14 @@ impl RecvStream { /// Read data contiguously from the stream. /// - /// Yields the number of bytes read into `buf` on success, or `None` if the stream was finished. + /// Yields the [`Credit`] for exactly the bytes read into `buf`, which [`Credit::len()`] + /// counts, or `None` if the stream was finished. Stream credit advances on the read, while + /// the connection window these bytes occupy stays spent until the credit is dropped, so drop + /// it once the bytes are consumed. /// /// This operation is cancel-safe. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - Read { - stream: self, - buf: ReadBuf::new(buf), - } - .await + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + poll_fn(|cx| self.poll_read(cx, buf)).await } /// Read an exact number of bytes contiguously from the stream. @@ -99,27 +98,16 @@ impl RecvStream { .await } - /// Read data contiguously from the stream, keeping its connection flow control credit. + /// Attempts to read from the stream into the provided buffer /// - /// Like [`read()`], but yields the [`Credit`] for exactly the bytes read into `buf`, or `None` - /// if the stream was finished. Stream credit still advances, so a stream window smaller than - /// an application message keeps the message flowing, while the connection window these bytes - /// occupy stays spent until the credit is dropped. + /// See [`read()`] for what the yielded credit means. /// - /// This operation is cancel-safe. + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. /// /// [`read()`]: RecvStream::read - pub async fn read_held(&mut self, buf: &mut [u8]) -> Result, ReadError> { - poll_fn(|cx| self.poll_read_held(cx, buf)).await - } - - /// Attempts to read from the stream into the provided buffer, keeping the bytes' connection - /// flow control credit. - /// - /// See [`read_held()`] for details. - /// - /// [`read_held()`]: RecvStream::read_held - pub fn poll_read_held( + pub fn poll_read( &mut self, cx: &mut Context<'_>, buf: &mut [u8], @@ -143,33 +131,12 @@ impl RecvStream { }) } - /// Attempts to read from the stream into the provided buffer - /// - /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this - /// returns zero bytes read (and `buf` has a non-zero length), that indicates that the remote - /// side has [`finish`]ed the stream and the local side has already read all bytes. - /// - /// If no data is available for reading, this returns `Poll::Pending` and arranges for the - /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is - /// closed. - /// - /// [`finish`]: crate::quic::driver::SendStream::finish - pub fn poll_read( - &mut self, - cx: &mut Context<'_>, - buf: &mut [u8], - ) -> Poll> { - let mut buf = ReadBuf::new(buf); - ready!(self.poll_read_buf(cx, &mut buf))?; - Poll::Ready(Ok(buf.filled().len())) - } - /// Attempts to read from the stream into the provided buffer, which may be uninitialized /// /// On success, returns `Poll::Ready(Ok(()))` and places data into the unfilled portion of /// `buf`. If this does not write any bytes to `buf` (and `buf.remaining()` is non-zero), that /// indicates that the remote side has [`finish`]ed the stream and the local side has already - /// read all bytes. + /// read all bytes. The bytes' connection credit returns as they are read. /// /// If no data is available for reading, this returns `Poll::Pending` and arranges for the /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is @@ -192,30 +159,40 @@ impl RecvStream { /// Read the next segment of data /// /// Yields `None` if the stream was finished. Otherwise, yields a segment of data and its - /// offset in the stream. If `ordered` is `true`, the chunk's offset will be immediately after - /// the last data yielded by `read()` or `read_chunk()`. If `ordered` is `false`, segments may - /// be received in any order, and the `Chunk`'s `offset` field can be used to determine - /// ordering in the caller. Unordered reads are less prone to head-of-line blocking within a - /// stream, but require the application to manage reassembling the original data. + /// offset in the stream, with the [`Credit`] for its bytes (see [`read()`]). If `ordered` is + /// `true`, the chunk's offset will be immediately after the last data yielded by `read()` or + /// `read_chunk()`. If `ordered` is `false`, segments may be received in any order, and the + /// `Chunk`'s `offset` field can be used to determine ordering in the caller. Unordered reads + /// are less prone to head-of-line blocking within a stream, but require the application to + /// manage reassembling the original data. /// /// Slightly more efficient than `read` due to not copying. Chunk boundaries do not correspond /// to peer writes, and hence cannot be used as framing. /// /// This operation is cancel-safe. + /// + /// [`read()`]: RecvStream::read pub async fn read_chunk( &mut self, max_length: usize, ordered: bool, - ) -> Result, ReadError> { - ReadChunk { - stream: self, - max_length, - ordered, - } + ) -> Result, ReadError> { + poll_fn(|cx| { + self.poll_read_generic(cx, ordered, |chunks| match chunks.next(max_length) { + Ok(Some(chunk)) => ReadStatus::Readable((chunk, chunks.hold())), + res => (None, res.err()).into(), + }) + .map_ok(|read| { + read.map(|(chunk, inner)| { + let conn = self.conn.0.clone(); + (chunk, Credit { conn, inner }) + }) + }) + }) .await } - /// Attempts to read a chunk from the stream. + /// Attempts to read a chunk from the stream, returning its credit as it reads. /// /// On success, returns `Poll::Ready(Ok(Some(chunk)))`. If `Poll::Ready(Ok(None))` /// is returned, it implies that EOF has been reached. @@ -223,7 +200,7 @@ impl RecvStream { /// If no data is available for reading, the method returns `Poll::Pending` /// and arranges for the current task (via cx.waker()) to receive a notification /// when the stream becomes readable or is closed. - fn poll_read_chunk( + pub(super) fn poll_read_chunk( &mut self, cx: &mut Context<'_>, max_length: usize, @@ -235,34 +212,6 @@ impl RecvStream { }) } - /// Read the next segment of data in order, keeping its connection flow control credit. - /// - /// Like [`read_chunk()`] with `ordered`, but yields the [`Credit`] for exactly the chunk's - /// bytes. See [`read_held()`] for how held credit behaves. - /// - /// This operation is cancel-safe. - /// - /// [`read_chunk()`]: RecvStream::read_chunk - /// [`read_held()`]: RecvStream::read_held - pub async fn read_chunk_held( - &mut self, - max_length: usize, - ) -> Result, ReadError> { - poll_fn(|cx| { - self.poll_read_generic(cx, true, |chunks| match chunks.next(max_length) { - Ok(Some(chunk)) => ReadStatus::Readable((chunk.bytes, chunks.hold())), - res => (None, res.err()).into(), - }) - .map_ok(|read| { - read.map(|(bytes, inner)| { - let conn = self.conn.0.clone(); - (bytes, Credit { conn, inner }) - }) - }) - }) - .await - } - /// Read the next segments of data /// /// Fills `bufs` with the segments of data beginning immediately after the @@ -511,7 +460,8 @@ fn fill(chunks: &mut Chunks<'_>, buf: &mut ReadBuf<'_>) -> ReadStatus<()> { } } -/// Connection flow control credit for bytes read with [`RecvStream::read_held`]. +/// Connection flow control credit for bytes read with [`RecvStream::read`] or +/// [`RecvStream::read_chunk`]. /// /// The peer cannot reuse the connection window these bytes occupy until this is dropped, which /// returns it even after the stream or connection is gone. @@ -795,27 +745,6 @@ impl From for io::Error { } } -/// Future produced by [`RecvStream::read()`]. -/// -/// [`RecvStream::read()`]: crate::quic::driver::RecvStream::read -struct Read<'a> { - stream: &'a mut RecvStream, - buf: ReadBuf<'a>, -} - -impl Future for Read<'_> { - type Output = Result, ReadError>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; - match this.buf.filled().len() { - 0 if this.buf.capacity() != 0 => Poll::Ready(Ok(None)), - n => Poll::Ready(Ok(Some(n))), - } - } -} - /// Future produced by [`RecvStream::read_exact()`]. /// /// [`RecvStream::read_exact()`]: crate::quic::driver::RecvStream::read_exact @@ -853,23 +782,6 @@ pub enum ReadExactError { ReadError(#[from] ReadError), } -/// Future produced by [`RecvStream::read_chunk()`]. -/// -/// [`RecvStream::read_chunk()`]: crate::quic::driver::RecvStream::read_chunk -struct ReadChunk<'a> { - stream: &'a mut RecvStream, - max_length: usize, - ordered: bool, -} - -impl Future for ReadChunk<'_> { - type Output = Result, ReadError>; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let (max_length, ordered) = (self.max_length, self.ordered); - self.stream.poll_read_chunk(cx, max_length, ordered) - } -} - /// Future produced by [`RecvStream::read_chunks()`]. /// /// [`RecvStream::read_chunks()`]: crate::quic::driver::RecvStream::read_chunks diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 2cd9914ba17..b0025e51593 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1930,9 +1930,9 @@ async fn filling_the_window_with_held_credit_is_reported_once() { assert!(server.poll_credit_exhausted(&mut cx).is_pending()); let mut recv = server.accept_uni().await.unwrap(); - let mut held = recv.read_chunk_held(usize::MAX).await.unwrap().unwrap().1; + let mut held = recv.read_chunk(usize::MAX, true).await.unwrap().unwrap().1; while (held.len() as usize) < WINDOW { - held.merge(recv.read_chunk_held(usize::MAX).await.unwrap().unwrap().1); + held.merge(recv.read_chunk(usize::MAX, true).await.unwrap().unwrap().1); } assert_eq!(counter.wakes(), 1); assert!(server.poll_credit_exhausted(&mut cx).is_ready()); @@ -1947,12 +1947,84 @@ async fn filling_the_window_with_held_credit_is_reported_once() { writer.await.unwrap(); } +/// Stream 256 MiB over loopback, reading chunks whose credit returns inside the read, or drops +/// straight after it as `read_chunk` does: the per-read cost of handing credit to the caller. +/// Run by nightly. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "release benchmark, run by nightly"] +async fn benchmark_held_reads() { + const BYTES: usize = 256 << 20; + const RUNS: usize = 3; + let _guard = subscribe(); + println!("max_chunk,reads,plain_ns_per_read,held_ns_per_read"); + for max in [1200usize, 16 * 1024, 64 * 1024] { + let mut ns = [f64::MAX; 2]; + let mut reads = 0; + // Alternate the modes so drift in the host hits both alike; keep each best run. + for _ in 0..RUNS { + for (mode, held) in [false, true].into_iter().enumerate() { + let endpoint = endpoint(); + let (client, server) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let mut send = client.open_uni().await.unwrap(); + let writer = tokio::spawn(async move { + let block = vec![7u8; 1 << 20]; + for _ in 0..BYTES / block.len() { + send.write_all(&block).await.unwrap(); + } + send.finish().unwrap(); + // Keep the connection up until the reader is done. + let _ = send.stopped().await; + }); + + let mut recv = server.accept_uni().await.unwrap(); + let (mut read, mut count) = (0, 0); + let start = std::time::Instant::now(); + loop { + let len = if held { + match recv.read_chunk(max, true).await.unwrap() { + Some((chunk, credit)) => { + drop(credit); + chunk.bytes.len() + } + None => break, + } + } else { + // `read_to_end`'s path, which returns the credit before handing over. + match std::future::poll_fn(|cx| recv.poll_read_chunk(cx, max, true)) + .await + .unwrap() + { + Some(chunk) => chunk.bytes.len(), + None => break, + } + }; + read += len; + count += 1; + } + let elapsed = start.elapsed().as_nanos() as f64 / count as f64; + assert_eq!(read, BYTES); + ns[mode] = ns[mode].min(elapsed); + reads = count; + drop(recv); + writer.await.unwrap(); + } + } + println!("{max},{reads},{:.1},{:.1}", ns[0], ns[1]); + } +} + /// Fill `buf` with held reads, returning the credit for all of it. async fn read_held_exact(recv: &mut RecvStream, buf: &mut [u8]) -> Credit { - let mut held = recv.read_held(buf).await.unwrap().unwrap(); + let mut held = recv.read(buf).await.unwrap().unwrap(); while (held.len() as usize) < buf.len() { let read = held.len() as usize; - held.merge(recv.read_held(&mut buf[read..]).await.unwrap().unwrap()); + held.merge(recv.read(&mut buf[read..]).await.unwrap().unwrap()); } held } diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index c1fdeefeff6..5ad4f2501c8 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -45,8 +45,9 @@ impl RecvStream { // Unfortunately, we have to wrap ReadError for a bunch of functions. - /// Read some data into the buffer and return the amount read. See [`crate::quic::driver::RecvStream::read`]. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + /// Read some data into the buffer, returning the credit for the bytes read. See + /// [`crate::quic::driver::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { self.inner.read(buf).await.map_err(|e| self.map_error(e)) } @@ -58,24 +59,16 @@ impl RecvStream { }) } - /// Read a chunk of data from the stream. See [`crate::quic::driver::RecvStream::read_chunk`]. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - self.inner - .read_chunk(max_length, true) - .await - .map(|chunk| chunk.map(|chunk| chunk.bytes)) - .map_err(|e| self.map_error(e)) - } - - /// Read a chunk of data, keeping its connection flow control credit until the credit is - /// dropped. See [`crate::quic::driver::RecvStream::read_chunk_held`]. - pub async fn read_chunk_held( + /// Read a chunk of data from the stream, with the credit for its bytes. See + /// [`crate::quic::driver::RecvStream::read_chunk`]. + pub async fn read_chunk( &mut self, max_length: usize, ) -> Result, ReadError> { self.inner - .read_chunk_held(max_length) + .read_chunk(max_length, true) .await + .map(|read| read.map(|(chunk, credit)| (chunk.bytes, credit))) .map_err(|e| self.map_error(e)) } @@ -128,12 +121,13 @@ impl web_transport_trait::RecvStream for RecvStream { Self::stop(self, code).ok(); } + // This interface has no credit, so it returns each read's as it reads. async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { - self.read(dst).await + Ok(self.read(dst).await?.map(|credit| credit.len() as usize)) } async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { - self.read_chunk(max).await + Ok(self.read_chunk(max).await?.map(|(bytes, _)| bytes)) } async fn closed(&mut self) -> Result<(), Self::Error> { diff --git a/rs/moq-tokio/src/transport/held.rs b/rs/moq-tokio/src/transport/held.rs index f5e351e8f61..30282af8d43 100644 --- a/rs/moq-tokio/src/transport/held.rs +++ b/rs/moq-tokio/src/transport/held.rs @@ -68,7 +68,7 @@ impl super::HeldRecvStream for crate::quic::web::RecvStream { &mut self, max: usize, ) -> impl Future, Self::Error>> + Send { - crate::quic::web::RecvStream::read_chunk_held(self, max) + crate::quic::web::RecvStream::read_chunk(self, max) } } diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index ffdd71de508..3678b1bcccc 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -83,8 +83,8 @@ type ClientFuture = std::pin::Pin + Send>>; async fn read_all(recv: &mut moq_tokio::quic::web::RecvStream) -> Vec { let mut out = Vec::new(); let mut buf = [0u8; 4096]; - while let Some(n) = recv.read(&mut buf).await.expect("read") { - out.extend_from_slice(&buf[..n]); + while let Some(credit) = recv.read(&mut buf).await.expect("read") { + out.extend_from_slice(&buf[..credit.len() as usize]); } out } @@ -494,7 +494,7 @@ fn a_dropped_stream_carries_a_webtransport_code() { // The server writes this and then drops the stream unfinished. let mut recv = session.accept_uni().await.expect("accept_uni"); let mut buf = [0u8; 4096]; - let n = recv.read(&mut buf).await.expect("read").expect("payload"); + let n = recv.read(&mut buf).await.expect("read").expect("payload").len() as usize; assert_eq!(&buf[..n], PAYLOAD); // Tell it we have the payload, so the reset below cannot race it. diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 3570cf01028..2e552ce1a7d 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -25,6 +25,7 @@ case "${1:-}" in run just rs bench-feed-fetch run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate + run just rs test -p moq-tokio --release -E 'test(benchmark_held_reads)' --run-ignored only --success-output immediate # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test # Each kio routine once, including the contended bench's worker threads. From 1d063615733a4e80bc43dc68ca872b6112b6d642 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:41:18 -0700 Subject: [PATCH 27/51] style: rustfmt receive credit changes Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/streams/state.rs | 5 ++++- rs/moq-tokio/src/quic.rs | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index f460d2c7c6c..e7697b9a0f4 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -2744,7 +2744,10 @@ mod tests { // The gap fills, so everything is readable, but not read yet. data(&mut client, id, 32, 16, false); - assert!(!client.credit_stats().exhausted(), "unread bytes can still be read"); + assert!( + !client.credit_stats().exhausted(), + "unread bytes can still be read" + ); let rest = read_held(&mut client, id); let stats = client.credit_stats(); diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 5ed880deb53..1afbed0e4d9 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -621,7 +621,9 @@ fn validate_windows(quic: &Config) -> crate::Result<()> { if let Some(window) = quic.receive_window && window < moq_net::transport::MIN_RECEIVE_WINDOW { - return Err(crate::Error::ReceiveWindowTooSmall(moq_net::transport::MIN_RECEIVE_WINDOW)); + return Err(crate::Error::ReceiveWindowTooSmall( + moq_net::transport::MIN_RECEIVE_WINDOW, + )); } for (name, window, wire) in [ ("receive_window", quic.receive_window, true), From cc58807825b4193ea340a986bda0af56667b2218 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:42:03 -0700 Subject: [PATCH 28/51] fix(moq-quic): bind Credit to the connection that handed it out A Credit from one connection released into another succeeded whenever the other held at least as many bytes, silently moving receive window between them. Each StreamsState now carries a process-unique ConnectionTag that Chunks::hold stamps on its Credit; release and merge panic on a foreign one. An empty default Credit adopts the tag of the first credit merged in. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 18 +++++++- rs/moq-quic/src/connection/streams/recv.rs | 27 ++++++++++-- rs/moq-quic/src/connection/streams/state.rs | 47 ++++++++++++++++++--- 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 50ac930e1b7..7580f622668 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -76,6 +76,21 @@ use spaces::Retransmits; use spaces::{PacketNumberFilter, PacketSpace, SendableFrames, SentPacket, ThinRetransmits}; mod stats; + +/// Identifies one connection, so state handed out by it cannot be returned to another. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ConnectionTag(u64); + +impl ConnectionTag { + /// No connection, for state that has not come from one yet. + pub(crate) const NONE: Self = Self(0); + + /// A tag no other connection in this process has. + pub(crate) fn next() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + Self(NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) + } +} pub use stats::{ConnectionStats, CreditStats, FrameStats, PathStats, UdpStats}; mod streams; @@ -1369,8 +1384,7 @@ impl Connection { /// [`Self::poll_transmit()`] soon so the peer learns of the larger window. /// /// # Panics - /// If more credit is released than this connection's reads have held, which means `credit` - /// came from another connection. + /// If `credit` came from another connection. pub fn release(&mut self, credit: Credit) -> ShouldTransmit { let transmit = self.streams.release(credit); self.spaces[SpaceId::Data].pending.max_data |= transmit.should_transmit(); diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index 8ab1e4d212b..526b4343e37 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -6,6 +6,7 @@ use tracing::debug; use super::state::get_or_insert_recv; use super::{ClosedStream, Retransmits, ShouldTransmit, StreamId, StreamsState}; +use crate::connection::ConnectionTag; use crate::connection::assembler::{Assembler, Chunk, IllegalOrderedRead}; use crate::connection::streams::state::StreamRecv; use crate::{TransportError, VarInt, frame}; @@ -344,7 +345,10 @@ impl<'a> Chunks<'a> { pub fn hold(&mut self) -> Credit { let bytes = mem::take(&mut self.read); self.streams.held += bytes; - Credit { bytes } + Credit { + bytes, + tag: self.streams.tag, + } } /// Mark the read data as consumed from the stream. @@ -404,11 +408,13 @@ impl Drop for Chunks<'_> { /// /// The peer cannot reuse this credit until it is passed to /// [`Connection::release()`][crate::Connection::release] on the connection it came from, so -/// dropping it leaks receive window. +/// dropping it leaks receive window. A default `Credit` is empty and belongs to no connection +/// until it merges one that does. #[must_use = "dropping Credit leaks connection flow control credit"] #[derive(Debug, Default, PartialEq, Eq)] pub struct Credit { bytes: u64, + pub(super) tag: ConnectionTag, } impl Credit { @@ -433,11 +439,26 @@ impl Credit { self.bytes ); self.bytes -= bytes; - Self { bytes } + Self { + bytes, + tag: self.tag, + } } /// Take over the bytes covered by `other`. + /// + /// # Panics + /// If `other` came from another connection. pub fn merge(&mut self, other: Self) { + if other.tag != self.tag { + assert!( + self.tag == ConnectionTag::NONE || other.tag == ConnectionTag::NONE, + "merged credit from another connection" + ); + if self.tag == ConnectionTag::NONE { + self.tag = other.tag; + } + } self.bytes += other.bytes; } diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 0b792695d9a..56820451f1f 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -15,6 +15,7 @@ use super::{ use crate::{ Dir, MAX_STREAM_COUNT, Side, StreamId, TransportError, VarInt, coding::BufMutExt, + connection::ConnectionTag, connection::stats::{CreditStats, FrameStats}, frame::{self, FrameStruct, StreamMetaVec}, transport_parameters::TransportParameters, @@ -124,6 +125,8 @@ pub struct StreamsState { /// Bytes read with [`Chunks::hold`](super::Chunks::hold) whose connection credit the /// application has not released yet pub(super) held: u64, + /// Which connection [`Credit`]s handed out here belong to + pub(super) tag: ConnectionTag, /// Total quantity of outgoing data retained in send buffers, including acknowledged data pub(super) buffered_data: u64, /// Configured upper bound for `buffered_data`. @@ -184,6 +187,7 @@ impl StreamsState { data_sent: 0, data_recvd: 0, held: 0, + tag: ConnectionTag::next(), buffered_data: 0, send_window, stream_receive_window: stream_receive_window.into(), @@ -1002,6 +1006,10 @@ impl StreamsState { /// Return the connection credit held by `credit` to the peer. pub(crate) fn release(&mut self, credit: Credit) -> ShouldTransmit { + assert!( + credit.is_empty() || credit.tag == self.tag, + "released credit from another connection" + ); let bytes = credit.into_bytes(); assert!( bytes <= self.held, @@ -2707,14 +2715,43 @@ mod tests { assert_eq!(client.local_max_data, initial + 16); } + /// Another connection holding at least as much credit still refuses it. #[test] - #[should_panic(expected = "only 0 held")] + #[should_panic(expected = "released credit from another connection")] fn release_from_another_connection() { - let mut client = make(Side::Client); let id = StreamId::new(Side::Server, Dir::Uni, 0); - data(&mut client, id, 0, 32, false); - let credit = read_held(&mut client, id); - let _ = make(Side::Client).release(credit); + let mut a = make(Side::Client); + data(&mut a, id, 0, 32, false); + let credit = read_held(&mut a, id); + let mut b = make(Side::Client); + data(&mut b, id, 0, 32, false); + let _held = read_held(&mut b, id); + let _ = b.release(credit); + } + + #[test] + #[should_panic(expected = "merged credit from another connection")] + fn merge_from_another_connection() { + let id = StreamId::new(Side::Server, Dir::Uni, 0); + let mut a = make(Side::Client); + data(&mut a, id, 0, 32, false); + let mut b = make(Side::Client); + data(&mut b, id, 0, 32, false); + let mut credit = read_held(&mut a, id); + credit.merge(read_held(&mut b, id)); + } + + /// An empty default credit takes on the connection of the first credit merged into it. + #[test] + fn default_credit_adopts_its_connection() { + let id = StreamId::new(Side::Server, Dir::Uni, 0); + let mut a = make(Side::Client); + data(&mut a, id, 0, 32, false); + let mut credit = Credit::default(); + credit.merge(read_held(&mut a, id)); + assert_eq!(credit.len(), 32); + let _ = a.release(credit); + assert_eq!(a.credit_stats().held, 0); } /// Ingest and read rounds across many streams, with and without holding and releasing each From b85724762ce4c2a9f4ad6201f360d186c31c76d9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:46:42 -0700 Subject: [PATCH 29/51] feat(moq-uring): hold receive credit through native QUIC Reads through the moq-net adapter keep their connection credit until the complete unit is handed off, and the connection reports when held credit fills the window so moq-net can STOP_SENDING the lowest priority stream. The default window grows to 128 MiB with the stream window following it, and the relay refuses a uring receive window below MIN_RECEIVE_WINDOW. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/uring.rs | 30 +++++++- rs/moq-tokio/src/transport/held.rs | 11 ++- rs/moq-uring/src/quic/driver/connection.rs | 39 ++++++++++ rs/moq-uring/src/quic/driver/mod.rs | 44 +++++++---- rs/moq-uring/src/quic/driver/stream.rs | 83 +++++++++++++-------- rs/moq-uring/src/quic/mod.rs | 13 ++-- rs/moq-uring/src/quic/web.rs | 18 +++++ rs/moq-uring/src/transport.rs | 3 +- rs/moq-uring/src/transport/adapter.rs | 86 ++++++++++++++++------ rs/moq-uring/src/transport/held.rs | 39 ++++++++++ rs/moq-uring/tests/credit.rs | 21 ++++-- 11 files changed, 296 insertions(+), 91 deletions(-) create mode 100644 rs/moq-uring/src/transport/held.rs diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 5df9462027d..5c650e0bb5c 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -422,6 +422,15 @@ fn transport(quic: &moq_tokio::quic::Resolved) -> anyhow::Result= moq_net::transport::MIN_RECEIVE_WINDOW, + "quic.receive_window {window} cannot fit the largest message, {} bytes", + moq_net::transport::MIN_RECEIVE_WINDOW + ); + } let mut transport = moq_uring::quic::Transport::default(); #[cfg(feature = "qlog")] @@ -868,16 +877,33 @@ mod tests { #[allow(clippy::field_reassign_with_default)] fn windows_are_applied() { let mut quic = moq_tokio::quic::Config::default(); - quic.receive_window = Some(64 << 20); + quic.receive_window = Some(128 << 20); quic.stream_receive_window = Some(8 << 20); quic.send_window = Some(32 << 20); let transport = transport(&quic.resolve()).expect("windows must be accepted"); - assert_eq!(transport.receive_window, Some(64 << 20)); + assert_eq!(transport.receive_window, Some(128 << 20)); assert_eq!(transport.stream_receive_window, Some(8 << 20)); assert_eq!(transport.send_window, Some(32 << 20)); } + /// Incomplete messages hold connection credit, so the window has to fit the largest. + #[test] + #[allow(clippy::field_reassign_with_default)] + fn a_receive_window_below_the_largest_message_is_refused() { + let min = moq_net::transport::MIN_RECEIVE_WINDOW; + let mut quic = moq_tokio::quic::Config::default(); + quic.receive_window = Some(min - 1); + let err = transport(&quic.resolve()).expect_err("below the largest message"); + assert!(err.to_string().contains("largest message"), "{err}"); + + quic.receive_window = Some(min); + assert_eq!( + transport(&quic.resolve()).expect("the minimum").receive_window, + Some(min) + ); + } + /// Leaving the windows unset is the ordinary case. The worker then keeps /// its own credits rather than inheriting another backend's default. #[test] diff --git a/rs/moq-tokio/src/transport/held.rs b/rs/moq-tokio/src/transport/held.rs index 30282af8d43..63c543ffcf8 100644 --- a/rs/moq-tokio/src/transport/held.rs +++ b/rs/moq-tokio/src/transport/held.rs @@ -1,22 +1,21 @@ //! How each backend reads with connection credit. -use bytes::Bytes; -use moq_net::transport::Unheld; - /// Read a plain chunk, whose connection credit the backend returned as it read. +#[cfg(any(test, feature = "tcp", feature = "websocket", feature = "iroh"))] pub(crate) async fn read_unheld( stream: &mut S, max: usize, -) -> Result, S::Error> { +) -> Result, S::Error> { let chunk = stream.read_chunk(max).await?; Ok(chunk.map(|bytes| { let len = bytes.len() as u64; - (bytes, Unheld(len)) + (bytes, moq_net::transport::Unheld(len)) })) } /// Give a backend that returns connection credit as it reads, so never fills its window with /// held credit, the held interface. +#[cfg(any(test, feature = "tcp", feature = "websocket", feature = "iroh"))] macro_rules! unheld { ($session:ty, $recv:ty) => { impl $crate::transport::HeldRecvStream for $recv { @@ -67,7 +66,7 @@ impl super::HeldRecvStream for crate::quic::web::RecvStream { fn read_chunk_held( &mut self, max: usize, - ) -> impl Future, Self::Error>> + Send { + ) -> impl Future, Self::Error>> + Send { crate::quic::web::RecvStream::read_chunk(self, max) } } diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index a672a9417b2..59986aeda23 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -74,6 +74,11 @@ pub(crate) struct State { datagram_recv_waiters: kio::WaiterList, datagram_send_waiters: kio::WaiterList, + /// Held credit filled the receive window since + /// [`Connection::poll_credit_exhausted`] last resolved. + credit_exhausted: bool, + credit_waiters: kio::WaiterList, + /// The terminal error, set exactly once; everything fails with it after. closed: Option, closed_waiters: kio::WaiterList, @@ -104,6 +109,8 @@ impl State { sends: FxHashMap::default(), datagram_recv_waiters: kio::WaiterList::new(), datagram_send_waiters: kio::WaiterList::new(), + credit_exhausted: false, + credit_waiters: kio::WaiterList::new(), closed: None, closed_waiters: kio::WaiterList::new(), dead: false, @@ -158,6 +165,18 @@ impl Inner { self.state.borrow().closed.clone() } + /// Note when held credit comes to fill the receive window, waking whoever + /// watches for it. + /// + /// Call after anything that holds, returns, or discards connection credit. + pub(crate) fn check_credit(&self) { + if self.conn.borrow_mut().poll_credit_exhausted() { + let mut state = self.state.borrow_mut(); + state.credit_exhausted = true; + state.credit_waiters.wake(); + } + } + /// Wake the driver so it flushes what a handle just queued. pub(crate) fn kick(&self) { self.state.borrow_mut().driver.wake(); @@ -342,6 +361,23 @@ impl Connection { pub fn stats(&self) -> moq_quic::ConnectionStats { self.shared.conn.borrow().stats() } + + /// Poll until held [`Credit`](super::Credit) fills the receive window: the + /// peer may send nothing more and every byte it sent is held, so no stream + /// makes progress until some credit is dropped. + /// + /// Resolves once per stall, and again only after more credit is held or + /// credit drops without relieving it. + pub fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + let waiter = self.park.hold(cx); + let mut state = self.shared.state.borrow_mut(); + // Credit may have dropped since the stall was noted. + if std::mem::take(&mut state.credit_exhausted) && self.shared.conn.borrow().stats().credit.exhausted() { + return Poll::Ready(()); + } + waiter.register(&mut state.credit_waiters); + Poll::Pending + } } impl Clone for Connection { @@ -723,6 +759,9 @@ impl Driver { loop { self.endpoint_events(); self.sweep(); + // Data for a stopped stream or a reset returns its credit, possibly + // without relieving a stall. + self.shared.check_credit(); if self.shared.state.borrow().dead { return Poll::Ready(()); diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index 30e53a05a34..826d1525d87 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -30,10 +30,12 @@ use super::qlog; use super::{Congestion, Error, Identity, SEGMENT, Transport, client, endpoint::CID_LEN, server}; use crate::udp; -/// Per-stream flow control credit. -const STREAM_WINDOW: u32 = 4 * 1024 * 1024; -/// Per-connection flow control credit. -const CONNECTION_WINDOW: u32 = 16 * 1024 * 1024; +/// Per-connection flow control credit, which also bounds incomplete messages, so it has to fit +/// the largest one. Each stream may use all of it. +const CONNECTION_WINDOW: u32 = 128 * 1024 * 1024; +const _: () = assert!(CONNECTION_WINDOW as u64 >= moq_net::transport::MIN_RECEIVE_WINDOW); +/// Unacknowledged outgoing data to buffer, whatever the peer allows. +const SEND_WINDOW: u32 = 16 * 1024 * 1024; /// How many datagrams to buffer in each direction. const DATAGRAM_WINDOW: usize = 64 * SEGMENT; @@ -276,17 +278,17 @@ fn transport_config(config: &Transport) -> Result varint_window("stream receive window", window)?, - None => STREAM_WINDOW.into(), - }; + // A value that cannot go on the wire is an error: clamping it would ignore + // what was set. let connection_window = match config.receive_window { Some(window) => varint_window("receive window", window)?, None => CONNECTION_WINDOW.into(), }; - let send_window = config.send_window.unwrap_or(u64::from(CONNECTION_WINDOW)); + let stream_window = match config.stream_receive_window { + Some(window) => varint_window("stream receive window", window)?, + None => connection_window, + }; + let send_window = config.send_window.unwrap_or(u64::from(SEND_WINDOW)); transport.stream_receive_window(stream_window); transport.receive_window(connection_window); transport.send_window(send_window); @@ -400,7 +402,7 @@ impl rustls::client::danger::ServerCertVerifier for NoVerify { #[cfg(test)] mod tests { - use super::{CONNECTION_WINDOW, STREAM_WINDOW, Transport, transport_config}; + use super::{CONNECTION_WINDOW, SEND_WINDOW, Transport, transport_config}; /// moq-quic keeps the windows private, but its `Debug` prints them. The leading /// space keeps `receive_window` from matching `stream_receive_window`, and @@ -415,23 +417,23 @@ mod tests { assert!(has_window( &transport, "stream_receive_window", - u64::from(STREAM_WINDOW) + u64::from(CONNECTION_WINDOW) )); assert!(has_window(&transport, "receive_window", u64::from(CONNECTION_WINDOW))); - assert!(has_window(&transport, "send_window", u64::from(CONNECTION_WINDOW))); + assert!(has_window(&transport, "send_window", u64::from(SEND_WINDOW))); } #[test] fn configured_windows_replace_the_defaults() { let config = Transport { - receive_window: Some(64 << 20), + receive_window: Some(256 << 20), stream_receive_window: Some(8 << 20), send_window: Some(32 << 20), ..Default::default() }; let transport = transport_config(&config).expect("windows"); - assert!(has_window(&transport, "receive_window", 64 << 20)); + assert!(has_window(&transport, "receive_window", 256 << 20)); assert!(has_window(&transport, "stream_receive_window", 8 << 20)); assert!(has_window(&transport, "send_window", 32 << 20)); assert!(!has_window(&transport, "receive_window", u64::from(CONNECTION_WINDOW))); @@ -454,6 +456,16 @@ mod tests { assert!(has_window(&transport, "send_window", u64::MAX)); } + #[test] + fn the_stream_window_follows_the_receive_window() { + let config = Transport { + receive_window: Some(256 << 20), + ..Default::default() + }; + let transport = transport_config(&config).expect("windows"); + assert!(has_window(&transport, "stream_receive_window", 256 << 20)); + } + #[test] fn a_receive_window_past_a_varint_is_an_error() { let over = Transport { diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index 1be7b4e6a99..01e2520480c 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -241,42 +241,64 @@ impl RecvStream { } } - /// Read into `dst`, keeping the connection flow control credit for exactly - /// the bytes read, or `None` once the peer has finished the stream. + /// Read up to `max` bytes, keeping the connection flow control credit for + /// exactly the bytes read, or `None` once the peer has finished the stream. /// /// Stream credit still advances, so a stream window smaller than an /// application message keeps the message flowing, while the connection /// window these bytes occupy stays spent until the credit is dropped. - pub fn poll_read_held(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { + pub fn poll_read_held(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { + let waiter = self.park.hold(cx); + let credit = |inner| Credit { + shared: self.shared.clone(), + inner, + }; + if max == 0 { + return Poll::Ready(Ok(Some((Bytes::new(), credit(moq_quic::Credit::default()))))); + } + if !self.backlog.is_empty() { + let n = max.min(self.backlog.len()); + let bytes = self.backlog.split_to(n).freeze(); + let inner = self.backlog_credit.split(n as u64); + // A `poll_closed` parked at the read-ahead cap waits for this room. + self.shared.wake_readable(self.id); + return Poll::Ready(Ok(Some((bytes, credit(inner))))); + } + if self.finished { + return Poll::Ready(Ok(None)); + } let mut held = moq_quic::Credit::default(); - self.poll_read_into(cx, dst, Some(&mut held)).map_ok(|read| { - read.map(|_| Credit { - shared: self.shared.clone(), - inner: held, - }) - }) - } - - /// [`poll_read`](web_transport_trait::poll::RecvStream::poll_read), with - /// the credit for the bytes read moved into `held` instead of returned to - /// the peer. - fn poll_read_into( - &mut self, - cx: &mut Context<'_>, - dst: &mut [u8], - held: Option<&mut moq_quic::Credit>, - ) -> Poll, Error>> { + match Self::read(&self.shared, self.id, max, Some(&mut held)) { + Read::Chunk(bytes) => Poll::Ready(Ok(Some((bytes, credit(held))))), + Read::Finished => { + self.finished = true; + Poll::Ready(Ok(None)) + } + Read::Blocked => { + if let Some(err) = self.shared.closed() { + return Poll::Ready(Err(err)); + } + self.shared.park_readable(self.id, waiter); + Poll::Pending + } + Read::Reset(code) => Poll::Ready(Err(Error::Reset(code))), + } + } + + /// [`poll_read`](web_transport_trait::poll::RecvStream::poll_read), which + /// returns the credit for the bytes read to the peer. + fn poll_read_plain(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { let waiter = self.park.hold(cx); if dst.is_empty() { return Poll::Ready(Ok(Some(0))); } if !self.backlog.is_empty() { - return Poll::Ready(Ok(Some(self.drain(dst, held)))); + return Poll::Ready(Ok(Some(self.drain(dst)))); } if self.finished { return Poll::Ready(Ok(None)); } - match Self::read(&self.shared, self.id, dst.len(), held) { + match Self::read(&self.shared, self.id, dst.len(), None) { Read::Chunk(bytes) => { // moq-quic hands out at most `dst.len()`. dst[..bytes.len()].copy_from_slice(&bytes); @@ -319,6 +341,7 @@ impl RecvStream { .recv_stream(self.id) .stop(VarInt::from_u64(code).unwrap_or(VarInt::MAX)); self.stopped = true; + self.shared.check_credit(); self.shared.kick(); } @@ -352,23 +375,20 @@ impl RecvStream { if transmit { shared.kick(); } + shared.check_credit(); read } - /// Move up to `dst.len()` read-ahead bytes out of the backlog, with their - /// credit moved into `held` or else returned to the peer. + /// Move up to `dst.len()` read-ahead bytes out of the backlog, returning + /// their credit to the peer. /// /// Wakes a `poll_closed` parked at the read-ahead cap: the room it was /// waiting for is what this just made. - fn drain(&mut self, dst: &mut [u8], held: Option<&mut moq_quic::Credit>) -> usize { + fn drain(&mut self, dst: &mut [u8]) -> usize { let n = dst.len().min(self.backlog.len()); dst[..n].copy_from_slice(&self.backlog[..n]); self.backlog.advance(n); - let credit = self.backlog_credit.split(n as u64); - match held { - Some(held) => held.merge(credit), - None => release(&self.shared, credit), - } + release(&self.shared, self.backlog_credit.split(n as u64)); if n > 0 { self.shared.wake_readable(self.id); } @@ -380,7 +400,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { type Error = Error; fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - self.poll_read_into(cx, dst, None) + self.poll_read_plain(cx, dst) } fn stop(&mut self, code: u32) { @@ -459,6 +479,7 @@ fn release(shared: &Shared, credit: moq_quic::Credit) { if shared.conn.borrow_mut().release(credit).should_transmit() { shared.kick(); } + shared.check_credit(); } /// Connection flow control credit for bytes read with diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index b8ccd230f33..2a4331d9ad3 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -115,14 +115,17 @@ pub struct Transport { /// may have open at once. MoQ opens a stream per group, so busy endpoints /// want this high. pub max_streams: u64, - /// How much unread data one peer may have in flight across every stream, - /// in bytes. `None` leaves the 16 MiB default. + /// How much unread data and incomplete messages one peer may have in flight + /// across every stream, in bytes. `None` leaves the 128 MiB default. A moq + /// session needs it to fit the largest message it assembles, + /// [`MIN_RECEIVE_WINDOW`](moq_net::transport::MIN_RECEIVE_WINDOW), or a + /// partial one can stall the connection. pub receive_window: Option, /// How much unread data one peer may have in flight on a single stream, - /// in bytes. `None` leaves the 4 MiB default. + /// in bytes. `None` follows [`receive_window`](Self::receive_window). /// - /// Keep it under [`receive_window`](Self::receive_window) so one stream - /// cannot fill the connection. + /// Stream credit returns as bytes are read, whether or not their message is + /// complete, so a smaller window only adds round trips to a large frame. pub stream_receive_window: Option, /// How much unacknowledged outgoing data to buffer, in bytes, whatever /// the peer allows. `None` leaves the 16 MiB default. diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 192592f9b21..9d49f7b3479 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -515,6 +515,11 @@ impl Session { } } + /// See [`Connection::poll_credit_exhausted`]. + pub fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + self.conn.poll_credit_exhausted(cx) + } + /// Rewrite error codes out of the HTTP/3 mapping in web mode. fn map_err(&self, err: Error) -> Error { match self.web { @@ -968,6 +973,19 @@ impl RecvStream { false => unmap_legacy(err), } } + + /// Read up to `max` bytes with their connection credit. See + /// [`super::RecvStream::poll_read_held`]. + pub fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Error>> { + match self.inner.poll_read_held(cx, max) { + Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), + other => other, + } + } } impl web_transport_trait::poll::RecvStream for RecvStream { diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index 6a26f1d66b7..403b6575d3d 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -1,3 +1,4 @@ //! Adapt backend poll sessions to moq-net without changing thread affinity. mod adapter; -pub use adapter::{Error, RecvStream, SendStream, Session}; +mod held; +pub use adapter::{Error, HeldRecvStream, HeldSession, RecvStream, SendStream, Session}; diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index 7b0435d83f7..626359207ee 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -1,6 +1,6 @@ use bytes::{Buf, BufMut, Bytes}; use moq_net::transport::{self as moq, poll}; -use std::task::{Context, Poll}; +use std::task::{Context, Poll, ready}; use web_transport_trait::{self as wt, poll as backend}; /// A backend failure preserving the session and stream code registries. @@ -44,6 +44,26 @@ impl moq::Stats for Stats { self.0.estimated_send_rate() } } +/// A backend receive stream that can keep the connection credit of what it reads. +pub trait HeldRecvStream: backend::RecvStream { + /// The connection credit a read keeps until it is dropped. + type Credit: moq::Credit; + + /// Read up to `max` bytes with their credit, or `None` once finished. + #[allow(clippy::type_complexity)] + fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>>; +} + +/// A backend session that reports when held credit fills its receive window. +pub trait HeldSession: backend::Session { + /// See [`poll_credit_exhausted`](poll::Session::poll_credit_exhausted). + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()>; +} + /// A WebTransport session adapted to moq-net. #[derive(Clone)] pub struct Session(S, moq::Binding); @@ -53,7 +73,7 @@ impl Session { Self(session, binding) } } -impl + Clone + 'static> poll::Session for Session { +impl + Clone + 'static> poll::Session for Session { type SendStream = SendStream; type RecvStream = RecvStream; type Error = Error; @@ -98,8 +118,8 @@ impl + Clone + 'st fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { backend::Session::poll_closed(&mut self.0, cx).map(Error) } - fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { - Poll::Pending + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + HeldSession::poll_credit_exhausted(&mut self.0, cx) } fn stats(&self) -> impl moq::Stats { Stats(backend::Session::stats(&self.0)) @@ -130,37 +150,35 @@ impl poll::SendStream for SendStream { } /// An incoming WebTransport stream adapted to moq-net. pub struct RecvStream(S); -// The WebTransport backend returns connection credit as it reads, so there is none to hold. -impl poll::RecvStream for RecvStream { +impl poll::RecvStream for RecvStream { type Error = Error; - type Credit = moq::Unheld; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - backend::RecvStream::poll_read(&mut self.0, cx, dst) - .map_err(Error) - .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) + type Credit = S::Credit; + fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { + let read = ready!(self.0.poll_read_held(cx, dst.len())).map_err(Error)?; + Poll::Ready(Ok(read.map(|(bytes, credit)| { + dst[..bytes.len()].copy_from_slice(&bytes); + credit + }))) } fn poll_read_buf( &mut self, cx: &mut Context<'_>, buf: &mut B, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_buf(&mut self.0, cx, buf) - .map_err(Error) - .map_ok(|n| n.map(|n| moq::Unheld(n as u64))) + ) -> Poll, Self::Error>> { + // Hand over the chunk the backend owns, without zeroing a destination first. + let max = buf.remaining_mut().min(64 * 1024); + let read = ready!(self.0.poll_read_held(cx, max)).map_err(Error)?; + Poll::Ready(Ok(read.map(|(bytes, credit)| { + buf.put(bytes); + credit + }))) } fn poll_read_chunk( &mut self, cx: &mut Context<'_>, max: usize, - ) -> Poll, Self::Error>> { - backend::RecvStream::poll_read_chunk(&mut self.0, cx, max) - .map_err(Error) - .map_ok(|chunk| { - chunk.map(|b| { - let n = b.len() as u64; - (b, moq::Unheld(n)) - }) - }) + ) -> Poll, Self::Error>> { + self.0.poll_read_held(cx, max).map_err(Error) } fn stop(&mut self, code: u32) { backend::RecvStream::stop(&mut self.0, code) @@ -223,6 +241,26 @@ mod tests { Poll::Ready(Err(Failure)) } } + impl HeldRecvStream for Stream { + type Credit = moq::Unheld; + fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Failure>> { + backend::RecvStream::poll_read_chunk(self, cx, max).map_ok(|chunk| { + chunk.map(|bytes| { + let len = bytes.len() as u64; + (bytes, moq::Unheld(len)) + }) + }) + } + } + impl HeldSession for Backend { + fn poll_credit_exhausted(&mut self, _: &mut Context<'_>) -> Poll<()> { + Poll::Pending + } + } struct Sample; impl wt::Stats for Sample { fn rtt(&self) -> Option { diff --git a/rs/moq-uring/src/transport/held.rs b/rs/moq-uring/src/transport/held.rs new file mode 100644 index 00000000000..7a9fb860331 --- /dev/null +++ b/rs/moq-uring/src/transport/held.rs @@ -0,0 +1,39 @@ +//! How the io_uring QUIC backend reads with connection credit. + +use std::task::{Context, Poll}; + +use bytes::Bytes; + +use crate::quic::{self as driver, web}; + +impl moq_net::transport::Credit for driver::Credit { + fn len(&self) -> u64 { + self.len() + } + + fn split(&mut self, bytes: u64) -> Self { + self.split(bytes) + } + + fn merge(&mut self, other: Self) { + self.merge(other) + } +} + +impl super::HeldRecvStream for web::RecvStream { + type Credit = driver::Credit; + + fn poll_read_held( + &mut self, + cx: &mut Context<'_>, + max: usize, + ) -> Poll, Self::Error>> { + web::RecvStream::poll_read_held(self, cx, max) + } +} + +impl super::HeldSession for web::Session { + fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { + web::Session::poll_credit_exhausted(self, cx) + } +} diff --git a/rs/moq-uring/tests/credit.rs b/rs/moq-uring/tests/credit.rs index 6a9fcf3787b..f25941af372 100644 --- a/rs/moq-uring/tests/credit.rs +++ b/rs/moq-uring/tests/credit.rs @@ -1,6 +1,7 @@ //! Held receive credit through the io_uring driver: held reads keep the -//! connection window spent until their credit drops, and bytes `poll_closed` -//! reads ahead keep theirs until the application reads them. +//! connection window spent until their credit drops, bytes `poll_closed` +//! reads ahead keep theirs until the application reads them, and the +//! connection reports each time held credit fills the window. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. @@ -10,7 +11,7 @@ mod support; use std::net::UdpSocket; -use std::task::Poll; +use std::task::{Context, Poll, Waker}; use std::time::Duration; use moq_uring::{Config, Error, Worker, quic, udp}; @@ -49,18 +50,18 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future quic::Credit { - let mut buf = vec![0u8; len]; let mut held: Option = None; while held.as_ref().map_or(0, quic::Credit::len) < len as u64 { let filled = held.as_ref().map_or(0, |held| held.len() as usize); - let credit = within( + let (bytes, credit) = within( handle, "stream data", - std::future::poll_fn(|cx| recv.poll_read_held(cx, &mut buf[filled..])), + std::future::poll_fn(|cx| recv.poll_read_held(cx, len - filled)), ) .await .expect("read") .expect("not finished"); + assert_eq!(bytes.len() as u64, credit.len(), "credit covers exactly the bytes read"); match &mut held { Some(held) => held.merge(credit), None => held = Some(credit), @@ -146,14 +147,22 @@ fn held_reads_spend_the_connection_window() { .await .expect("accept_uni"); + let mut cx = Context::from_waker(Waker::noop()); + let mut exhausted = |conn: &mut quic::Connection| conn.poll_credit_exhausted(&mut cx).is_ready(); + assert!(!exhausted(&mut accepted)); + let mut held = read_held(&handle, &mut recv, WINDOW).await; let stats = accepted.stats().credit; assert_eq!((stats.held, stats.available), (WINDOW as u64, 0)); + assert!(exhausted(&mut accepted), "held credit filled the window"); + assert!(!exhausted(&mut accepted), "reported once per stall"); // Far below the usual update threshold, and still enough to unblock. drop(held.split(100)); + assert!(!exhausted(&mut accepted), "a release relieves the stall"); held.merge(read_held(&handle, &mut recv, 100).await); assert_eq!(accepted.stats().credit.held, WINDOW as u64); + assert!(exhausted(&mut accepted), "holding the room again stalls again"); // Read-ahead keeps the credit the application has not taken yet. drop(held); From 33479f3f254016789211eed2a3cb931b7878ae8b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:01:16 -0700 Subject: [PATCH 30/51] test(moq-uring): a partial frame gives way over the worker's QUIC path Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/tests/receive_credit.rs | 224 +++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 rs/moq-uring/tests/receive_credit.rs diff --git a/rs/moq-uring/tests/receive_credit.rs b/rs/moq-uring/tests/receive_credit.rs new file mode 100644 index 00000000000..92246d41984 --- /dev/null +++ b/rs/moq-uring/tests/receive_credit.rs @@ -0,0 +1,224 @@ +//! Over the worker's QUIC path, incomplete frames hold the subscriber's connection +//! credit, and when they fill its window the subscriber stops the least urgent stream +//! and keeps going. +//! +//! The subscriber's connection window is far smaller than a moq session would allow, +//! which keeps the payloads small. Without recovery the high-priority frame never +//! arrives: the window is spent on a low-priority frame that can never complete. +//! +//! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. + +#![cfg(all(target_os = "linux", feature = "quic"))] + +#[path = "support.rs"] +mod support; + +use std::net::UdpSocket; +use std::task::Poll; +use std::time::Duration; + +use bytes::Bytes; +use moq_net::{Timestamp, frame, origin, track::Subscription}; +use moq_uring::{Config, Error, Worker, quic, udp}; + +const ALPN: &str = "moq-lite-06"; +/// The subscriber's connection receive window. +const WINDOW: usize = 256 * 1024; + +fn worker() -> Option { + match Worker::new(Config::default()) { + Ok(worker) => Some(worker), + Err(Error::Unsupported(reason)) if std::env::var_os("MOQ_STRICT").is_none() => { + eprintln!("skipping io_uring receive credit test: {reason}"); + None + } + Err(err) => panic!("worker setup failed: {err}"), + } +} + +fn timed() -> moq_net::track::Info { + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) +} + +/// Await `future`, failing loudly rather than hanging if it never resolves. +async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future) -> T { + let mut deadline = moq_uring::Timer::after(handle, Duration::from_secs(10)); + let mut future = std::pin::pin!(future); + kio::wait(|waiter| { + let mut cx = waiter.context(); + if let Poll::Ready(value) = future.as_mut().poll(&mut cx) { + return Poll::Ready(Some(value)); + } + deadline.poll(waiter).map(|()| None) + }) + .await + .unwrap_or_else(|| panic!("timed out waiting for {what}")) +} + +/// The next frame's payload. +async fn next_frame(handle: &moq_uring::Handle, sub: &mut moq_net::track::Subscriber) -> Bytes { + let mut group = within(handle, "a group", sub.recv_group()) + .await + .expect("group") + .expect("track ended"); + within(handle, "a frame", group.read_frame()) + .await + .expect("frame") + .expect("group ended") + .payload +} + +/// Two partial frames that together exceed the window: the lower-priority one is +/// stopped, and the higher-priority one completes on the credit it returns. +#[test] +fn partial_frame_gives_way() { + let Some(mut worker) = worker() else { return }; + let handle = worker.handle(); + + // The model and its origins, driven on a tokio thread as in `session.rs`. + let (pub_origin, pub_driver) = origin::Producer::new(origin::Config::default()); + let (sub_origin, sub_driver) = origin::Producer::new(origin::Config::default()); + let origins = std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("tokio runtime"); + rt.block_on(async move { + tokio::join!(moq_net::time::run(pub_driver), moq_net::time::run(sub_driver)); + }); + }); + + let broadcast = pub_origin.create_broadcast("room").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let low = broadcast.create_track("low", timed()).expect("low track"); + let high = broadcast.create_track("high", timed()).expect("high track"); + + let certs = support::certs().expect("certificates"); + let server_config = quic::server::Config::new( + quic::Identity::open(&certs.cert, &certs.key) + .expect("identity") + .server(&[ALPN], quic::server::ClientAuth::None) + .expect("tls"), + ); + let socket = |handle: &moq_uring::Handle| { + handle + .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) + .expect("socket") + }; + let server_sock = socket(&handle); + let mut dial = quic::client::Config::new(server_sock.local_addr().expect("server addr"), "localhost"); + dial.alpn = vec![ALPN.to_string()]; + dial.verify = false; + dial.transport.receive_window = Some(WINDOW as u64); + let client_sock = socket(&handle); + + let server_handle = handle.clone(); + handle.spawn(async move { + let conn = quic::server::accept(server_sock, &server_config) + .await + .expect("quic accept"); + let (session, driver) = moq_net::Server::new() + .with_publisher(&pub_origin) + .accept_lite( + std::time::Instant::now(), + moq_uring::transport::Session::new(quic::web::Session::raw(conn), moq_net::transport::Binding::InBand), + ) + .await + .expect("accept_lite"); + let _ = server_handle.run(driver).await; + session.closed().await; + }); + + let sub = sub_origin.clone(); + worker + .block_on(async move { + let conn = quic::client::connect(client_sock, &dial).await.expect("quic connect"); + let (session, driver) = moq_net::Client::new() + .with_subscriber(sub.clone()) + .connect_lite( + std::time::Instant::now(), + moq_uring::transport::Session::new( + quic::web::Session::raw(conn), + moq_net::transport::Binding::InBand, + ), + ) + .await + .expect("connect_lite"); + let task_handle = handle.clone(); + handle.spawn(async move { + let _ = task_handle.run(driver).await; + }); + + let consumer = sub.consume(); + within(&handle, "the route", consumer.routed("room")) + .await + .expect("routed"); + let remote = within(&handle, "the broadcast", consumer.request_broadcast("room", None)) + .await + .expect("broadcast resolves"); + let subscribe = |name: &str, priority: u8| { + let track = remote.track(name).expect("track"); + async move { + track + .subscribe(Subscription::default().with_priority(priority)) + .await + .expect("subscribe") + } + }; + let mut low_sub = within(&handle, "the low subscription", subscribe("low", 1)).await; + let mut high_sub = within(&handle, "the high subscription", subscribe("high", 200)).await; + + // Three quarters of the window, held by a frame that never finishes. + let mut low_group = low.append_group().expect("low group"); + let mut low_frame = low_group + .create_frame(frame::Info { + size: WINDOW as u64, + timestamp: Some(Timestamp::ZERO), + }) + .expect("low frame"); + low_frame.write(vec![1u8; WINDOW * 3 / 4]).expect("partial payload"); + + // Wait until the subscriber holds all of it, so nothing else got there first. + let mut partial = within(&handle, "the low group", low_sub.recv_group()) + .await + .expect("group") + .expect("track ended"); + let mut partial = within(&handle, "the low frame header", partial.next_frame()) + .await + .expect("frame") + .expect("group ended"); + let mut seen = 0; + while seen < WINDOW * 3 / 4 { + seen += within(&handle, "the partial payload", partial.read_chunk()) + .await + .expect("chunk") + .expect("frame ended") + .len(); + } + + // Half the window, which only fits once the low frame lets go. + let payload = Bytes::from(vec![2u8; WINDOW / 2]); + let mut high_group = high.append_group().expect("high group"); + high_group + .write_frame(Timestamp::ZERO, payload.clone()) + .expect("high frame"); + high_group.finish().expect("finish high"); + assert_eq!(next_frame(&handle, &mut high_sub).await, payload); + + // The session survives, and the low track keeps delivering what fits. + drop(low_frame); + drop(low_group); + let mut group = low.append_group().expect("after group"); + group.write_frame(Timestamp::ZERO, &b"after"[..]).expect("after frame"); + group.finish().expect("finish after"); + assert_eq!(next_frame(&handle, &mut low_sub).await, &b"after"[..]); + + session.abort(moq_net::Error::Cancel); + }) + .expect("worker"); + + drop(worker); + drop(broadcast); + drop(sub_origin); + origins.join().expect("origin drivers"); +} From e1bac997548335d0cc28a1184db4d0525b4f91c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:10:26 -0700 Subject: [PATCH 31/51] fix(moq-tokio): keep the scripted session behind the websocket feature Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 43b91e3a545..61e8bfc6fca 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -2233,6 +2233,7 @@ mod tests { #[cfg(feature = "websocket")] crate::transport::held::unheld!(Scripted, ScriptedRecv); + #[cfg(feature = "websocket")] impl web_transport_trait::Session for Scripted { type SendStream = ScriptedSend; type RecvStream = ScriptedRecv; From 071d21f6794c2ce1dcbb9614185476aa65646fdc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:30:41 -0700 Subject: [PATCH 32/51] quest: split JavaScript sender and benchmark coverage out of receive-memory Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/README.md | 1 + quest/m1/quic/receive-memory-coverage.md | 43 ++++++++++++++++++++++++ quest/m1/quic/receive-memory.md | 10 +++--- 3 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 quest/m1/quic/receive-memory-coverage.md diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index 90542b510a6..db5c05223e8 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -63,6 +63,7 @@ cache limits remain separate; bitrate pacing composes with this memory cap. ## Required - [Bound message assembly](/quest/m1/quic/receive-memory.md) - complete headers and frames return credit, and receiver priority cancellation restores progress +- [Prove receive memory](/quest/m1/quic/receive-memory-coverage.md) - a JavaScript sender exhausts both native receivers and they recover; nightly measures the cost - [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - `RESET_STREAM_AT`, so a reset WebTransport stream still delivers its header - [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - strict diff --git a/quest/m1/quic/receive-memory-coverage.md b/quest/m1/quic/receive-memory-coverage.md new file mode 100644 index 00000000000..417990398dd --- /dev/null +++ b/quest/m1/quic/receive-memory-coverage.md @@ -0,0 +1,43 @@ +# [M] Prove receive memory with JavaScript senders and benchmarks + +## Goal + +A JavaScript sender drives both native receivers into receive-credit +exhaustion and they recover, and nightly measures what bounded receive memory +costs. This is coverage for the [message assembly](/quest/m1/quic/receive-memory.md) +work, with no new transport behavior. + +## Plan + +Split out of #5735, which lands native credit holding, pressure recovery, and +the minimum receive window with Rust-only end-to-end tests over tokio and +io_uring. + +A JavaScript sender over the stock receivers cannot stall them. The minimum +window fits the largest unit, and a strict-priority sender finishes one +stream before the next, so every frame completes. The test therefore needs a +receiver whose connection window is far below the minimum. Neither stock +config allows that, so the test assembles a WebTransport (HTTP/3) server +from each raw driver: tokio's `quic::driver` with `quic::web::Server`, and +an io_uring endpoint in web mode. Keep it test-only; do not add a public way +to configure a window below `MIN_RECEIVE_WINDOW`. + +The sender is a Bun script using `@moq/web-transport`, pinned to the test +server's certificate hash with WebSocket disabled. It mirrors the Rust tests +in `rs/moq-tokio/tests/receive_credit.rs`. A low-priority track sends a frame +larger than the window. A high-priority frame then has to complete, and a +later low-priority group has to arrive after the low stream is stopped. Run +it from Rust the way `rs/moq-tokio/tests/max_age.rs` drives its Bun client, +and wire it into the interop workflow, since normal CI has no Bun. + +The benchmark sweeps concurrent streams and frame sizes, and compares a +finite stream window with one that follows the connection window. Report +retained bytes (held plus unread credit), throughput, and frame completion +latency. Sweep streams and frame sizes on separate axes, so a cost that grows +with the stream count shows up as a slope. Wire it into `sh/gh/nightly.sh` +next to `benchmark_held_reads`. Unit tests stay on mocked time; the benchmark +is the only wall-clock part. + +## Required + +- [Bound message assembly](/quest/m1/quic/receive-memory.md) - the native credit, recovery, and window validation this exercises diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md index 5e2bd90a1b7..afab88facbc 100644 --- a/quest/m1/quic/receive-memory.md +++ b/quest/m1/quic/receive-memory.md @@ -84,11 +84,10 @@ Regression coverage must include many partial headers and frames filling the shared allowance, a finite stream window smaller than a frame, the largest supported encoded unit, read-ahead across unit boundaries, cancellation and late reset/data races, unidentified and control-only pressure, and continued -progress after selecting victims. Exercise a JavaScript sender against both -native receive paths and run all-language interop. Measure retained bytes, -throughput, and completion latency across concurrent streams and frame sizes, -including finite versus very large stream windows. Wire checks and benchmarks -into CI; no wall-clock sleeps in unit tests. +progress after selecting victims, and run all-language interop. Wire checks +into CI; no wall-clock sleeps in unit tests. The JavaScript sender and the +retained-bytes, throughput, and latency sweep moved to +[their own quest](/quest/m1/quic/receive-memory-coverage.md). Public API: native receive-credit integration and validation of the existing window configuration; no new header-specific quota. Wire: existing QUIC @@ -99,6 +98,7 @@ separate guide quest is needed. ## Related +- [Prove receive memory](/quest/m1/quic/receive-memory-coverage.md) - a JavaScript sender against both native receivers, and the nightly benchmark sweep - [Scope track priority](/quest/m1/track-priority-scope.md) - broadcast fairness domains and subscription priority semantics - [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - share priority semantics without requiring sender-side recovery - [QUIC bitrate caps](/quest/m2/rate-quic.md) - pace credit grants inside the independently bounded memory allowance From 62d3fe8d8c754b133461ea5c0d8119d5808885cc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:23:06 -0700 Subject: [PATCH 33/51] quest: complete receive-memory Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/README.md | 1 - quest/m1/quic/peer-limits.md | 7 +- quest/m1/quic/receive-memory-coverage.md | 8 +- quest/m1/quic/receive-memory.md | 105 ----------------------- quest/m2/rate-quic.md | 5 +- 5 files changed, 8 insertions(+), 118 deletions(-) delete mode 100644 quest/m1/quic/receive-memory.md diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index db5c05223e8..430fc0bb6f8 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -62,7 +62,6 @@ cache limits remain separate; bitrate pacing composes with this memory cap. ## Required -- [Bound message assembly](/quest/m1/quic/receive-memory.md) - complete headers and frames return credit, and receiver priority cancellation restores progress - [Prove receive memory](/quest/m1/quic/receive-memory-coverage.md) - a JavaScript sender exhausts both native receivers and they recover; nightly measures the cost - [Reliable stream reset](/quest/m1/quic/reliable-reset.md) - `RESET_STREAM_AT`, so a reset WebTransport stream still delivers its header diff --git a/quest/m1/quic/peer-limits.md b/quest/m1/quic/peer-limits.md index 7a97758dab9..a11e140826f 100644 --- a/quest/m1/quic/peer-limits.md +++ b/quest/m1/quic/peer-limits.md @@ -35,9 +35,10 @@ its own. - Refuse a `peer` value below the client default rather than silently shrinking. -The [receive-memory plan](/quest/m1/quic/receive-memory.md) requires client -and peer memory allowances to fit the largest supported encoded unit, -including framing. Preserve that validation and retained-credit accounting +Message assembly holds connection credit until each unit is complete, so +client and peer receive windows must fit the largest supported encoded unit, +`moq_net::transport::MIN_RECEIVE_WINDOW`; the relay refuses a smaller +`[quic]` window at config load. Preserve that validation and retained-credit accounting when applying runtime limits; wider stream counts must not create a second unbounded assembly pool. diff --git a/quest/m1/quic/receive-memory-coverage.md b/quest/m1/quic/receive-memory-coverage.md index 417990398dd..81e8835d7f1 100644 --- a/quest/m1/quic/receive-memory-coverage.md +++ b/quest/m1/quic/receive-memory-coverage.md @@ -4,8 +4,8 @@ A JavaScript sender drives both native receivers into receive-credit exhaustion and they recover, and nightly measures what bounded receive memory -costs. This is coverage for the [message assembly](/quest/m1/quic/receive-memory.md) -work, with no new transport behavior. +costs. This is coverage for the message-assembly work in #5735, with no new +transport behavior. ## Plan @@ -37,7 +37,3 @@ latency. Sweep streams and frame sizes on separate axes, so a cost that grows with the stream count shows up as a slope. Wire it into `sh/gh/nightly.sh` next to `benchmark_held_reads`. Unit tests stay on mocked time; the benchmark is the only wall-clock part. - -## Required - -- [Bound message assembly](/quest/m1/quic/receive-memory.md) - the native credit, recovery, and window validation this exercises diff --git a/quest/m1/quic/receive-memory.md b/quest/m1/quic/receive-memory.md deleted file mode 100644 index afab88facbc..00000000000 --- a/quest/m1/quic/receive-memory.md +++ /dev/null @@ -1,105 +0,0 @@ -# [XL] Bound incomplete messages with QUIC connection credit - -## Goal - -Native MoQ receivers bound incomplete frames and protocol messages by the -connection memory allowance, returning connection credit only when each -complete unit is parsed and handed off or discarded. When concurrent partial -messages prevent progress, the receiver cancels streams by MoQ priority and -continues. This works with Rust and JavaScript senders through tokio and -io_uring; completed media caching remains separately governed. - -## Plan - -Decided in the 2026-10-10 interview: cover headers and frame/group data alike. -The frame preallocation budget only bounds speculative allocation, and the -cache has soft limits; neither bounds aggregate incomplete messages. Use -peer-visible QUIC flow control instead of a separate header quota error. -The native transport primitive is a separate prerequisite; do not activate -whole-message retention without its recovery policy in this quest. - -That primitive landed as a provisional seam to finalize here: both native -drivers' receive streams have a copying `poll_read_held` that yields a -`Credit` for exactly the bytes read. Dropping a `Credit` returns it, so a -reader buffer keeps one merged `Credit` and drops `split(n)` for each -complete unit; read-ahead stays in the remainder. io_uring's `poll_closed` -read-ahead keeps its credit until the application reads it. Both drivers' -`Connection::stats().credit` reports held and still-available credit. -`poll_read_held` stays separate until this quest folds it into `poll_read`. -moq-net's transport trait and the WebTransport wrappers do not carry it yet; -reshape any of it to fit the consumer. - -Hold connection credit through reader buffers, type/length prefixes, headers, -extensions, and frame assembly. Attribute read-ahead bytes to the next unit -rather than releasing them with the completed one. Release on complete-unit -handoff, not after application playback or cache eviction. Avoid retaining -an entire multi-frame group as one unit. Account for received-but-unread and -out-of-order transport bytes as well as parser-held bytes; they share the -same allowance. Speculative allocations and copies need separate accounting -or bounds: QUIC byte credit is not a claim about total process RSS. - -Keep the existing stream-window setting and honor explicit overrides. -Give the native path a very large default so it normally adds no round trips; -stream credit is replenished on reads, independently of message completion. -Do not remove or ignore the setting without benchmark evidence and a new -maintainer decision. The connection memory allowance is the shared cap. - -Require that allowance to fit every supported encoded message, including -framing and extensions. Validate configuration and align native defaults -rather than discovering an impossible frame after receiving it. The current -32 MiB frame limit exceeds io_uring's 16 MiB connection default; a legacy -64 MiB announcement body also needs room for framing beyond 64 MiB. Derive -the minimum from the supported units, not just the payload limit. Lowering -a protocol/frame size limit is a separate maintainer decision, not an implicit -way around this requirement. Runtime memory-limit updates obey the same -minimum; already-advertised credit cannot be revoked. A rate limit can pace -a smaller amount of outstanding credit while the memory allowance remains -large enough for the complete unit. - -MoQ selects victims; the transport only reports pressure and sends -STOP_SENDING. Do not add sender-initiated shedding. This keeps recovery -available with browser senders and avoids treating intentional rate pacing -as receiver deadlock. The sender's required RESET_STREAM response is still -normal QUIC behavior, not a second recovery policy. - -Use the receiver's assembly and credit accounting to recover when partial -units exhaust progress capacity. Do not require peer DATA_BLOCKED, an -arbitrary timeout, loss, or a deadline. Demonstrate that useful work can -complete after cancellation, including when incoming streams keep arriving; -merely cycling through victims is not recovery. Distinguish bytes still able -to arrive under advertised credit from memory held by incomplete units. - -Within the existing broadcast fairness domains, cancel lower-priority media -and less useful groups first, respecting subscription priority and group -order. Prefer low-priority attributed media over unidentified headers so -headers have a chance to identify themselves; unidentified streams remain -eligible if they prevent progress. Control streams have the highest priority -but are eligible too. If canceling an essential control stream terminates the -session under its protocol, surface that outcome instead of leaving tasks -parked. There is no permanently protected class or separate control reserve. -Use deterministic tie-breaking without comparing unrelated broadcasts' raw -priority values. Do not invent another subscription-priority API here. - -Regression coverage must include many partial headers and frames filling the -shared allowance, a finite stream window smaller than a frame, the largest -supported encoded unit, read-ahead across unit boundaries, cancellation and -late reset/data races, unidentified and control-only pressure, and continued -progress after selecting victims, and run all-language interop. Wire checks -into CI; no wall-clock sleeps in unit tests. The JavaScript sender and the -retained-bytes, throughput, and latency sweep moved to -[their own quest](/quest/m1/quic/receive-memory-coverage.md). - -Public API: native receive-credit integration and validation of the existing -window configuration; no new header-specific quota. Wire: existing QUIC -flow-control and STOP_SENDING/RESET_STREAM behavior, no MoQ format change. -Browser/iroh receive-side guarantees and completed-cache limits are outside -this quest. Update existing configuration and transport docs inline; no -separate guide quest is needed. - -## Related - -- [Prove receive memory](/quest/m1/quic/receive-memory-coverage.md) - a JavaScript sender against both native receivers, and the nightly benchmark sweep -- [Scope track priority](/quest/m1/track-priority-scope.md) - broadcast fairness domains and subscription priority semantics -- [Hierarchical stream scheduling](/quest/m1/quic/scheduler.md) - share priority semantics without requiring sender-side recovery -- [QUIC bitrate caps](/quest/m2/rate-quic.md) - pace credit grants inside the independently bounded memory allowance -- [Relay peers get wider limits](/quest/m1/quic/peer-limits.md) - client and peer windows must both fit supported units diff --git a/quest/m2/rate-quic.md b/quest/m2/rate-quic.md index b0648fffb42..6dc277b1eee 100644 --- a/quest/m2/rate-quic.md +++ b/quest/m2/rate-quic.md @@ -18,8 +18,8 @@ backpressure the peer cannot ignore, rather than metering and closing. so a modified client cannot exceed the cap. An honest one goes flow-control limited (its congestion controller does not back off) and queues until [the grant](/quest/m2/rate-grant.md) clamps its encoder. - Keep rate pacing separate from the connection memory allowance in - [message assembly](/quest/m1/quic/receive-memory.md). The memory allowance + Keep rate pacing separate from the connection memory allowance that + message assembly holds credit against. The memory allowance must fit the largest supported encoded unit even when it takes many rate intervals to receive. Limit each grant by both rate allowance and available memory backing; do not wait for a whole-message credit release to grant @@ -84,5 +84,4 @@ that a one-second rate window can also be the complete assembly allowance. ## Required - [Peer limits](/quest/m1/quic/peer-limits.md) - adds the runtime `set_limits(Limits)` seam and shrink-as-debt behavior this extends -- [Bound message assembly](/quest/m1/quic/receive-memory.md) - separate retained memory from paced outstanding credit - [Bitrate claim](/quest/m2/rate-claim.md) - the cap this enforces From 661b8bb20458f261fcf9c9903ec58b371eaa2d12 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:08:49 -0700 Subject: [PATCH 34/51] fix(moq-net): pick a pressure victim in linear time Summing each broadcast's held credit searched a list of the broadcasts seen so far, so a stall cost O(streams x broadcasts). Sum by path in a map, adding up runs of a shared Arc before hashing, and benchmark both axes nightly. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/pressure.rs | 59 ++++++++++++++++++++++++++++++++++---- sh/gh/nightly.sh | 1 + 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index fb368f0f49f..546e73db3e5 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -21,6 +21,7 @@ //! always leaves room for one to complete. use std::{ + collections::HashMap, sync::{ Arc, Mutex, atomic::{AtomicBool, AtomicU64, Ordering}, @@ -136,16 +137,26 @@ fn victim(entries: &Slab) -> Option { .filter(|(_, entry)| held(entry) > 0 && !entry.slot.evicted.load(Ordering::Relaxed)) }; - // Media: the broadcast holding the most, then its least useful stream. - let mut domains: Vec<(&Arc, u64)> = Vec::new(); + // Media: the broadcast holding the most, then its least useful stream. Keyed by path + // rather than by `Arc`, since separate subscriptions to one broadcast share no `Arc`. + // A run of streams sharing one `Arc` sums before it is hashed. + let mut domains: HashMap<&PathOwned, u64> = HashMap::new(); + let mut run: Option<(&Arc, u64)> = None; for (_, entry) in holding() { if let Rank::Media { broadcast, .. } = &entry.rank { - match domains.iter_mut().find(|(path, _)| *path == broadcast) { - Some((_, total)) => *total += held(entry), - None => domains.push((broadcast, held(entry))), + match &mut run { + Some((path, total)) if Arc::ptr_eq(path, broadcast) => *total += held(entry), + _ => { + if let Some((path, total)) = run.replace((broadcast, held(entry))) { + *domains.entry(path.as_ref()).or_default() += total; + } + } } } } + if let Some((path, total)) = run { + *domains.entry(path.as_ref()).or_default() += total; + } // The larger holder, then the lower path, so the choice never depends on order. let domain = domains .into_iter() @@ -153,7 +164,7 @@ fn victim(entries: &Slab) -> Option { if let Some((domain, _)) = domain { return holding() .filter_map(|(id, entry)| match &entry.rank { - Rank::Media { broadcast, priority } if broadcast == domain => Some((id, priority, entry)), + Rank::Media { broadcast, priority } if broadcast.as_ref() == domain => Some((id, priority, entry)), _ => None, }) // `Priority` sorts the most urgent first, so the greatest is the least useful. @@ -370,4 +381,40 @@ mod tests { assert_eq!(count.0.load(Ordering::Relaxed), 1); assert!(ticket.poll_evicted(&Context::from_waker(&waker))); } + + /// Choose a victim among streams spread across broadcasts. The cost per stream must stay + /// flat along both axes. Run by nightly. + #[test] + #[ignore = "release benchmark, run by nightly"] + fn benchmark_victim() { + println!("streams,broadcasts,ns_per_choice,ns_per_stream"); + for streams in [64usize, 1024, 16384] { + for broadcasts in [1usize, 64, 1024, 16384] { + if broadcasts > streams { + continue; + } + let paths: Vec<_> = (0..broadcasts) + .map(|b| Arc::new(Path::new(&format!("broadcast/{b}")).to_owned())) + .collect(); + let pressure = Pressure::default(); + let _tickets: Vec<_> = (0..streams) + .map(|i| { + let rank = Rank::Media { + broadcast: paths[i % broadcasts].clone(), + priority: Priority::new((i % 256) as u8, 0, i as u64), + }; + hold(&pressure, rank, 1000 + i as u64) + }) + .collect(); + let rounds = (1 << 22) / streams; + let state = pressure.state.lock(); + let start = std::time::Instant::now(); + for _ in 0..rounds { + std::hint::black_box(victim(std::hint::black_box(&state.entries))); + } + let ns = start.elapsed().as_nanos() as f64 / rounds as f64; + println!("{streams},{broadcasts},{ns:.0},{:.1}", ns / streams as f64); + } + } + } } diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 0c8d681d3e0..84c4f3a7c74 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -26,6 +26,7 @@ case "${1:-}" in run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate run just rs test -p moq-tokio --release -E 'test(benchmark_held_reads)' --run-ignored only --success-output immediate + run just rs test -p moq-net --release -E 'test(benchmark_victim)' --run-ignored only --success-output immediate # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test # Exercise the optional thread-CPU clock as well as the normal elapsed-time smoke. From 2d468d4574f6e20438cf7159125e1451c3b29cd4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:15:43 -0700 Subject: [PATCH 35/51] fix(moq-net): report a memory window fill only while it lasts A fill latched when held credit reached the limit, and stayed latched if the credit returned before the monitor polled, so a stale report cancelled a stream that was no longer blocking anything. Recheck the window when consuming the report, as the QUIC drivers do. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/transport/memory.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 12cef7337e6..d09244f4401 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -479,7 +479,15 @@ impl poll::Session for Session { } fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()> { - self.side.conn.fills[self.side.local].poll(cx) + let local = self.side.local; + // A fill whose credit returned before this poll relieves nothing, as in the QUIC + // drivers: report it only while the window is still exhausted. + loop { + ready!(self.side.conn.fills[local].poll(cx)); + if self.side.conn.windows[local].read().exhausted { + return Poll::Ready(()); + } + } } fn stats(&self) -> impl super::Stats { @@ -1115,6 +1123,25 @@ mod tests { drop((held, more)); } + /// A fill whose credit returns before the monitor polls is not reported, so nothing is + /// cancelled once progress is possible again. + #[test] + fn a_fill_released_before_the_poll_is_not_reported() { + let (mut a, mut b) = pair(window(8)); + let (mut send, mut recv) = uni(&mut a, &mut b); + + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 8]))).unwrap(), 8); + let (_, mut held) = ready(recv.poll_read_chunk(&mut noop(), 8)).unwrap().unwrap(); + drop(held.split(4)); + assert!(b.poll_credit_exhausted(&mut noop()).is_pending()); + + // Filling the window again is still reported. + assert_eq!(ready(write(&mut send, &Bytes::from(vec![1u8; 8]))).unwrap(), 4); + let (_, more) = ready(recv.poll_read_chunk(&mut noop(), 8)).unwrap().unwrap(); + ready(b.poll_credit_exhausted(&mut noop())); + drop((held, more)); + } + /// The peer waiting on a fill wakes when it happens. #[test] fn exhaustion_wakes_the_monitor() { From 093c45b02a50002ce572ca3727ad3415b6baf8d1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:23:24 -0700 Subject: [PATCH 36/51] fix(moq-quic): credit a reset stream's bytes once when it is then stopped RESET_STREAM returns connection credit up to the final size, and a later stop returned the unread bytes again, over-granting MAX_DATA and underflowing the unread count in credit_stats. A stop after a reset now returns nothing, and add_read_credits asserts credit never outruns the bytes received. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/streams/recv.rs | 8 +++-- rs/moq-quic/src/connection/streams/state.rs | 34 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index 526b4343e37..f9c83b8fe0c 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -95,8 +95,12 @@ impl Recv { self.stopped = true; self.assembler.clear(); - // Issue flow control credit for unread data - let read_credits = self.end - self.assembler.bytes_read(); + // Issue flow control credit for unread data. A reset already returned it up to the final + // size, so returning it again would grant the peer window it never gave back. + let read_credits = match self.state { + RecvState::Recv { .. } => self.end - self.assembler.bytes_read(), + RecvState::ResetRecvd { .. } => 0, + }; // This may send a spurious STOP_SENDING if we've already received all data, but it's a bit // fiddly to distinguish that from the case where we've received a FIN but are missing some // data that the peer might still be trying to retransmit, in which case a STOP_SENDING is diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 5c33fd3e0bf..72b4e0936aa 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -1000,6 +1000,13 @@ impl StreamsState { /// again. pub(super) fn add_read_credits(&mut self, credits: u64) -> ShouldTransmit { self.credited += credits; + debug_assert!( + self.credited + self.held <= self.data_recvd, + "credited {} and held {} past the {} bytes received", + self.credited, + self.held, + self.data_recvd + ); if credits > self.receive_window_shrink_debt { let net_credits = credits - self.receive_window_shrink_debt; self.local_max_data = self.local_max_data.saturating_add(net_credits); @@ -2356,6 +2363,8 @@ mod tests { // credit, expecting all of them added to local_max_data let credits = 1024u64; + // As if that many bytes arrived and were read: credit never outruns what was received. + server.data_recvd += credits; let should_transmit = server.add_read_credits(credits); assert_eq!(server.receive_window_shrink_debt, 0); assert_eq!(server.local_max_data, prev_local_max_data + credits); @@ -2381,6 +2390,7 @@ mod tests { let credits = 1024u64; for _ in 0..2 { let expected_receive_window_shrink_debt = server.receive_window_shrink_debt - credits; + server.data_recvd += credits; let should_transmit = server.add_read_credits(credits); assert_eq!( server.receive_window_shrink_debt, @@ -2395,6 +2405,7 @@ mod tests { let prev_local_max_data = server.local_max_data; let expected_local_max_data = server.local_max_data + (credits - server.receive_window_shrink_debt); + server.data_recvd += credits; let _should_transmit = server.add_read_credits(credits); assert_eq!(server.receive_window_shrink_debt, 0); assert_eq!(server.local_max_data, expected_local_max_data); @@ -2403,6 +2414,7 @@ mod tests { // credit again, all should be added to local_max_data let credits = 1024 * 512; let expected_local_max_data = server.local_max_data + credits; + server.data_recvd += credits; let should_transmit = server.add_read_credits(credits); assert_eq!(server.receive_window_shrink_debt, 0); assert_eq!(server.local_max_data, expected_local_max_data); @@ -2587,6 +2599,7 @@ mod tests { Stop, Reset, StopThenReset, + ResetThenStop, } fn data(client: &mut StreamsState, id: StreamId, offset: u64, len: usize, fin: bool) { @@ -2632,7 +2645,13 @@ mod tests { /// it takes and whether the held bytes are released before or after its state is reclaimed. #[test] fn held_credit_returns_once() { - for end in [End::Fin, End::Stop, End::Reset, End::StopThenReset] { + for end in [ + End::Fin, + End::Stop, + End::Reset, + End::StopThenReset, + End::ResetThenStop, + ] { let mut client = make(Side::Client); let initial = client.local_max_data; let id = StreamId::new(Side::Server, Dir::Uni, 0); @@ -2698,6 +2717,19 @@ mod tests { reset(&mut client, id, 128); 128 } + End::ResetThenStop => { + // The reset returned everything past the held prefix; the stop adds nothing. + reset(&mut client, id, 128); + let mut pending = Retransmits::default(); + RecvStream { + id, + state: &mut client, + pending: &mut pending, + } + .stop(0u32.into()) + .unwrap(); + 128 + } }; assert!( !client.recv.contains_key(&id), From ebb81d91e8b2190eb61d380270ab0a483ce1a62f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:44:47 -0700 Subject: [PATCH 37/51] test: link the receive credit tests into each crate's integration binary Follow #5689's layout, so the moq-net, moq-tokio and moq-uring `receive_credit` tests build as modules of `tests/integration` rather than as their own binaries. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/tests/integration/main.rs | 1 + rs/moq-net/tests/{ => integration}/receive_credit.rs | 2 +- rs/moq-tokio/tests/integration/main.rs | 1 + rs/moq-tokio/tests/{ => integration}/receive_credit.rs | 0 rs/moq-uring/tests/integration/main.rs | 1 + rs/moq-uring/tests/{ => integration}/receive_credit.rs | 3 +-- 6 files changed, 5 insertions(+), 3 deletions(-) rename rs/moq-net/tests/{ => integration}/receive_credit.rs (99%) rename rs/moq-tokio/tests/{ => integration}/receive_credit.rs (100%) rename rs/moq-uring/tests/{ => integration}/receive_credit.rs (99%) diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index d155bb218f5..bf11c99ba25 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -33,6 +33,7 @@ mod mesh_withdraw; mod model; mod multi_cdn; mod quiet_catalog; +mod receive_credit; mod rejoin; mod replacement; mod request_caps; diff --git a/rs/moq-net/tests/receive_credit.rs b/rs/moq-net/tests/integration/receive_credit.rs similarity index 99% rename from rs/moq-net/tests/receive_credit.rs rename to rs/moq-net/tests/integration/receive_credit.rs index e617b8fa804..8b4c7e9d594 100644 --- a/rs/moq-net/tests/receive_credit.rs +++ b/rs/moq-net/tests/integration/receive_credit.rs @@ -7,7 +7,7 @@ //! stands in for QUIC connection flow control. Without recovery each of these stalls: //! the sender has spent every byte of credit on messages that can never complete. -mod support; +use crate::support; use std::time::Duration; diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 79552d9449e..46e1819dbfa 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -8,6 +8,7 @@ mod backend; mod broadcast; mod close_code; mod max_age; +mod receive_credit; mod reconnect; mod stream_code; mod subscription_end_integrity; diff --git a/rs/moq-tokio/tests/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs similarity index 100% rename from rs/moq-tokio/tests/receive_credit.rs rename to rs/moq-tokio/tests/integration/receive_credit.rs diff --git a/rs/moq-uring/tests/integration/main.rs b/rs/moq-uring/tests/integration/main.rs index ae4d797d51d..e03fe8440fa 100644 --- a/rs/moq-uring/tests/integration/main.rs +++ b/rs/moq-uring/tests/integration/main.rs @@ -11,6 +11,7 @@ mod ecn; mod endpoint; mod identity; mod qlog; +mod receive_credit; mod session; mod support; mod teardown; diff --git a/rs/moq-uring/tests/receive_credit.rs b/rs/moq-uring/tests/integration/receive_credit.rs similarity index 99% rename from rs/moq-uring/tests/receive_credit.rs rename to rs/moq-uring/tests/integration/receive_credit.rs index 92246d41984..e0d952f71a6 100644 --- a/rs/moq-uring/tests/receive_credit.rs +++ b/rs/moq-uring/tests/integration/receive_credit.rs @@ -10,8 +10,7 @@ #![cfg(all(target_os = "linux", feature = "quic"))] -#[path = "support.rs"] -mod support; +use crate::support; use std::net::UdpSocket; use std::task::Poll; From f229baa915b78167847967dd8bdd6bb5b79a379a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:07:37 -0700 Subject: [PATCH 38/51] fix(moq-net): evict a subgroup tail parked on its fill A tail stream waiting for its fill's head still holds credit for what it read ahead, and that credit can be what starves the fill. The fill handoff now polls eviction like the alias and SUBSCRIBE_OK waits do. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 37 ++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 7d87c9e59ef..aa9df72e10e 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3937,7 +3937,10 @@ where // A group starting partway through is the tail of one the fill began, and // without that head it has a hole at the front. - Some(start) => match self.claim_fill(fill, track, sequence, Some(start), reading).await? { + Some(start) => match self + .claim_fill(stream, fill, track, sequence, Some(start), reading) + .await? + { Some(producer) => Ok(Opened::Group(producer, start)), None => { tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); @@ -3947,7 +3950,7 @@ where // A stream that ends without an object: the group is over and had nothing // outside the fill's range, so the head it delivered is the whole group. - None => match self.claim_fill(fill, track, sequence, None, reading).await? { + None => match self.claim_fill(stream, fill, track, sequence, None, reading).await? { Some(producer) => Ok(Opened::Group(producer, 0)), None => Ok(Opened::Group(create(track)?, 0)), }, @@ -3966,9 +3969,11 @@ where /// with the subscription, so a publisher that promises a fill and never delivers one /// costs this stream and nothing else. Meanwhile this stream stops holding the /// subscription's end open: the fill's own stream holds it if its header arrived, and - /// the grace gives up on it if not. + /// the grace gives up on it if not. It still holds credit for what it read ahead, so it + /// also ends on eviction: the head may be the very stream that credit starves. async fn claim_fill( &self, + stream: &mut Reader, fill: &kio::Producer, track: &track::Producer, sequence: u64, @@ -3980,6 +3985,9 @@ where if let Poll::Ready(err) = track.poll_closed(waiter) { return Poll::Ready(Err(err)); } + if stream.poll_evicted(&waiter.context()) { + return Poll::Ready(Err(Error::Evicted)); + } let settled = fill.poll(waiter, |fill| match **fill { Fill::Requested | Fill::Serving(_) | Fill::Active => Poll::Pending, @@ -10960,6 +10968,29 @@ mod stitch_tests { assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); } + /// A tail parked on its head still holds credit for what it read ahead, which may be + /// what starves the head's stream, so eviction has to reach it there. + #[moq_net_sim::test] + async fn an_evicted_tail_stops_waiting_for_its_head() { + let h = Harness::new( + Fill::Serving(Some(Timescale::MICRO)), + vec![tail_stream(SEQUENCE, 2, &[b"tail-2"])], + ); + let pressure = crate::pressure::Pressure::default(); + let mut tail = h.stream().await; + tail.track(&pressure, crate::pressure::Rank::Unidentified); + + let mut subscriber = h.subscriber.clone(); + let mut recv = std::pin::pin!(subscriber.recv_group(&mut tail)); + assert!(futures::poll!(recv.as_mut()).is_pending(), "parked on the head"); + assert!(pressure.evict()); + assert!(matches!( + futures::poll!(recv.as_mut()), + Poll::Ready(Err(Error::Evicted)) + )); + assert_eq!(pressure.held(), 0); + } + /// A tail parked on its head stops holding the end open, so the finished fill does until /// the head is claimed: otherwise the subscription looks settled between the fill /// finishing and the tail waking, and ends before the tail is read. From 78065cf64f37b92df44c61743a47009721efed52 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:14:52 -0700 Subject: [PATCH 39/51] fix(moq-net): end an evicted data stream wherever its handler waits A subgroup tail parked on its fill handoff held the read-ahead it peeked, but only a pending read hears an eviction, so the stream chosen to free the window never stopped and the fill it waited on could not arrive. Guard each IETF data stream handler, `recv_group` and `recv_fill`, with an `Eviction` handle that hears the monitor apart from the borrowed reader. Every wait inside now ends with Evicted, replacing the hand-placed checks on the alias and SUBSCRIBE_OK waits. The lite data path only ever pends on its reader, so it needs no guard. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 12 +++--- rs/moq-net/src/ietf/subscriber.rs | 72 ++++++++++++++++++++++++------- rs/moq-net/src/pressure.rs | 65 +++++++++++++++++++++++----- 3 files changed, 117 insertions(+), 32 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index fdace948873..4e6a6eead7b 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -9,7 +9,7 @@ use bytes::{Buf, Bytes, BytesMut}; use crate::{ Error, StreamError, coding::*, - pressure::{Pressure, Rank, Ticket}, + pressure::{Eviction, Pressure, Rank, Ticket}, transport::Credit, }; @@ -115,11 +115,11 @@ impl Reader { self.release(used as u64); } - /// Whether this stream was chosen to free the receive allowance, registering `cx` to - /// hear when it is. A caller that parks the reader on something else races this, since - /// only a pending read registers on its own. - pub(crate) fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { - self.evicted(Some(cx)) + /// Hear this stream's eviction without borrowing the reader. Only a pending read + /// registers on its own, so a handler that parks the reader on something else races + /// this with [`Eviction::guard`]. + pub(crate) fn eviction(&self) -> Eviction { + self.ticket.as_ref().map_or_else(Eviction::none, Ticket::eviction) } /// Whether this stream was evicted, registering `cx` when given. The first time, stop diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 7d87c9e59ef..b4858dd8d77 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -3519,7 +3519,14 @@ where Err(request::from_code(code, request::Kind::TrackStatus, self.version)) } + /// Serve a subgroup stream. It ends once the pressure monitor evicts the stream, even + /// from a wait parked off it, such as on its alias or its fill. pub async fn recv_group(&mut self, stream: &mut Reader) -> Result<(), Error> { + let eviction = stream.eviction(); + eviction.guard(self.serve_group(stream)).await + } + + async fn serve_group(&mut self, stream: &mut Reader) -> Result<(), Error> { let group: ietf::GroupHeader = stream.decode().await?; if group.sub_group_id != 0 { @@ -3530,17 +3537,7 @@ where // SUBSCRIBE_OK or PUBLISH can be reordered behind this stream. Hold only the // subgroup header while waiting so the data stream cannot consume flow control. let aliases = self.state.lock().aliases.consume(); - let resolved = { - let mut resolve = std::pin::pin!(resolve_track_alias(&self.runtime, aliases, group.track_alias)); - // Parked off the stream, so an eviction would otherwise wait out the alias. - std::future::poll_fn(|cx| match stream.poll_evicted(cx) { - true => Poll::Ready(Err(Error::Evicted)), - false => resolve.as_mut().poll(cx), - }) - .await - }; - let request_id = match resolved { - Err(Error::Evicted) => return Err(Error::Evicted), + let request_id = match resolve_track_alias(&self.runtime, aliases, group.track_alias).await { Ok(request_id) => request_id, // Ours: we cancelled the subscription and the publisher has not stopped yet. Err(err @ Error::Cancel) => { @@ -4080,7 +4077,14 @@ where /// into a group producer of its own and hands that to the subgroup stream carrying the /// rest of the group; see [`Fill`]. A reset stream is the publisher's fill-failure /// signal, and arrives here as a read error, which drops the head and the join with it. + /// Serve a fetch stream. It ends once the pressure monitor evicts the stream, even from a + /// wait parked off it, such as on SUBSCRIBE_OK. pub async fn recv_fill(&mut self, stream: &mut Reader) -> Result<(), Error> { + let eviction = stream.eviction(); + eviction.guard(self.serve_fill(stream)).await + } + + async fn serve_fill(&mut self, stream: &mut Reader) -> Result<(), Error> { // The dispatcher peeked the stream type to get here. let _ = stream.varint().await?; let header: ietf::FetchHeader = stream.decode().await?; @@ -4119,10 +4123,6 @@ where // be reordered ahead of it. Taking the fill in the same step is what refuses a second // stream for a request that asked for one fill. let timescale = kio::wait(|waiter| { - // Parked off the stream, so an eviction would otherwise wait for SUBSCRIBE_OK. - if stream.poll_evicted(&waiter.context()) { - return Poll::Ready(Err(Error::Evicted)); - } let accepted = fill.poll(waiter, |fill| match **fill { Fill::Requested => Poll::Pending, _ => Poll::Ready(()), @@ -10960,6 +10960,48 @@ mod stitch_tests { assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); } + /// A tail parked on its head, holding the read-ahead it peeked, still hears the pressure + /// monitor choose it. It stops with Evicted and lets go of its credit, which the head may + /// need to arrive, and the head still lands. + #[moq_net_sim::test] + async fn an_evicted_tail_waiting_on_its_fill_lets_go() { + use crate::coding::StreamCodes as _; + + let h = Harness::new( + Fill::Serving(Some(Timescale::MICRO)), + vec![ + tail_stream(SEQUENCE, 2, &[b"tail-2"]), + fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), + ], + ); + let pressure = h.subscriber.pressure.clone(); + let mut tail = h.stream().await; + tail.track(&pressure, Rank::Unidentified); + + let mut serve_tail = h.subscriber.clone(); + let res = { + let mut parked = std::pin::pin!(serve_tail.recv_group(&mut tail)); + let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), parked.as_mut()).await; + assert!(waited.is_err(), "the tail did not wait for its head: {waited:?}"); + assert!(pressure.held() > 0, "the parked tail holds its read-ahead"); + + assert!(pressure.evict()); + moq_net_sim::timeout(std::time::Duration::from_secs(1), parked) + .await + .expect("the evicted tail never woke") + }; + assert!(matches!(res, Err(Error::Evicted)), "{res:?}"); + super::super::session::stop_on_error(&mut tail, res).expect("the session survives"); + assert_eq!(pressure.held(), 0, "the evicted tail let go of its credit"); + assert_eq!( + h.session.log.stops(), + vec![VERSION.encode_stream_code(&crate::StreamError::Evicted)] + ); + + let mut fill = h.stream().await; + h.subscriber.clone().recv_fill(&mut fill).await.expect("the head lands"); + } + /// A tail parked on its head stops holding the end open, so the finished fill does until /// the head is claimed: otherwise the subscription looks settled between the fill /// finishing and the tail waking, and ends before the tail is read. diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 546e73db3e5..a581101cc49 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -80,6 +80,22 @@ struct Slot { waker: Mutex>, } +impl Slot { + fn poll_evicted(&self, cx: &Context<'_>) -> bool { + if self.evicted.load(Ordering::Acquire) { + return true; + } + let mut waker = self.waker.lock().unwrap_or_else(|err| err.into_inner()); + match waker.as_mut() { + Some(waker) => waker.clone_from(cx.waker()), + None => *waker = Some(cx.waker().clone()), + } + drop(waker); + // Checked again: an eviction between the load and the store took no waker. + self.evicted.load(Ordering::Acquire) + } +} + impl Pressure { /// Track a new receiver at `rank`. pub fn track(&self, rank: Rank) -> Ticket { @@ -210,17 +226,44 @@ impl Ticket { /// Whether the monitor chose this stream, registering `cx` to hear when it does. pub fn poll_evicted(&self, cx: &Context<'_>) -> bool { - if self.slot.evicted.load(Ordering::Acquire) { - return true; - } - let mut waker = self.slot.waker.lock().unwrap_or_else(|err| err.into_inner()); - match waker.as_mut() { - Some(waker) => waker.clone_from(cx.waker()), - None => *waker = Some(cx.waker().clone()), - } - drop(waker); - // Checked again: an eviction between the load and the store took no waker. - self.slot.evicted.load(Ordering::Acquire) + self.slot.poll_evicted(cx) + } + + /// Hear this stream's eviction apart from the reader holding the ticket. + pub fn eviction(&self) -> Eviction { + Eviction(Some(self.slot.clone())) + } +} + +/// Hears that the monitor chose a stream without borrowing its reader, so a handler that +/// lent the reader to a wait parked elsewhere still ends. Empty for an untracked reader. +pub(crate) struct Eviction(Option>); + +impl Eviction { + /// For an untracked reader, which is never evicted. + pub fn none() -> Self { + Self(None) + } + + /// Run `serve` to its end, or fail it with [`Error::Evicted`](crate::Error::Evicted) + /// once the monitor chooses the stream, whatever `serve` is parked on. + pub async fn guard(&self, serve: impl Future>) -> Result { + let Some(slot) = &self.0 else { + return serve.await; + }; + let mut serve = std::pin::pin!(serve); + std::future::poll_fn(|cx| { + if let Poll::Ready(res) = serve.as_mut().poll(cx) { + return Poll::Ready(res); + } + // After `serve`, whose reads register on the same slot, so the eviction wakes + // this task at the top rather than a wait that will not look. + match slot.poll_evicted(cx) { + true => Poll::Ready(Err(crate::Error::Evicted)), + false => Poll::Pending, + } + }) + .await } } From baee41e9335605781d3d31d76d9c38fc71a89352 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:31:04 -0700 Subject: [PATCH 40/51] fix(moq-net): end evicted control streams wherever their handler waits A lite prefix stream waiting on the peer's SETUP for its link cost kept the read-ahead it buffered past ANNOUNCE_OK, and never polled its reader, so an eviction never reached it. Share the IETF data guard: `Eviction::check` fails a pending state machine once its stream is evicted, and `Eviction::guard` wraps an async handler. The lite announce prefix and every accepted lite control stream check it at the top of their machine; the IETF uni and request dispatch roots guard their handlers, and drop an evicted stream without failing the session. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/session.rs | 20 ++++++++-- rs/moq-net/src/lite/publisher.rs | 12 +++++- rs/moq-net/src/lite/subscriber.rs | 62 ++++++++++++++++++++++++++++++- rs/moq-net/src/pressure.rs | 25 +++++++------ 4 files changed, 101 insertions(+), 18 deletions(-) diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index ac709b5ae6b..41387007c43 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -974,9 +974,15 @@ where let uni = uni.clone(); let fatal = fatal.clone(); + // Its handler may park off the reader. Dropping it lets go of the stream's credit. + let eviction = reader.eviction(); tasks.push(async move { - if let Err(err) = uni.serve(reader).await { - fatal.lock().get_or_insert(err); + match eviction.guard(uni.serve(reader)).await { + Ok(()) => {} + Err(Error::Evicted) => tracing::debug!("uni stream evicted"), + Err(err) => { + fatal.lock().get_or_insert(err); + } } }); } @@ -1238,9 +1244,15 @@ where // header here could block later streams whose unread data holds flow control. let bidi = bidi.clone(); let fatal = fatal.clone(); + // Its handler may park off the reader. Dropping it lets go of the stream's credit. + let eviction = stream.reader.eviction(); tasks.push(async move { - if let Err(err) = bidi.serve(stream).await { - fatal.lock().get_or_insert(err); + match eviction.guard(bidi.serve(stream)).await { + Ok(()) => {} + Err(Error::Evicted) => tracing::debug!("request stream evicted"), + Err(err) => { + fatal.lock().get_or_insert(err); + } } }); }; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 5612f6e8400..3aec35ce377 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -239,6 +239,7 @@ where self.children.push(Control { shared: self.shared.clone(), runtime: self.runtime.clone(), + eviction: stream.reader.eviction(), state: ControlState::Start { stream }, }); } @@ -285,6 +286,8 @@ struct Control { shared: Arc>, // Handed to the children that arm timers (PROBE, announce linger). runtime: crate::time::Clock, + /// Heard here because a child may park off the stream's reader. + eviction: crate::pressure::Eviction, state: ControlState, } @@ -313,7 +316,8 @@ impl kio::Task for Control { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { - if let Err(err) = ready!(self.poll_serve(waiter)) { + let res = self.poll_serve(waiter); + if let Err(err) = ready!(self.eviction.check(&waiter.context(), res)) { let err = lite::error::control(&mut self.shared.session.clone(), err); tracing::warn!(%err, "control stream error"); } @@ -5361,6 +5365,7 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream }, }; assert!(control.poll(&kio::Waiter::noop()).is_ready()); @@ -5396,6 +5401,7 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Start { stream }, }; assert!(control.poll(&kio::Waiter::noop()).is_ready()); @@ -6334,6 +6340,7 @@ mod tests { let mut control = Control { shared: shared.clone(), runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Subscribe(RequestServe::new(shared, probe.stream.unwrap())), }; assert!( @@ -6386,6 +6393,7 @@ mod tests { let mut control = Control { shared: shared.clone(), runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Subscribe(RequestServe::new(shared, probe.stream.unwrap())), }; let pending = @@ -6425,6 +6433,7 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream: probe.stream.unwrap(), }, @@ -6447,6 +6456,7 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), + eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream: probe.stream.unwrap(), }, diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 2285028b9e1..c4980749d4c 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -1315,6 +1315,8 @@ impl ProbeStream { struct AnnouncePrefix { subscriber: Subscriber, prefix: PathOwned, + /// Heard here because the link cost wait parks off the stream's reader. + eviction: crate::pressure::Eviction, state: PrefixState, } @@ -1358,6 +1360,7 @@ impl AnnouncePrefix { Self { subscriber, prefix, + eviction: crate::pressure::Eviction::none(), state: PrefixState::Open, } } @@ -1365,7 +1368,9 @@ impl AnnouncePrefix { /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - self.poll_run(waiter) + let res = self.poll_run(waiter); + self.eviction + .check(&waiter.context(), res) .map_err(|err| lite::error::control(&mut self.subscriber.session.clone(), err)) } @@ -1380,6 +1385,7 @@ impl AnnouncePrefix { &mut cx ))?; stream.reader.track(&self.subscriber.pressure, Rank::Control); + self.eviction = stream.reader.eviction(); stream.writer.buffer(&lite::ControlType::Announce)?; // Lite04/05: ask the peer to filter out announces that already passed @@ -4185,6 +4191,60 @@ mod tests { let _a = slots.acquire().expect("the failed stream kept a slot"); let _b = slots.acquire().expect("the failed stream kept a slot"); } + + /// A prefix stream waiting on the peer's SETUP for its link cost holds what it read + /// past ANNOUNCE_OK, so it still has to hear the pressure monitor choose it. + #[moq_net_sim::test] + async fn an_evicted_prefix_waiting_on_its_link_cost_lets_go() { + const VERSION: Version = Version::Lite06; + let mut script = Vec::new(); + lite::AnnounceOk { + origin: crate::Hop::new(9).unwrap(), + active: 1, + } + .encode(&mut crate::coding::Encoder::new(&mut script, VERSION.into()), VERSION) + .unwrap(); + lite::AnnounceBroadcast::Active { + epoch: None, + suffix: lite::PathRef::literal(Path::new("a")), + hops: lite::HopsRef::literal(crate::Hops::new()), + cost: crate::origin::Cost::default(), + } + .encode(&mut crate::coding::Encoder::new(&mut script, VERSION.into()), VERSION) + .unwrap(); + + let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + let subscriber = Subscriber::new(SubscriberConfig { + runtime: crate::time::Clock::sim(), + session: crate::lite::test_transport::ScriptedSession::new(script), + origin, + recv_bandwidth: None, + version: VERSION, + // No configured cost, and a SETUP that never arrives. + peer_setup: crate::lite::PeerSetup::new(false, crate::transport::Binding::InBand), + cost: None, + peer_hop: None, + going_away: Default::default(), + auth: crate::auth::Handle::new(false), + }); + let pressure = subscriber.pressure.clone(); + let mut prefix = AnnouncePrefix::new(subscriber, Path::new("").to_owned()); + + let res = { + let mut run = std::pin::pin!(kio::wait(|waiter| prefix.poll(waiter))); + let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), run.as_mut()).await; + assert!(waited.is_err(), "the prefix did not wait for its link cost: {waited:?}"); + assert!(pressure.held() > 0, "the parked prefix holds its read-ahead"); + + assert!(pressure.evict()); + moq_net_sim::timeout(std::time::Duration::from_secs(1), run) + .await + .expect("the evicted prefix never woke") + }; + assert!(matches!(res, Err(Error::Evicted)), "{res:?}"); + drop(prefix); + assert_eq!(pressure.held(), 0, "the evicted prefix let go of its credit"); + } } /// The four wire fields a subscription's half-open range encodes to. diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index a581101cc49..e9e623ba223 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -248,23 +248,24 @@ impl Eviction { /// Run `serve` to its end, or fail it with [`Error::Evicted`](crate::Error::Evicted) /// once the monitor chooses the stream, whatever `serve` is parked on. pub async fn guard(&self, serve: impl Future>) -> Result { - let Some(slot) = &self.0 else { - return serve.await; - }; let mut serve = std::pin::pin!(serve); std::future::poll_fn(|cx| { - if let Poll::Ready(res) = serve.as_mut().poll(cx) { - return Poll::Ready(res); - } - // After `serve`, whose reads register on the same slot, so the eviction wakes - // this task at the top rather than a wait that will not look. - match slot.poll_evicted(cx) { - true => Poll::Ready(Err(crate::Error::Evicted)), - false => Poll::Pending, - } + let res = serve.as_mut().poll(cx); + self.check(cx, res) }) .await } + + /// [`Self::guard`] for a state machine owning the stream: a pending `res` fails with + /// [`Error::Evicted`](crate::Error::Evicted) once the monitor chooses the stream. Call + /// it on the machine's result, after its reads, which register on the same slot, so the + /// eviction wakes the machine at the top rather than a wait that will not look. + pub fn check(&self, cx: &Context<'_>, res: Poll>) -> Poll> { + match (res, &self.0) { + (Poll::Pending, Some(slot)) if slot.poll_evicted(cx) => Poll::Ready(Err(crate::Error::Evicted)), + (res, _) => res, + } + } } impl Drop for Ticket { From 26ae1e7c5b2b2e08dad8fd013f2ca95a11d140a6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 20:13:04 -0700 Subject: [PATCH 41/51] fix(moq-net): evict a stream without its reader An eviction only flagged the stream, so a handler parked off its reader's read kept the credit and the stream until it polled again, and every such wait needed its own guard. Each new wait reopened the hole. The tracked reader's credit now lives in a shared `Hold`, and the pressure monitor evicts through it: it drops the credit and sends STOP_SENDING through a new `RecvStream::stopper()` handle, whatever the handler waits on. The handler hears `Evicted` on its next read. Every transport gains the stopper; those whose reads return their credit use `()`. The per-site `Eviction` guards are gone. Also restores the two closing braces the main merge dropped from moq-quic's state.rs and moq-uring's adapter.rs. Co-Authored-By: Claude Opus 5.5 --- rs/moq-ffi/src/transport/adapter.rs | 2 + rs/moq-net/src/client.rs | 4 + rs/moq-net/src/coding/reader.rs | 182 ++++-- rs/moq-net/src/fuzz.rs | 2 + rs/moq-net/src/ietf/adapter.rs | 39 ++ rs/moq-net/src/ietf/auth.rs | 4 +- rs/moq-net/src/ietf/publisher.rs | 2 +- rs/moq-net/src/ietf/session.rs | 22 +- rs/moq-net/src/ietf/subscriber.rs | 55 +- rs/moq-net/src/lite/publisher.rs | 14 +- rs/moq-net/src/lite/session.rs | 6 +- rs/moq-net/src/lite/subscriber.rs | 44 +- rs/moq-net/src/lite/test_transport.rs | 19 + rs/moq-net/src/pressure.rs | 556 ++++++++++++------- rs/moq-net/src/server.rs | 2 + rs/moq-net/src/transport.rs | 26 +- rs/moq-net/src/transport/memory.rs | 36 +- rs/moq-net/tests/integration/support/mock.rs | 33 ++ rs/moq-quic/src/connection/streams/state.rs | 2 + rs/moq-tokio/src/quic/driver/mod.rs | 2 +- rs/moq-tokio/src/quic/driver/recv_stream.rs | 41 ++ rs/moq-tokio/src/quic/web/recv.rs | 22 + rs/moq-tokio/src/transport.rs | 11 +- rs/moq-tokio/src/transport/held.rs | 16 + rs/moq-tokio/src/transport/owned.rs | 4 + rs/moq-uring/src/quic/driver/mod.rs | 2 +- rs/moq-uring/src/quic/driver/stream.rs | 33 ++ rs/moq-uring/src/quic/mod.rs | 2 +- rs/moq-uring/src/quic/web.rs | 25 + rs/moq-uring/src/transport/adapter.rs | 14 + rs/moq-uring/src/transport/held.rs | 11 + rs/moq-wasm/src/transport/adapter.rs | 2 + sh/gh/nightly.sh | 2 +- 33 files changed, 879 insertions(+), 358 deletions(-) diff --git a/rs/moq-ffi/src/transport/adapter.rs b/rs/moq-ffi/src/transport/adapter.rs index 61abe300f86..0ceb9e08da9 100644 --- a/rs/moq-ffi/src/transport/adapter.rs +++ b/rs/moq-ffi/src/transport/adapter.rs @@ -142,6 +142,8 @@ pub struct RecvStream(S); impl poll::RecvStream for RecvStream { type Error = Error; type Credit = moq::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { backend::RecvStream::poll_read(&mut self.0, cx, dst) .map_err(Error) diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index 25d1a7669e4..ca898b805e6 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -717,6 +717,8 @@ mod tests { impl crate::transport::poll::RecvStream for FakeRecvStream { type Error = FakeError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, @@ -1096,6 +1098,8 @@ mod tests { impl crate::transport::poll::RecvStream for LocalRecv { type Error = FakeError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 4e6a6eead7b..d6daba37d2a 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -1,6 +1,7 @@ use std::{ cmp, fmt::Debug, + sync::Arc, task::{Context, Poll, ready}, }; @@ -9,8 +10,7 @@ use bytes::{Buf, Bytes, BytesMut}; use crate::{ Error, StreamError, coding::*, - pressure::{Eviction, Pressure, Rank, Ticket}, - transport::Credit, + pressure::{Erase, Hold, Holder, Pressure, Rank}, }; /// A reader for decoding messages from a stream. @@ -28,13 +28,12 @@ pub struct Reader { stream: S, buffer: BytesMut, version: V, - /// Covers `buffer` plus `unit`, less `free`, while tracked. - credit: Option, /// Bytes of the incomplete frame already taken out of `buffer`. unit: u64, /// Leading bytes read before tracking began, whose credit already returned. free: u64, - ticket: Option, + /// While tracked, the credit for `buffer` plus `unit`, less `free`. + hold: Option>>, /// The receive allowance filled and this stream was chosen to free it. evicted: bool, } @@ -56,57 +55,47 @@ impl Reader { stream, buffer: Default::default(), version, - credit: None, unit: 0, free: 0, - ticket: None, + hold: None, evicted: false, } } - /// Hold connection credit until each unit completes, cancellable by `pressure` at - /// `rank`. - pub(crate) fn track(&mut self, pressure: &Pressure, rank: Rank) { - if self.ticket.is_some() { + /// Hold connection credit until each unit completes, evictable by `pressure` at `rank`. + pub(crate) fn track(&mut self, pressure: &Pressure, rank: Rank) + where + Hold: Erase, + { + if self.hold.is_some() { return self.rank(rank); } self.free = self.buffer.len() as u64 + self.unit; - self.ticket = Some(pressure.track(rank)); + let code = self.version.encode_stream_code(&StreamError::Evicted); + self.hold = Some(pressure.track(rank, self.stream.stopper(), code)); } /// Re-rank a tracked stream once its header identifies it. pub(crate) fn rank(&self, rank: Rank) { - if let Some(ticket) = &self.ticket { - ticket.rank(rank); + if let Some(hold) = &self.hold { + hold.rank(rank); } } /// Keep the credit for bytes just read, if tracked; otherwise it returns now. fn take(&mut self, credit: S::Credit) { - let Some(ticket) = &self.ticket else { return }; - let held = match &mut self.credit { - Some(held) => { - held.merge(credit); - held.len() - } - None => { - let len = credit.len(); - self.credit = Some(credit); - len - } - }; - ticket.held(held); + if let Some(hold) = &self.hold { + hold.take(credit); + } } /// Return the credit for `bytes` that completed a unit. fn release(&mut self, bytes: u64) { let free = self.free.min(bytes); self.free -= free; - let (Some(ticket), Some(credit)) = (&self.ticket, &mut self.credit) else { - return; - }; - drop(credit.split(bytes - free)); - ticket.held(credit.len()); + if let Some(hold) = &self.hold { + hold.release(bytes - free); + } } /// Drop `used` decoded bytes from the buffer, returning their credit. @@ -115,27 +104,20 @@ impl Reader { self.release(used as u64); } - /// Hear this stream's eviction without borrowing the reader. Only a pending read - /// registers on its own, so a handler that parks the reader on something else races - /// this with [`Eviction::guard`]. - pub(crate) fn eviction(&self) -> Eviction { - self.ticket.as_ref().map_or_else(Eviction::none, Ticket::eviction) - } - - /// Whether this stream was evicted, registering `cx` when given. The first time, stop - /// it and let go of everything it holds. + /// Whether this stream was evicted, registering `cx` when given. The eviction already + /// stopped the stream and took its credit, so what is buffered is all that is left. fn evicted(&mut self, cx: Option<&Context<'_>>) -> bool { if self.evicted { return true; } - let evicted = match (&self.ticket, cx) { + let evicted = match (&self.hold, cx) { (None, _) => false, - (Some(ticket), Some(cx)) => ticket.poll_evicted(cx), - (Some(ticket), None) => ticket.is_evicted(), + (Some(hold), Some(cx)) => hold.poll_evicted(cx), + (Some(hold), None) => hold.is_evicted(), }; if evicted { self.evicted = true; - self.abort(&Error::Evicted); + self.free = self.buffer.len() as u64 + self.unit; } evicted } @@ -455,15 +437,17 @@ impl Reader { /// Abort the stream with the given error, returning the credit it holds. pub fn abort(&mut self, err: impl Into) { - // STOP_SENDING is a stream operation, so it carries a stream code from the - // negotiated protocol's registry. A session code, or the other protocol's, would be - // read against the wrong table and mean something else. - self.stream.stop(self.version.encode_stream_code(&err.into())); + // An evicted stream was stopped with its own code already. + if !self.hold.as_ref().is_some_and(|hold| hold.is_evicted()) { + // STOP_SENDING is a stream operation, so it carries a stream code from the + // negotiated protocol's registry. A session code, or the other protocol's, would + // be read against the wrong table and mean something else. + self.stream.stop(self.version.encode_stream_code(&err.into())); + } // What is buffered stays readable, but nothing more is worth holding for. self.free = self.buffer.len() as u64 + self.unit; - self.credit = None; - if let Some(ticket) = &self.ticket { - ticket.held(0); + if let Some(hold) = &self.hold { + hold.clear(); } } @@ -473,10 +457,9 @@ impl Reader { stream: self.stream, buffer: self.buffer, version, - credit: self.credit, unit: self.unit, free: self.free, - ticket: self.ticket, + hold: self.hold, evicted: self.evicted, } } @@ -500,6 +483,8 @@ mod tests { impl crate::transport::poll::RecvStream for StopLog { type Error = crate::lite::test_transport::SinkError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, @@ -607,6 +592,8 @@ mod tests { impl crate::transport::poll::RecvStream for Chunks { type Error = crate::lite::test_transport::SinkError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, @@ -813,6 +800,8 @@ mod tests { impl crate::transport::poll::RecvStream for Burst { type Error = crate::lite::test_transport::SinkError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, @@ -873,6 +862,9 @@ mod tests { Context::from_waker(Waker::noop()) } + /// A registry of memory streams. + type Tracked = Pressure>; + /// A memory stream whose writer stays open, read by a reader the returned pressure /// tracks. `window` bounds what the reader's side accepts. fn tracked( @@ -880,7 +872,7 @@ mod tests { ) -> ( memory::SendStream, Reader, - Pressure, + Tracked, (memory::Session, memory::Session), ) { let (mut a, mut b) = memory::pair(memory::Config { @@ -893,7 +885,7 @@ mod tests { let Poll::Ready(Ok(recv)) = b.poll_accept_uni(&mut noop()) else { panic!("accept") }; - let pressure = Pressure::default(); + let pressure = Tracked::default(); let mut reader = Reader::new(recv, Version::Lite05); reader.track(&pressure, Rank::Unidentified); (send, reader, pressure, (a, b)) @@ -956,7 +948,7 @@ mod tests { assert_eq!(write(&mut send, &[0x07, 0x40]), Poll::Ready(2)); assert_eq!(varint(reader.poll_varint(&mut noop())), 7); - let pressure = Pressure::default(); + let pressure = Tracked::default(); reader.track(&pressure, Rank::Unidentified); assert_eq!(pressure.held(), 0); @@ -1010,6 +1002,82 @@ mod tests { assert_eq!(pressure.held(), 0); } + /// Hands out `max`-byte reads of a looping buffer of units, so only the reader costs. + struct Endless { + data: Bytes, + pos: usize, + max: usize, + } + + impl crate::transport::poll::RecvStream for Endless { + type Error = crate::lite::test_transport::SinkError; + type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} + + fn poll_read( + &mut self, + _: &mut Context<'_>, + dst: &mut [u8], + ) -> Poll, Self::Error>> { + let n = dst.len().min(self.max).min(self.data.len() - self.pos); + dst[..n].copy_from_slice(&self.data[self.pos..self.pos + n]); + self.pos = (self.pos + n) % self.data.len(); + Poll::Ready(Ok(Some(crate::transport::Unheld(n as u64)))) + } + + fn stop(&mut self, _code: u32) {} + + fn poll_closed(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + /// Decode size-prefixed units, untracked and tracked: the per-read and per-unit cost + /// of holding credit in the reader. Run by nightly. + #[test] + #[ignore = "release benchmark, run by nightly"] + fn benchmark_tracked_reads() { + const UNIT: usize = 1000; + const UNITS: usize = 1 << 16; + const RUNS: usize = 5; + let mut unit = Vec::new(); + Encoder::new(&mut unit, Version::Lite05.into()) + .varint(UNIT as u64) + .unwrap(); + unit.resize(unit.len() + UNIT, 7); + let data = Bytes::from(unit.repeat(1024)); + + println!("max_read,untracked_ns_per_unit,tracked_ns_per_unit"); + for max in [1200usize, 16 * 1024, 64 * 1024] { + let mut ns = [f64::MAX; 2]; + for _ in 0..RUNS { + for (mode, tracked) in [false, true].into_iter().enumerate() { + let stream = Endless { + data: data.clone(), + pos: 0, + max, + }; + let mut reader = Reader::new(stream, Version::Lite05); + let pressure = Pressure::>::default(); + if tracked { + reader.track(&pressure, Rank::Unidentified); + } + let start = std::time::Instant::now(); + for _ in 0..UNITS { + let size = varint(reader.poll_varint(&mut noop())); + let Poll::Ready(Ok(payload)) = reader.poll_read_exact(&mut noop(), size as usize) else { + panic!("short unit"); + }; + std::hint::black_box(payload); + } + ns[mode] = ns[mode].min(start.elapsed().as_nanos() as f64 / UNITS as f64); + } + } + println!("{max},{:.1},{:.1}", ns[0], ns[1]); + } + } + /// An evicted reader stops its stream and fails its next read. #[test] fn eviction_stops_the_stream() { diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 68ed79e49da..3a53a98acd9 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1169,6 +1169,8 @@ struct Chunks { impl crate::transport::poll::RecvStream for Chunks { type Error = NoError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, diff --git a/rs/moq-net/src/ietf/adapter.rs b/rs/moq-net/src/ietf/adapter.rs index 19520bc6577..e02adb2c861 100644 --- a/rs/moq-net/src/ietf/adapter.rs +++ b/rs/moq-net/src/ietf/adapter.rs @@ -165,6 +165,11 @@ impl VirtualRecvStream { impl crate::transport::poll::RecvStream for VirtualRecvStream { type Error = crate::Error; type Credit = Unheld; + type Stopper = VirtualStopper; + + fn stopper(&self) -> VirtualStopper { + VirtualStopper(self.rx.clone()) + } fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { if dst.is_empty() { @@ -243,6 +248,15 @@ impl crate::transport::poll::RecvStream for VirtualRecvStream { } } +/// Stops a [`VirtualRecvStream`] without borrowing it, discarding what is routed to it. +pub struct VirtualStopper(Queue); + +impl crate::transport::Stopper for VirtualStopper { + fn stop(&self, _code: u32) { + self.0.close(); + } +} + impl Drop for VirtualRecvStream { fn drop(&mut self) { // The reader is gone: close the queue so routed follow-ups are discarded @@ -508,10 +522,35 @@ impl Credit for AdapterCredit { } type RealCredit = <::RecvStream as crate::transport::poll::RecvStream>::Credit; +type RealStopper = + <::RecvStream as crate::transport::poll::RecvStream>::Stopper; + +/// Stops an [`AdapterRecv`] without borrowing it. +pub enum AdapterStopper { + Real(T), + Virtual(VirtualStopper), +} + +impl crate::transport::Stopper for AdapterStopper { + fn stop(&self, code: u32) { + match self { + Self::Real(s) => s.stop(code), + Self::Virtual(s) => s.stop(code), + } + } +} impl crate::transport::poll::RecvStream for AdapterRecv { type Error = crate::Error; type Credit = AdapterCredit>; + type Stopper = AdapterStopper>; + + fn stopper(&self) -> Self::Stopper { + match self { + Self::Real(s) => AdapterStopper::Real(s.stopper()), + Self::Virtual(s) => AdapterStopper::Virtual(s.stopper()), + } + } fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { match self { diff --git a/rs/moq-net/src/ietf/auth.rs b/rs/moq-net/src/ietf/auth.rs index ea8f7658a85..90d51865ca4 100644 --- a/rs/moq-net/src/ietf/auth.rs +++ b/rs/moq-net/src/ietf/auth.rs @@ -244,7 +244,7 @@ pub(super) async fn run_present( peer_setup: peer::PeerSetup, version: Version, going_away: crate::goaway::GoingAway, - pressure: crate::pressure::Pressure, + pressure: crate::pressure::Pressure, ) -> Result<(), Error> where S: crate::transport::poll::Boxable, @@ -311,7 +311,7 @@ struct Present { handle: Handle, version: Version, id: u64, - pressure: crate::pressure::Pressure, + pressure: crate::pressure::Pressure, } #[cfg(feature = "async")] diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 2c6cee2465b..ee4a2d1ae6f 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -541,7 +541,7 @@ pub(super) struct Publisher { pub(super) subscriptions: crate::session::Slots, pub(super) fetches: crate::session::Slots, /// Tracks each stream this half opens. - pub(super) pressure: crate::pressure::Pressure, + pub(super) pressure: crate::pressure::Pressure, // Ranks every subscription's group streams against each other for the transport. priority: PriorityQueue, } diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index 41387007c43..34dea080f77 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -181,7 +181,7 @@ where // Every stream the session reads holds its credit until a whole message arrives, and // the monitor cancels one by rank whenever that fills the receive allowance. - let pressure = crate::pressure::Pressure::default(); + let pressure = crate::pressure::Pressure::::default(); let driver = async move { // Held for the life of the session. @@ -974,15 +974,9 @@ where let uni = uni.clone(); let fatal = fatal.clone(); - // Its handler may park off the reader. Dropping it lets go of the stream's credit. - let eviction = reader.eviction(); tasks.push(async move { - match eviction.guard(uni.serve(reader)).await { - Ok(()) => {} - Err(Error::Evicted) => tracing::debug!("uni stream evicted"), - Err(err) => { - fatal.lock().get_or_insert(err); - } + if let Err(err) = uni.serve(reader).await { + fatal.lock().get_or_insert(err); } }); } @@ -1244,15 +1238,9 @@ where // header here could block later streams whose unread data holds flow control. let bidi = bidi.clone(); let fatal = fatal.clone(); - // Its handler may park off the reader. Dropping it lets go of the stream's credit. - let eviction = stream.reader.eviction(); tasks.push(async move { - match eviction.guard(bidi.serve(stream)).await { - Ok(()) => {} - Err(Error::Evicted) => tracing::debug!("request stream evicted"), - Err(err) => { - fatal.lock().get_or_insert(err); - } + if let Err(err) = bidi.serve(stream).await { + fatal.lock().get_or_insert(err); } }); }; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index b4858dd8d77..43bed146d73 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -707,7 +707,7 @@ pub(super) struct Subscriber { // Namespaces the peer may have announced at once (`session::Limits::announces`). pub(super) announces: crate::session::Slots, // Every stream this half reads, cancelled by rank when held receive credit fills. - pub(super) pressure: Pressure, + pub(super) pressure: Pressure, } /// The prefixes to issue SUBSCRIBE_NAMESPACE for: `origin`'s permitted scope, @@ -815,7 +815,7 @@ where /// Track the streams this half reads with the session's `pressure`. Set before the /// halves that clone this subscriber start. - pub fn with_pressure(mut self, pressure: Pressure) -> Self { + pub fn with_pressure(mut self, pressure: Pressure) -> Self { self.pressure = pressure; self } @@ -3519,14 +3519,7 @@ where Err(request::from_code(code, request::Kind::TrackStatus, self.version)) } - /// Serve a subgroup stream. It ends once the pressure monitor evicts the stream, even - /// from a wait parked off it, such as on its alias or its fill. pub async fn recv_group(&mut self, stream: &mut Reader) -> Result<(), Error> { - let eviction = stream.eviction(); - eviction.guard(self.serve_group(stream)).await - } - - async fn serve_group(&mut self, stream: &mut Reader) -> Result<(), Error> { let group: ietf::GroupHeader = stream.decode().await?; if group.sub_group_id != 0 { @@ -4077,14 +4070,7 @@ where /// into a group producer of its own and hands that to the subgroup stream carrying the /// rest of the group; see [`Fill`]. A reset stream is the publisher's fill-failure /// signal, and arrives here as a read error, which drops the head and the join with it. - /// Serve a fetch stream. It ends once the pressure monitor evicts the stream, even from a - /// wait parked off it, such as on SUBSCRIBE_OK. pub async fn recv_fill(&mut self, stream: &mut Reader) -> Result<(), Error> { - let eviction = stream.eviction(); - eviction.guard(self.serve_fill(stream)).await - } - - async fn serve_fill(&mut self, stream: &mut Reader) -> Result<(), Error> { // The dispatcher peeked the stream type to get here. let _ = stream.varint().await?; let header: ietf::FetchHeader = stream.decode().await?; @@ -10960,9 +10946,10 @@ mod stitch_tests { assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); } - /// A tail parked on its head, holding the read-ahead it peeked, still hears the pressure - /// monitor choose it. It stops with Evicted and lets go of its credit, which the head may - /// need to arrive, and the head still lands. + /// A tail parked on its head, holding the read-ahead it peeked, lets go the moment the + /// pressure monitor chooses it: the eviction itself returns the credit, which the head may + /// need to arrive, and stops the stream, without the tail being polled. The head still + /// lands, and the tail finishes from what it had already buffered. #[moq_net_sim::test] async fn an_evicted_tail_waiting_on_its_fill_lets_go() { use crate::coding::StreamCodes as _; @@ -10977,29 +10964,25 @@ mod stitch_tests { let pressure = h.subscriber.pressure.clone(); let mut tail = h.stream().await; tail.track(&pressure, Rank::Unidentified); + let evicted = vec![VERSION.encode_stream_code(&crate::StreamError::Evicted)]; let mut serve_tail = h.subscriber.clone(); - let res = { - let mut parked = std::pin::pin!(serve_tail.recv_group(&mut tail)); - let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), parked.as_mut()).await; - assert!(waited.is_err(), "the tail did not wait for its head: {waited:?}"); - assert!(pressure.held() > 0, "the parked tail holds its read-ahead"); + let mut parked = std::pin::pin!(serve_tail.recv_group(&mut tail)); + let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), parked.as_mut()).await; + assert!(waited.is_err(), "the tail did not wait for its head: {waited:?}"); + assert!(pressure.held() > 0, "the parked tail holds its read-ahead"); - assert!(pressure.evict()); - moq_net_sim::timeout(std::time::Duration::from_secs(1), parked) - .await - .expect("the evicted tail never woke") - }; - assert!(matches!(res, Err(Error::Evicted)), "{res:?}"); - super::super::session::stop_on_error(&mut tail, res).expect("the session survives"); - assert_eq!(pressure.held(), 0, "the evicted tail let go of its credit"); - assert_eq!( - h.session.log.stops(), - vec![VERSION.encode_stream_code(&crate::StreamError::Evicted)] - ); + assert!(pressure.evict()); + assert_eq!(pressure.held(), 0, "the eviction let go of the parked tail's credit"); + assert_eq!(h.session.log.stops(), evicted, "the eviction stopped the parked tail"); let mut fill = h.stream().await; h.subscriber.clone().recv_fill(&mut fill).await.expect("the head lands"); + moq_net_sim::timeout(std::time::Duration::from_secs(1), parked) + .await + .expect("the tail never took its head") + .expect("the tail finishes from its buffer"); + assert_eq!(h.session.log.stops(), evicted, "an evicted stream is stopped once"); } /// A tail parked on its head stops holding the end open, so the finished fill does until diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index faf165eb451..7e252540d12 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -177,7 +177,7 @@ pub(super) struct Publisher { /// Aborts the session on a publication outside our grant (dialing side only). enforce: Option, /// Tracks each accepted control stream's reads. - pub(super) pressure: crate::pressure::Pressure, + pub(super) pressure: crate::pressure::Of, } impl Publisher { @@ -240,7 +240,6 @@ where self.children.push(Control { shared: self.shared.clone(), runtime: self.runtime.clone(), - eviction: stream.reader.eviction(), state: ControlState::Start { stream }, }); } @@ -287,8 +286,6 @@ struct Control { shared: Arc>, // Handed to the children that arm timers (PROBE, announce linger). runtime: crate::time::Clock, - /// Heard here because a child may park off the stream's reader. - eviction: crate::pressure::Eviction, state: ControlState, } @@ -317,8 +314,7 @@ impl kio::Task for Control { type Output = (); fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { - let res = self.poll_serve(waiter); - if let Err(err) = ready!(self.eviction.check(&waiter.context(), res)) { + if let Err(err) = ready!(self.poll_serve(waiter)) { let err = lite::error::control(&mut self.shared.session.clone(), err); tracing::warn!(%err, "control stream error"); } @@ -5480,7 +5476,6 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream }, }; assert!(control.poll(&kio::Waiter::noop()).is_ready()); @@ -5516,7 +5511,6 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Start { stream }, }; assert!(control.poll(&kio::Waiter::noop()).is_ready()); @@ -6525,7 +6519,6 @@ mod tests { let mut control = Control { shared: shared.clone(), runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Subscribe(RequestServe::new(shared, probe.stream.unwrap())), }; assert!( @@ -6578,7 +6571,6 @@ mod tests { let mut control = Control { shared: shared.clone(), runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Subscribe(RequestServe::new(shared, probe.stream.unwrap())), }; let pending = @@ -6618,7 +6610,6 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream: probe.stream.unwrap(), }, @@ -6641,7 +6632,6 @@ mod tests { let mut control = Control { shared: probe.shared, runtime: crate::time::Clock::sim(), - eviction: crate::pressure::Eviction::none(), state: ControlState::Goaway { stream: probe.stream.unwrap(), }, diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 50b206195ee..7a6974a2605 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -375,7 +375,7 @@ pub(crate) struct Driver { /// Presenting our tokens, one AUTH stream each. auth: Present, /// Cancels a stream whenever held receive credit fills the allowance. - monitor: Monitor, + monitor: Monitor>, pub(crate) local_close: std::sync::Arc, /// Advertising our capabilities, or `None` once sent (or on a version with no /// Setup Stream). @@ -498,7 +498,7 @@ struct Present { /// The connection's own credential, held for the life of the session. _setup: Option, /// Tracks each AUTH stream's reads. - pressure: Pressure, + pressure: crate::pressure::Of, } impl Present { @@ -541,7 +541,7 @@ struct PresentToken { handle: crate::auth::Handle, going_away: crate::goaway::GoingAway, violation: kio::Shared>, - pressure: Pressure, + pressure: crate::pressure::Of, id: u64, state: PresentState, } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 839f3141ad1..e7323f97dd2 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -10,7 +10,7 @@ use crate::{ AsPath, Error, Path, PathOwned, Timescale, Timestamp, bandwidth, coding::{Reader, Stream}, lite, - pressure::{Pressure, Rank}, + pressure::Rank, track::{Position, Subscription}, }; @@ -90,7 +90,7 @@ pub(super) struct Subscriber { /// Broadcasts the peer may have announced at once (`session::Limits::announces`). pub(super) announces: crate::session::Slots, /// Every stream this half reads, cancelled by rank when held receive credit fills. - pub(super) pressure: Pressure, + pub(super) pressure: crate::pressure::Of, } #[derive(Clone)] @@ -1315,8 +1315,6 @@ impl ProbeStream { struct AnnouncePrefix { subscriber: Subscriber, prefix: PathOwned, - /// Heard here because the link cost wait parks off the stream's reader. - eviction: crate::pressure::Eviction, state: PrefixState, } @@ -1360,7 +1358,6 @@ impl AnnouncePrefix { Self { subscriber, prefix, - eviction: crate::pressure::Eviction::none(), state: PrefixState::Open, } } @@ -1368,9 +1365,7 @@ impl AnnouncePrefix { /// Fails the session on any error, [`Error::TooManyRequests`] included: a peer /// announcing more than the session allows loses the session. fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - let res = self.poll_run(waiter); - self.eviction - .check(&waiter.context(), res) + self.poll_run(waiter) .map_err(|err| lite::error::control(&mut self.subscriber.session.clone(), err)) } @@ -1385,7 +1380,6 @@ impl AnnouncePrefix { &mut cx ))?; stream.reader.track(&self.subscriber.pressure, Rank::Control); - self.eviction = stream.reader.eviction(); stream.writer.buffer(&lite::ControlType::Announce)?; // Lite04/05: ask the peer to filter out announces that already passed @@ -4193,9 +4187,11 @@ mod tests { } /// A prefix stream waiting on the peer's SETUP for its link cost holds what it read - /// past ANNOUNCE_OK, so it still has to hear the pressure monitor choose it. + /// past ANNOUNCE_OK. The pressure monitor choosing it returns that credit and stops the + /// stream itself, without the prefix being polled. #[moq_net_sim::test] async fn an_evicted_prefix_waiting_on_its_link_cost_lets_go() { + use crate::coding::StreamCodes as _; const VERSION: Version = Version::Lite06; let mut script = Vec::new(); lite::AnnounceOk { @@ -4214,9 +4210,11 @@ mod tests { .unwrap(); let origin = origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + let session = crate::lite::test_transport::ScriptedSession::new(script); + let log = session.log.clone(); let subscriber = Subscriber::new(SubscriberConfig { runtime: crate::time::Clock::sim(), - session: crate::lite::test_transport::ScriptedSession::new(script), + session, origin, recv_bandwidth: None, version: VERSION, @@ -4230,20 +4228,18 @@ mod tests { let pressure = subscriber.pressure.clone(); let mut prefix = AnnouncePrefix::new(subscriber, Path::new("").to_owned()); - let res = { - let mut run = std::pin::pin!(kio::wait(|waiter| prefix.poll(waiter))); - let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), run.as_mut()).await; - assert!(waited.is_err(), "the prefix did not wait for its link cost: {waited:?}"); - assert!(pressure.held() > 0, "the parked prefix holds its read-ahead"); + let mut run = std::pin::pin!(kio::wait(|waiter| prefix.poll(waiter))); + let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), run.as_mut()).await; + assert!(waited.is_err(), "the prefix did not wait for its link cost: {waited:?}"); + assert!(pressure.held() > 0, "the parked prefix holds its read-ahead"); - assert!(pressure.evict()); - moq_net_sim::timeout(std::time::Duration::from_secs(1), run) - .await - .expect("the evicted prefix never woke") - }; - assert!(matches!(res, Err(Error::Evicted)), "{res:?}"); - drop(prefix); - assert_eq!(pressure.held(), 0, "the evicted prefix let go of its credit"); + assert!(pressure.evict()); + assert_eq!(pressure.held(), 0, "the eviction let go of the parked prefix's credit"); + assert_eq!( + log.stops(), + vec![VERSION.encode_stream_code(&crate::StreamError::Evicted)], + "the eviction stopped the parked prefix" + ); } } diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index 838bb23d39a..6c339f00e11 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -270,6 +270,8 @@ pub struct PendingRecv; impl poll::RecvStream for PendingRecv { type Error = SinkError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { Poll::Pending @@ -318,6 +320,8 @@ pub struct DeadRecv(ResetError); impl poll::RecvStream for DeadRecv { type Error = ResetError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read(&mut self, _cx: &mut Context<'_>, _dst: &mut [u8]) -> Poll, Self::Error>> { Poll::Ready(Err(self.0)) @@ -705,9 +709,24 @@ pub enum Close { Reset, } +/// Records a [`ScriptedRecv`]'s stops from outside it, as its own `stop` does. +#[derive(Clone)] +pub struct ScriptedStopper(Log); + +impl crate::transport::Stopper for ScriptedStopper { + fn stop(&self, code: u32) { + self.0.stops.lock().unwrap().push(code); + } +} + impl poll::RecvStream for ScriptedRecv { type Error = SinkError; type Credit = crate::transport::Unheld; + type Stopper = ScriptedStopper; + + fn stopper(&self) -> ScriptedStopper { + ScriptedStopper(self.log.clone()) + } fn poll_read(&mut self, _cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let take = { diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index e9e623ba223..6d43998af74 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -1,11 +1,13 @@ //! Receive-side recovery for when held connection credit fills the allowance. //! -//! A [`Reader`](crate::coding::Reader) tracked here holds the connection credit for its -//! buffered bytes and its incomplete unit, so incomplete messages are bounded by the -//! transport's receive allowance. When the transport reports that the allowance is -//! spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), -//! nothing can arrive to complete any unit, so the [`Monitor`] cancels the least useful -//! holder with STOP_SENDING and the session continues with the credit it returns. +//! A [`Reader`](crate::coding::Reader) tracked here keeps the connection credit for its +//! buffered bytes and its incomplete unit in a [`Hold`], so incomplete messages are +//! bounded by the transport's receive allowance. When the transport reports that the +//! allowance is spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), +//! nothing can arrive to complete any unit, so the [`Monitor`] evicts the least useful +//! holder: the eviction itself drops the credit and stops the stream with STOP_SENDING, +//! whatever the stream's handler is waiting on, and the handler's next read fails with +//! [`Error::Evicted`](crate::Error::Evicted). //! //! Victims, first to last: //! - Media, from the broadcast holding the most credit, lowest priority first. Within a @@ -23,15 +25,17 @@ use std::{ collections::HashMap, sync::{ - Arc, Mutex, + Arc, Mutex, MutexGuard, PoisonError, Weak, atomic::{AtomicBool, AtomicU64, Ordering}, }, - task::{Context, Poll, Waker}, + task::{Context, Waker}, }; -use slab::Slab; - -use crate::{PathOwned, priority::Priority}; +use crate::{ + PathOwned, + priority::Priority, + transport::{Credit, MaybeSend, MaybeSync, Stopper, poll::RecvStream}, +}; /// How useful a tracked stream is, which orders the victims. #[derive(Clone, Debug)] @@ -50,86 +54,255 @@ pub(crate) enum Rank { Control, } -/// One session's tracked receivers. -#[derive(Clone, Default)] -pub(crate) struct Pressure { - state: kio::Lock, -} - -#[derive(Default)] -struct State { - entries: Slab, - /// Registration order, the last tiebreak: the newest stream goes first. - next: u64, -} - -struct Entry { - slot: Arc, - rank: Rank, - seq: u64, +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) } -/// What a [`Ticket`] shares with the registry without taking its lock. -#[derive(Default)] -struct Slot { - /// Bytes of credit the reader holds. +/// A tracked stream's credit, and what evicting it takes, shared by its reader and the +/// registry. +pub(crate) struct Hold { + /// Bytes of credit `credit` covers, read by the registry without the lock. held: AtomicU64, - /// The monitor chose this stream; its reader stops and lets go. + /// The monitor chose this stream: its credit is gone and the stream is stopped. evicted: AtomicBool, /// The reader's task, woken on eviction. waker: Mutex>, + rank: Mutex, + /// Registration order, the last tiebreak: the newest stream goes first. + seq: u64, + credit: Mutex>, + stopper: R::Stopper, + /// The STOP_SENDING code for an eviction, in the stream's negotiated registry. + code: u32, } -impl Slot { - fn poll_evicted(&self, cx: &Context<'_>) -> bool { +impl Hold { + /// Keep the credit for bytes just read, or return it at once if already evicted. + pub fn take(&self, credit: R::Credit) { + let mut held = lock(&self.credit); + // Checked under the lock: an eviction sets the flag before it takes the credit. if self.evicted.load(Ordering::Acquire) { + drop(held); + return drop(credit); + } + let len = match held.as_mut() { + Some(held) => { + held.merge(credit); + held.len() + } + None => { + let len = credit.len(); + *held = Some(credit); + len + } + }; + self.held.store(len, Ordering::Relaxed); + } + + /// Return the credit for `bytes` that completed a unit. + pub fn release(&self, bytes: u64) { + let mut held = lock(&self.credit); + let Some(credit) = held.as_mut() else { return }; + let freed = credit.split(bytes); + self.held.store(credit.len(), Ordering::Relaxed); + drop(held); + drop(freed); + } + + /// Return everything held: the reader gave up on the stream. + pub fn clear(&self) { + let credit = lock(&self.credit).take(); + self.held.store(0, Ordering::Relaxed); + drop(credit); + } + + /// Re-rank the stream, once its header identifies it. + pub fn rank(&self, rank: Rank) { + *lock(&self.rank) = rank; + } + + /// Whether the monitor chose this stream, without registering to hear when it does. + pub fn is_evicted(&self) -> bool { + self.evicted.load(Ordering::Acquire) + } + + /// Whether the monitor chose this stream, registering `cx` to hear when it does. + pub fn poll_evicted(&self, cx: &Context<'_>) -> bool { + if self.is_evicted() { return true; } - let mut waker = self.waker.lock().unwrap_or_else(|err| err.into_inner()); + let mut waker = lock(&self.waker); match waker.as_mut() { Some(waker) => waker.clone_from(cx.waker()), None => *waker = Some(cx.waker().clone()), } drop(waker); // Checked again: an eviction between the load and the store took no waker. - self.evicted.load(Ordering::Acquire) + self.is_evicted() } } -impl Pressure { - /// Track a new receiver at `rank`. - pub fn track(&self, rank: Rank) -> Ticket { - let slot = Arc::new(Slot::default()); +/// What the registry needs from a [`Hold`], whatever its stream type. +pub(crate) trait Holder { + /// Bytes of credit held. + fn held(&self) -> u64; + /// Whether it was evicted already. + fn is_evicted(&self) -> bool; + /// How useful the stream is. + fn rank(&self) -> Rank; + /// Registration order. + fn seq(&self) -> u64; + /// Drop the credit, stop the stream, and wake its reader to fail. + fn evict(&self); +} + +impl Holder for Hold { + fn held(&self) -> u64 { + self.held.load(Ordering::Relaxed) + } + + fn is_evicted(&self) -> bool { + Hold::is_evicted(self) + } + + fn rank(&self) -> Rank { + lock(&self.rank).clone() + } + + fn seq(&self) -> u64 { + self.seq + } + + fn evict(&self) { + self.evicted.store(true, Ordering::Release); + let credit = lock(&self.credit).take(); + let held = self.held.swap(0, Ordering::Relaxed); + tracing::debug!(rank = ?Holder::rank(self), held, "receive credit exhausted, cancelling a stream"); + // Stopped first, so the window the credit frees does not invite more of this stream. + self.stopper.stop(self.code); + drop(credit); + let waker = lock(&self.waker).take(); + if let Some(waker) = waker { + waker.wake(); + } + } +} + +/// The registry entry of a session whose tracked readers differ in stream type: the +/// moq-transport path, which reads both raw streams and its control adapter's. +#[cfg(not(target_family = "wasm"))] +pub(crate) type Erased = dyn Holder + Send + Sync; +#[cfg(target_family = "wasm")] +pub(crate) type Erased = dyn Holder; + +/// The registry of a session whose tracked readers all read its own streams: moq-lite's. +/// +/// An alias rather than the spelled-out type, so a `derive` on a struct holding it bounds +/// only `S`, not its stream type. +pub(crate) type Of = Pressure::RecvStream>>; + +/// Turns a reader's [`Hold`] into the registry's entry type. +pub(crate) trait Erase { + fn erase(self: Arc) -> Arc; +} + +impl Erase> for Hold { + fn erase(self: Arc) -> Arc { + self + } +} + +impl Erase for Hold +where + Self: MaybeSend + MaybeSync, +{ + fn erase(self: Arc) -> Arc { + self + } +} + +/// One session's tracked receivers, each a [`Hold`] seen as `H`. +pub(crate) struct Pressure { + state: kio::Lock>, +} + +impl Clone for Pressure { + fn clone(&self) -> Self { + Self { + state: self.state.clone(), + } + } +} + +impl Default for Pressure { + fn default() -> Self { + Self { + state: kio::Lock::new(State { + entries: Vec::new(), + next: 0, + prune: PRUNE, + }), + } + } +} + +/// Entries kept before the first sweep of dropped readers. +const PRUNE: usize = 64; + +struct State { + /// Weak, so a reader unregisters by being dropped, and swept on growth. + entries: Vec>, + next: u64, + /// Sweep dropped readers once `entries` reaches this. + prune: usize, +} + +/// A live holder, as of the choice. +struct Candidate { + hold: Arc, + rank: Rank, + held: u64, + seq: u64, +} + +impl Pressure { + /// Track a new receiver at `rank`, which `stopper` stops with `code` on eviction. + pub fn track(&self, rank: Rank, stopper: R::Stopper, code: u32) -> Arc> + where + Hold: Erase, + { let mut state = self.state.lock(); - let seq = state.next; - state.next += 1; - let id = state.entries.insert(Entry { - slot: slot.clone(), - rank, - seq, + let hold = Arc::new(Hold { + held: AtomicU64::new(0), + evicted: AtomicBool::new(false), + waker: Mutex::new(None), + rank: Mutex::new(rank), + seq: state.next, + credit: Mutex::new(None), + stopper, + code, }); - Ticket { - pressure: self.clone(), - id, - slot, + state.next += 1; + if state.entries.len() >= state.prune { + state.entries.retain(|entry| entry.strong_count() > 0); + state.prune = PRUNE.max(2 * state.entries.len()); } + state.entries.push(Arc::downgrade(&hold.clone().erase())); + hold } - /// Cancel the least useful holder of credit, returning whether there was one. + /// Evict the least useful holder of credit, returning whether there was one. pub fn evict(&self) -> bool { - let state = self.state.lock(); - let Some(id) = victim(&state.entries) else { - return false; - }; - let slot = &state.entries[id].slot; - slot.evicted.store(true, Ordering::Release); - let waker = slot.waker.lock().unwrap_or_else(|err| err.into_inner()).take(); - tracing::debug!(rank = ?state.entries[id].rank, held = slot.held.load(Ordering::Relaxed), "receive credit exhausted, cancelling a stream"); - drop(state); - if let Some(waker) = waker { - waker.wake(); + let victim = victim(&self.state.lock().entries); + // Evicted without the registry lock: dropping credit and stopping call into the + // transport, and readers track and drop meanwhile. + match victim { + Some(hold) => { + hold.evict(); + true + } + None => false, } - true } /// The bytes every tracked receiver holds, for tests. @@ -139,31 +312,39 @@ impl Pressure { state .entries .iter() - .map(|(_, e)| e.slot.held.load(Ordering::Relaxed)) + .filter_map(Weak::upgrade) + .map(|hold| hold.held()) .sum() } } /// Pick the victim: see the module docs for the order. -fn victim(entries: &Slab) -> Option { - let held = |entry: &Entry| entry.slot.held.load(Ordering::Relaxed); - let holding = || { - entries - .iter() - .filter(|(_, entry)| held(entry) > 0 && !entry.slot.evicted.load(Ordering::Relaxed)) - }; +fn victim(entries: &[Weak]) -> Option> { + let holding: Vec<_> = entries + .iter() + .filter_map(Weak::upgrade) + .filter_map(|hold| { + let held = hold.held(); + (held > 0 && !hold.is_evicted()).then(|| Candidate { + rank: hold.rank(), + seq: hold.seq(), + held, + hold, + }) + }) + .collect(); // Media: the broadcast holding the most, then its least useful stream. Keyed by path // rather than by `Arc`, since separate subscriptions to one broadcast share no `Arc`. // A run of streams sharing one `Arc` sums before it is hashed. let mut domains: HashMap<&PathOwned, u64> = HashMap::new(); let mut run: Option<(&Arc, u64)> = None; - for (_, entry) in holding() { + for entry in &holding { if let Rank::Media { broadcast, .. } = &entry.rank { match &mut run { - Some((path, total)) if Arc::ptr_eq(path, broadcast) => *total += held(entry), + Some((path, total)) if Arc::ptr_eq(path, broadcast) => *total += entry.held, _ => { - if let Some((path, total)) = run.replace((broadcast, held(entry))) { + if let Some((path, total)) = run.replace((broadcast, entry.held)) { *domains.entry(path.as_ref()).or_default() += total; } } @@ -177,118 +358,48 @@ fn victim(entries: &Slab) -> Option { let domain = domains .into_iter() .max_by(|(a, a_held), (b, b_held)| a_held.cmp(b_held).then_with(|| b.as_str().cmp(a.as_str()))); - if let Some((domain, _)) = domain { - return holding() - .filter_map(|(id, entry)| match &entry.rank { - Rank::Media { broadcast, priority } if broadcast.as_ref() == domain => Some((id, priority, entry)), + let chosen = match domain { + Some((domain, _)) => holding + .iter() + .filter_map(|entry| match &entry.rank { + Rank::Media { broadcast, priority } if broadcast.as_ref() == domain => Some((priority, entry)), _ => None, }) // `Priority` sorts the most urgent first, so the greatest is the least useful. - .max_by(|(_, a, a_entry), (_, b, b_entry)| { + .max_by(|(a, a_entry), (b, b_entry)| { a.cmp(b) - .then_with(|| held(a_entry).cmp(&held(b_entry))) + .then_with(|| a_entry.held.cmp(&b_entry.held)) .then_with(|| a_entry.seq.cmp(&b_entry.seq)) }) - .map(|(id, ..)| id); - } - - let largest = |control: bool| { - holding() - .filter(|(_, entry)| matches!(entry.rank, Rank::Control) == control) - .max_by(|(_, a), (_, b)| held(a).cmp(&held(b)).then_with(|| a.seq.cmp(&b.seq))) - .map(|(id, _)| id) - }; - largest(false).or_else(|| largest(true)) -} - -/// A tracked receiver's registration, removed on drop. -pub(crate) struct Ticket { - pressure: Pressure, - id: usize, - slot: Arc, -} - -impl Ticket { - /// Re-rank the stream, once its header identifies it. - pub fn rank(&self, rank: Rank) { - self.pressure.state.lock().entries[self.id].rank = rank; - } - - /// Record how many bytes of credit the reader holds. - pub fn held(&self, bytes: u64) { - self.slot.held.store(bytes, Ordering::Relaxed); - } - - /// Whether the monitor chose this stream, without registering to hear when it does. - pub fn is_evicted(&self) -> bool { - self.slot.evicted.load(Ordering::Acquire) - } - - /// Whether the monitor chose this stream, registering `cx` to hear when it does. - pub fn poll_evicted(&self, cx: &Context<'_>) -> bool { - self.slot.poll_evicted(cx) - } - - /// Hear this stream's eviction apart from the reader holding the ticket. - pub fn eviction(&self) -> Eviction { - Eviction(Some(self.slot.clone())) - } -} - -/// Hears that the monitor chose a stream without borrowing its reader, so a handler that -/// lent the reader to a wait parked elsewhere still ends. Empty for an untracked reader. -pub(crate) struct Eviction(Option>); - -impl Eviction { - /// For an untracked reader, which is never evicted. - pub fn none() -> Self { - Self(None) - } - - /// Run `serve` to its end, or fail it with [`Error::Evicted`](crate::Error::Evicted) - /// once the monitor chooses the stream, whatever `serve` is parked on. - pub async fn guard(&self, serve: impl Future>) -> Result { - let mut serve = std::pin::pin!(serve); - std::future::poll_fn(|cx| { - let res = serve.as_mut().poll(cx); - self.check(cx, res) - }) - .await - } - - /// [`Self::guard`] for a state machine owning the stream: a pending `res` fails with - /// [`Error::Evicted`](crate::Error::Evicted) once the monitor chooses the stream. Call - /// it on the machine's result, after its reads, which register on the same slot, so the - /// eviction wakes the machine at the top rather than a wait that will not look. - pub fn check(&self, cx: &Context<'_>, res: Poll>) -> Poll> { - match (res, &self.0) { - (Poll::Pending, Some(slot)) if slot.poll_evicted(cx) => Poll::Ready(Err(crate::Error::Evicted)), - (res, _) => res, + .map(|(_, entry)| entry), + None => { + let largest = |control: bool| { + holding + .iter() + .filter(|entry| matches!(entry.rank, Rank::Control) == control) + .max_by(|a, b| a.held.cmp(&b.held).then_with(|| a.seq.cmp(&b.seq))) + }; + largest(false).or_else(|| largest(true)) } - } -} - -impl Drop for Ticket { - fn drop(&mut self) { - self.pressure.state.lock().entries.remove(self.id); - } + }; + chosen.map(|entry| entry.hold.clone()) } /// Cancels a stream each time the session's receive allowance fills. -pub(crate) struct Monitor { +pub(crate) struct Monitor { session: S, - pressure: Pressure, + pressure: Pressure, } -impl Monitor { +impl Monitor { /// Watch `session`, cancelling streams tracked by `pressure`. - pub fn new(session: S, pressure: Pressure) -> Self { + pub fn new(session: S, pressure: Pressure) -> Self { Self { session, pressure } } /// Run until dropped. Each fill cancels one stream: the next fill can only come after /// it let go of its credit and the peer spent what that freed. - pub fn poll(&mut self, cx: &mut Context<'_>) -> Poll<()> { + pub fn poll(&mut self, cx: &mut Context<'_>) -> std::task::Poll<()> { while self.session.poll_credit_exhausted(cx).is_ready() { if !self.pressure.evict() { // Untracked readers or transport read-ahead hold it all; whoever holds it @@ -296,14 +407,51 @@ impl Monitor { tracing::warn!("receive credit exhausted with no tracked holder to cancel"); } } - Poll::Pending + std::task::Poll::Pending } } #[cfg(test)] mod tests { + use std::task::{Poll, Waker}; + use super::*; - use crate::{Path, track::Order}; + use crate::{Path, track::Order, transport::Unheld}; + + /// Logs the codes its streams were stopped with. + #[derive(Clone, Default)] + struct Stops(Arc>>); + + impl Stopper for Stops { + fn stop(&self, code: u32) { + lock(&self.0).push(code); + } + } + + /// A stream that only exists to name its credit and stopper. + struct Probe; + + impl RecvStream for Probe { + type Error = crate::lite::test_transport::SinkError; + type Credit = Unheld; + type Stopper = Stops; + + fn poll_read(&mut self, _: &mut Context<'_>, _: &mut [u8]) -> Poll, Self::Error>> { + Poll::Pending + } + + fn stop(&mut self, _: u32) {} + + fn stopper(&self) -> Stops { + Stops::default() + } + + fn poll_closed(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + type Probes = Pressure>; fn media(broadcast: &str, track: u8, group: u64) -> Rank { Rank::Media { @@ -313,26 +461,26 @@ mod tests { } /// Track a receiver holding `held` bytes. - fn hold(pressure: &Pressure, rank: Rank, held: u64) -> Ticket { - let ticket = pressure.track(rank); - ticket.held(held); - ticket + fn hold(pressure: &Probes, rank: Rank, held: u64) -> Arc> { + let hold = pressure.track::(rank, Stops::default(), 0x35); + hold.take(Unheld(held)); + hold } - /// Evict once, returning every ticket chosen so far. - fn evict(pressure: &Pressure, tickets: &[&Ticket]) -> Vec { + /// Evict once, returning every hold chosen so far. + fn evict(pressure: &Probes, holds: &[&Arc>]) -> Vec { assert!(pressure.evict()); - tickets + holds .iter() .enumerate() - .filter(|(_, ticket)| ticket.is_evicted()) + .filter(|(_, hold)| hold.is_evicted()) .map(|(i, _)| i) .collect() } #[test] fn media_goes_before_unidentified_before_control() { - let pressure = Pressure::default(); + let pressure = Probes::default(); let control = hold(&pressure, Rank::Control, 100); let unidentified = hold(&pressure, Rank::Unidentified, 50); let media = hold(&pressure, media("room", 255, 0), 10); @@ -347,7 +495,7 @@ mod tests { #[test] fn the_least_urgent_stream_of_the_fullest_broadcast_goes() { - let pressure = Pressure::default(); + let pressure = Probes::default(); // `small` holds less in total, so its less urgent track is not compared. let small = hold(&pressure, media("small", 0, 0), 30); let urgent = hold(&pressure, media("big", 200, 0), 20); @@ -361,7 +509,7 @@ mod tests { #[test] fn group_order_ranks_within_a_subscription() { - let pressure = Pressure::default(); + let pressure = Probes::default(); let old = hold(&pressure, media("room", 1, 1), 10); let new = hold(&pressure, media("room", 1, 2), 10); // Newest first by default, so the older group is the less useful one. @@ -371,7 +519,7 @@ mod tests { broadcast: Arc::new(Path::new("room").to_owned()), priority: Priority::new(1, 0, group).with_order(Order::Ascending), }; - let pressure = Pressure::default(); + let pressure = Probes::default(); let old = hold(&pressure, ascending(1), 10); let new = hold(&pressure, ascending(2), 10); assert_eq!(evict(&pressure, &[&old, &new]), [1]); @@ -379,7 +527,7 @@ mod tests { #[test] fn ties_go_to_the_larger_then_the_newer() { - let pressure = Pressure::default(); + let pressure = Probes::default(); let first = hold(&pressure, Rank::Unidentified, 10); let larger = hold(&pressure, Rank::Unidentified, 20); let newest = hold(&pressure, Rank::Unidentified, 10); @@ -390,24 +538,39 @@ mod tests { #[test] fn empty_and_dropped_receivers_are_never_chosen() { - let pressure = Pressure::default(); + let pressure = Probes::default(); let idle = hold(&pressure, media("room", 0, 0), 0); drop(hold(&pressure, media("room", 0, 0), 10)); assert!(!pressure.evict()); assert!(!idle.is_evicted()); } + /// Dropped readers are swept as the registry grows, so it stays the size of what is live. + #[test] + fn dropped_receivers_are_swept() { + let pressure = Probes::default(); + let live = hold(&pressure, Rank::Control, 1); + for _ in 0..10 * PRUNE { + drop(hold(&pressure, Rank::Control, 1)); + } + assert!(pressure.state.lock().entries.len() <= PRUNE + 1); + assert_eq!(pressure.held(), 1); + drop(live); + } + #[test] fn rerank_once_identified() { - let pressure = Pressure::default(); + let pressure = Probes::default(); let header = hold(&pressure, Rank::Unidentified, 10); let control = hold(&pressure, Rank::Control, 10); header.rank(media("room", 0, 0)); assert_eq!(evict(&pressure, &[&header, &control]), [0]); } + /// The eviction itself returns the credit and stops the stream, so a reader parked + /// on something else holds nothing; it hears of it on its next poll. #[test] - fn eviction_wakes_the_reader() { + fn eviction_lets_go_without_the_reader() { #[derive(Default)] struct Count(std::sync::atomic::AtomicUsize); impl std::task::Wake for Count { @@ -416,14 +579,23 @@ mod tests { } } - let pressure = Pressure::default(); - let ticket = hold(&pressure, Rank::Unidentified, 10); + let pressure = Probes::default(); + let stops = Stops::default(); + let hold = pressure.track::(Rank::Unidentified, stops.clone(), 0x35); + hold.take(Unheld(10)); let count = Arc::new(Count::default()); let waker = Waker::from(count.clone()); - assert!(!ticket.poll_evicted(&Context::from_waker(&waker))); + assert!(!hold.poll_evicted(&Context::from_waker(&waker))); + assert!(pressure.evict()); + assert_eq!(pressure.held(), 0); + assert_eq!(*lock(&stops.0), [0x35]); assert_eq!(count.0.load(Ordering::Relaxed), 1); - assert!(ticket.poll_evicted(&Context::from_waker(&waker))); + assert!(hold.poll_evicted(&Context::from_waker(&waker))); + + // A read that raced the eviction returns its credit at once. + hold.take(Unheld(5)); + assert_eq!(pressure.held(), 0); } /// Choose a victim among streams spread across broadcasts. The cost per stream must stay @@ -440,8 +612,8 @@ mod tests { let paths: Vec<_> = (0..broadcasts) .map(|b| Arc::new(Path::new(&format!("broadcast/{b}")).to_owned())) .collect(); - let pressure = Pressure::default(); - let _tickets: Vec<_> = (0..streams) + let pressure = Probes::default(); + let _holds: Vec<_> = (0..streams) .map(|i| { let rank = Rank::Media { broadcast: paths[i % broadcasts].clone(), diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index 6b1ba197fa2..d2d235f4f4f 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -1139,6 +1139,8 @@ mod tests { impl crate::transport::poll::RecvStream for FakeRecv { type Error = FakeError; type Credit = crate::transport::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read( &mut self, _cx: &mut std::task::Context<'_>, diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 47b4fd8358c..2c64e220bfe 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -154,6 +154,20 @@ impl Credit for Unheld { } } +/// Stops a [`poll::RecvStream`] without borrowing it, as [`stop`](poll::RecvStream::stop) +/// would, for when the stream's reader is parked somewhere else. +/// +/// A no-op once the stream has ended. A transport whose reads return their credit (an +/// [`Unheld`] credit) never needs a stream stopped this way, so it may use `()`. +pub trait Stopper: 'static { + /// Send STOP_SENDING with a stream `code`, unless the stream already ended. + fn stop(&self, code: u32); +} + +impl Stopper for () { + fn stop(&self, _code: u32) {} +} + /// A transport failure with optional session and stream application codes. /// /// Session codes and stream codes belong to separate registries. @@ -375,7 +389,7 @@ pub mod poll { pub trait Boxable: Session< SendStream: MaybeSend + SendStream, - RecvStream: MaybeSend + RecvStream, + RecvStream: MaybeSend + RecvStream, Error: MaybeSend, > + MaybeSend + MaybeSync @@ -385,7 +399,7 @@ pub mod poll { impl Boxable for S where S: Session< SendStream: MaybeSend + SendStream, - RecvStream: MaybeSend + RecvStream, + RecvStream: MaybeSend + RecvStream, Error: MaybeSend, > + MaybeSend + MaybeSync @@ -576,6 +590,9 @@ pub mod poll { /// The connection credit each read returns. type Credit: Credit; + /// What [`stopper`](Self::stopper) returns. + type Stopper: super::Stopper; + /// Poll to read some data into the provided slice. /// /// Returns the credit for the bytes read ([`Credit::len`] counts them), or @@ -666,6 +683,9 @@ pub mod poll { /// Call this method manually if you want to specify a code yourself. fn stop(&mut self, code: u32); + /// A handle that stops this stream as [`stop`](Self::stop) does, without borrowing it. + fn stopper(&self) -> Self::Stopper; + /// Poll until the stream has been closed by either side. /// /// This includes: @@ -750,6 +770,8 @@ mod tests { impl super::poll::RecvStream for Read { type Error = crate::Error; type Credit = Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read(&mut self, _: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { // A safe transport may read its destination before overwriting it. assert!(dst.iter().all(|byte| *byte == 0)); diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index 05df838b633..61681321ed0 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -677,9 +677,29 @@ pub struct RecvStream { park: kio::Park, } +/// Stops a [`RecvStream`] without borrowing it. +#[derive(Clone)] +pub struct Stopper { + channel: kio::Shared, + window: kio::Shared, + close: Close, +} + +impl super::Stopper for Stopper { + fn stop(&self, code: u32) { + // A closed session leaves its streams as they were; checked under the channel lock, + // as in `reset`. + let mut channel = self.channel.lock(); + if closed(&self.close).is_none() { + channel.abort(End::Stop(code), &self.window); + } + } +} + impl poll::RecvStream for RecvStream { type Error = Error; type Credit = Credit; + type Stopper = Stopper; fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, dst.len()))? else { @@ -746,14 +766,16 @@ impl poll::RecvStream for RecvStream { } fn stop(&mut self, code: u32) { - if std::mem::replace(&mut self.stopped, true) { - return; + if !std::mem::replace(&mut self.stopped, true) { + super::Stopper::stop(&self.stopper(), code); } - // A closed session leaves its streams as they were; checked under the channel lock, - // as in `reset`. - let mut channel = self.channel.lock(); - if closed(&self.close).is_none() { - channel.abort(End::Stop(code), &self.window); + } + + fn stopper(&self) -> Stopper { + Stopper { + channel: self.channel.clone(), + window: self.window.clone(), + close: self.close.clone(), } } diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 621ad704853..762641287e1 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -289,6 +289,8 @@ type Event = Result, memory::Error>; pub struct MockRecvStream { /// Taken once this side stops reading. inner: Option, + /// What [`MockStopper`] stops, kept because `inner` goes once this side stops. + stopper: memory::Stopper, line: Arc, conn: Arc, /// The next event, read from the transport but still crossing the link. @@ -305,6 +307,7 @@ pub struct MockRecvStream { impl MockRecvStream { fn new(inner: memory::RecvStream, line: Arc, conn: Arc) -> Self { Self { + stopper: poll::RecvStream::stopper(&inner), inner: Some(inner), line, conn, @@ -381,9 +384,39 @@ impl MockRecvStream { } } +/// Stops a [`MockRecvStream`] without borrowing it, telling the writer once the latency +/// has passed. +#[derive(Clone)] +pub struct MockStopper { + inner: memory::Stopper, + conn: Arc, +} + +impl moq_net::transport::Stopper for MockStopper { + fn stop(&self, code: u32) { + let latency = self.conn.latency(); + if latency.is_zero() { + return moq_net::transport::Stopper::stop(&self.inner, code); + } + let inner = self.inner.clone(); + drop(moq_net_sim::spawn(async move { + moq_net_sim::sleep(latency).await; + moq_net::transport::Stopper::stop(&inner, code); + })); + } +} + impl poll::RecvStream for MockRecvStream { type Error = memory::Error; type Credit = memory::Credit; + type Stopper = MockStopper; + + fn stopper(&self) -> MockStopper { + MockStopper { + inner: self.stopper.clone(), + conn: self.conn.clone(), + } + } fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let Some((chunk, credit)) = ready!(self.poll_read_chunk(cx, dst.len()))? else { diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 1b10cdf7af8..c33f81c70cc 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -3378,6 +3378,8 @@ mod tests { let stats = client.credit_stats(); assert_eq!((stats.held, stats.unread), (0, 0)); + } + /// A group token indexes its own connection's schedule, so another connection refuses it /// rather than reach whichever of its own groups shares the index. #[test] diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index b248a9fe098..8d0db753f02 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -74,7 +74,7 @@ pub use crate::quic::driver::connection::{ pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; pub use crate::quic::driver::recv_stream::{ - Credit, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, + Credit, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, Stopper, }; #[cfg(test)] pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 6b970ea0c16..2d1ffae81da 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -294,6 +294,15 @@ impl RecvStream { Ok(()) } + /// A handle that stops this stream as [`stop`](Self::stop) does, without borrowing it. + pub fn stopper(&self) -> Stopper { + Stopper { + conn: self.conn.0.clone(), + stream: self.stream, + is_0rtt: self.is_0rtt, + } + } + /// Check if this stream predates completion of the handshake on an incoming connection. /// /// True only if the stream was accepted before the handshake completed, which is only possible @@ -626,6 +635,38 @@ impl tokio::io::AsyncRead for RecvStream { } } +/// Stops a [`RecvStream`] without borrowing it. +/// +/// It does not keep the connection open: only [`RecvStream`] and the other handles count. +#[derive(Clone)] +pub struct Stopper { + conn: Arc, + stream: StreamId, + is_0rtt: bool, +} + +impl Stopper { + /// Stop accepting data, as [`RecvStream::stop`] does. + /// + /// The stream's own handle learns of it on its next read, which yields `ClosedStream`. + pub fn stop(&self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("Stopper::stop"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.recv_stream(self.stream).stop(error_code)?; + conn.check_credit(); + conn.wake(); + Ok(()) + } +} + +impl fmt::Debug for Stopper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Stopper").field("stream", &self.stream).finish() + } +} + impl Drop for RecvStream { fn drop(&mut self) { if self.all_data_read { diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 5ad4f2501c8..27412b41646 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -16,6 +16,20 @@ pub struct RecvStream { close: Arc, } +/// Stops a [`RecvStream`] without borrowing it. +#[derive(Clone)] +pub struct Stopper { + inner: crate::quic::driver::Stopper, + close: Arc, +} + +impl Stopper { + /// Stop accepting data, as [`RecvStream::stop`] does. + pub fn stop(&self, code: u32) -> Result<(), crate::quic::driver::ClosedStream> { + self.inner.stop(self.close.encode_stream_code(code)) + } +} + impl RecvStream { pub(crate) fn new(stream: crate::quic::driver::RecvStream, close: Arc) -> Self { Self { inner: stream, close } @@ -43,6 +57,14 @@ impl RecvStream { self.inner.stop(code) } + /// A handle that stops this stream as [`stop`](Self::stop) does, without borrowing it. + pub fn stopper(&self) -> Stopper { + Stopper { + inner: self.inner.stopper(), + close: self.close.clone(), + } + } + // Unfortunately, we have to wrap ReadError for a bunch of functions. /// Read some data into the buffer, returning the credit for the bytes read. See diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index bf3a40a5f93..d76a741d15e 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -17,8 +17,8 @@ use futures::FutureExt; use moq_net::transport::Credit as _; use web_transport_trait::poll as wt_poll; -pub(crate) mod held; mod grouped; +pub(crate) mod held; mod owned; pub(crate) use grouped::{Grouped, GroupedSend}; pub use owned::Error; @@ -31,11 +31,17 @@ pub trait HeldRecvStream: web_transport_trait::RecvStream { /// The connection credit a read keeps until it is dropped. type Credit: moq_net::transport::Credit + Send + Sync; + /// What [`stopper`](Self::stopper) returns. + type Stopper: moq_net::transport::Stopper + Clone + Send + Sync; + /// Read the next chunk, up to `max` bytes, with its credit, or `None` once finished. fn read_chunk_held( &mut self, max: usize, ) -> impl Future, Self::Error>> + Send; + + /// A handle that stops this stream without borrowing it. + fn stopper(&self) -> Self::Stopper; } /// A backend session that reports when held credit fills its receive window. @@ -689,6 +695,8 @@ pub struct RecvStream { /// waiting behind blocked I/O. Fired by [`stop`](wt_poll::RecvStream::stop) /// and by [`Drop`]. interrupt: Option>, + /// Taken up front, since an in-flight read owns the stream. + stopper: S::Stopper, } /// How much the closed-watch may read ahead of the caller before it parks and @@ -710,6 +718,7 @@ impl RecvStream { /// Wrap an async-interface receive stream. pub fn new(stream: S) -> Self { Self { + stopper: stream.stopper(), state: std::sync::Mutex::new(Some(RecvState::Idle(stream))), buffer: None, queued: std::collections::VecDeque::new(), diff --git a/rs/moq-tokio/src/transport/held.rs b/rs/moq-tokio/src/transport/held.rs index 63c543ffcf8..05f17cafa6c 100644 --- a/rs/moq-tokio/src/transport/held.rs +++ b/rs/moq-tokio/src/transport/held.rs @@ -20,6 +20,9 @@ macro_rules! unheld { ($session:ty, $recv:ty) => { impl $crate::transport::HeldRecvStream for $recv { type Credit = moq_net::transport::Unheld; + type Stopper = (); + + fn stopper(&self) {} fn read_chunk_held( &mut self, @@ -59,9 +62,22 @@ impl moq_net::transport::Credit for crate::quic::driver::Credit { } } +#[cfg(feature = "quic")] +impl moq_net::transport::Stopper for crate::quic::web::Stopper { + fn stop(&self, code: u32) { + // Err means the stream is already gone, which is what we wanted. + let _ = crate::quic::web::Stopper::stop(self, code); + } +} + #[cfg(feature = "quic")] impl super::HeldRecvStream for crate::quic::web::RecvStream { type Credit = crate::quic::driver::Credit; + type Stopper = crate::quic::web::Stopper; + + fn stopper(&self) -> Self::Stopper { + crate::quic::web::RecvStream::stopper(self) + } fn read_chunk_held( &mut self, diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index cfddb863cbc..3f73fd4112d 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -121,6 +121,10 @@ impl poll::SendStream for super::SendStream impl poll::RecvStream for super::RecvStream { type Error = Error; type Credit = super::Credit; + type Stopper = S::Stopper; + fn stopper(&self) -> S::Stopper { + self.stopper.clone() + } fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let read = ready!(self.poll_read_held(cx, dst.len())).map_err(Error)?; Poll::Ready(Ok(read.map(|(bytes, credit)| { diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index 8ccc9a0f452..96007e26915 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -17,7 +17,7 @@ mod stream; pub use connection::Connection; pub use endpoint::Endpoint; pub use group::SendGroup; -pub use stream::{Credit, RecvStream, SendStream}; +pub use stream::{Credit, RecvStream, SendStream, Stopper}; pub(crate) use connection::{End, Shared}; diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index d575d2ac695..70986e9b29e 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -335,6 +335,14 @@ impl RecvStream { self.finished || self.stopped } + /// A handle that stops this stream without borrowing it. + pub fn stopper(&self) -> Stopper { + Stopper { + shared: self.shared.clone(), + id: self.id, + } + } + /// [`stop`](web_transport_trait::poll::RecvStream::stop) with a /// full-width code, for the WebTransport HTTP/3 error mapping. pub(crate) fn stop_code(&mut self, code: u64) { @@ -459,6 +467,31 @@ impl web_transport_trait::poll::RecvStream for RecvStream { } } +/// Stops a [`RecvStream`] without borrowing it. +/// +/// The stream's own handle learns of it on its next read, which fails as stopped. What its +/// closed-watch read ahead stays with that handle until it stops or drops. +#[derive(Clone)] +pub struct Stopper { + shared: Shared, + id: StreamId, +} + +impl Stopper { + /// [`RecvStream::stop_code`] from outside the stream. + pub(crate) fn stop_code(&self, code: u64) { + // Err means the stream is already gone, which is what we wanted. + let _ = self + .shared + .conn + .borrow_mut() + .recv_stream(self.id) + .stop(VarInt::from_u64(code).unwrap_or(VarInt::MAX)); + self.shared.check_credit(); + self.shared.kick(); + } +} + impl Drop for RecvStream { fn drop(&mut self) { release(&self.shared, std::mem::take(&mut self.backlog_credit)); diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index a9d6930ac6d..c9537b11e0b 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -30,7 +30,7 @@ pub mod web; mod driver; -pub use driver::{Connection, Credit, RecvStream, SendGroup, SendStream}; +pub use driver::{Connection, Credit, RecvStream, SendGroup, SendStream, Stopper}; pub use endpoint::Endpoint; /// The QUIC payload size every full datagram in a GSO train uses, and the diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index ea5d6e17e65..35d2fc7ca8a 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -935,7 +935,32 @@ pub struct RecvStream { web: bool, } +/// Stops a [`RecvStream`] without borrowing it, with the same code mapping as its `stop`. +#[derive(Clone)] +pub struct Stopper { + inner: super::Stopper, + web: bool, +} + +impl Stopper { + /// Stop the stream with an application `code`. + pub fn stop(&self, code: u32) { + match self.web { + true => self.inner.stop_code(proto::error_to_http3(code)), + false => self.inner.stop_code(u64::from(code)), + } + } +} + impl RecvStream { + /// A handle that stops this stream without borrowing it. + pub fn stopper(&self) -> Stopper { + Stopper { + inner: self.inner.stopper(), + web: self.web, + } + } + fn map(&self, err: Error) -> Error { match self.web { true => unmap_err(err), diff --git a/rs/moq-uring/src/transport/adapter.rs b/rs/moq-uring/src/transport/adapter.rs index a6e07f3494d..5484568dc8f 100644 --- a/rs/moq-uring/src/transport/adapter.rs +++ b/rs/moq-uring/src/transport/adapter.rs @@ -49,6 +49,12 @@ pub trait HeldRecvStream: backend::RecvStream { /// The connection credit a read keeps until it is dropped. type Credit: moq::Credit; + /// What [`stopper`](Self::stopper) returns. + type Stopper: moq::Stopper; + + /// A handle that stops this stream without borrowing it. + fn stopper(&self) -> Self::Stopper; + /// Read up to `max` bytes with their credit, or `None` once finished. #[allow(clippy::type_complexity)] fn poll_read_held( @@ -62,6 +68,8 @@ pub trait HeldRecvStream: backend::RecvStream { pub trait HeldSession: backend::Session { /// See [`poll_credit_exhausted`](poll::Session::poll_credit_exhausted). fn poll_credit_exhausted(&mut self, cx: &mut Context<'_>) -> Poll<()>; +} + /// A backend session whose send streams may join send groups, as native QUIC's do. /// /// `pub` only because `pub(crate)` is refused (E0446): the adapter's `type Group` projects @@ -193,6 +201,10 @@ pub struct RecvStream(S); impl poll::RecvStream for RecvStream { type Error = Error; type Credit = S::Credit; + type Stopper = S::Stopper; + fn stopper(&self) -> S::Stopper { + self.0.stopper() + } fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { let read = ready!(self.0.poll_read_held(cx, dst.len())).map_err(Error)?; Poll::Ready(Ok(read.map(|(bytes, credit)| { @@ -283,6 +295,8 @@ mod tests { } impl HeldRecvStream for Stream { type Credit = moq::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read_held( &mut self, cx: &mut Context<'_>, diff --git a/rs/moq-uring/src/transport/held.rs b/rs/moq-uring/src/transport/held.rs index 7a9fb860331..2de4bcd52f4 100644 --- a/rs/moq-uring/src/transport/held.rs +++ b/rs/moq-uring/src/transport/held.rs @@ -20,8 +20,19 @@ impl moq_net::transport::Credit for driver::Credit { } } +impl moq_net::transport::Stopper for web::Stopper { + fn stop(&self, code: u32) { + web::Stopper::stop(self, code) + } +} + impl super::HeldRecvStream for web::RecvStream { type Credit = driver::Credit; + type Stopper = web::Stopper; + + fn stopper(&self) -> web::Stopper { + web::RecvStream::stopper(self) + } fn poll_read_held( &mut self, diff --git a/rs/moq-wasm/src/transport/adapter.rs b/rs/moq-wasm/src/transport/adapter.rs index 61abe300f86..0ceb9e08da9 100644 --- a/rs/moq-wasm/src/transport/adapter.rs +++ b/rs/moq-wasm/src/transport/adapter.rs @@ -142,6 +142,8 @@ pub struct RecvStream(S); impl poll::RecvStream for RecvStream { type Error = Error; type Credit = moq::Unheld; + type Stopper = (); + fn stopper(&self) {} fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { backend::RecvStream::poll_read(&mut self.0, cx, dst) .map_err(Error) diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 84c4f3a7c74..42e3d975c34 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -26,7 +26,7 @@ case "${1:-}" in run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate run just rs test -p moq-tokio --release -E 'test(benchmark_held_reads)' --run-ignored only --success-output immediate - run just rs test -p moq-net --release -E 'test(benchmark_victim)' --run-ignored only --success-output immediate + run just rs test -p moq-net --release -E 'test(benchmark_victim) | test(benchmark_tracked_reads)' --run-ignored only --success-output immediate # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test # Exercise the optional thread-CPU clock as well as the normal elapsed-time smoke. From 4af7531e86776287dbcbdb840d036a0a7a98d154 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 21:15:53 -0700 Subject: [PATCH 42/51] style(moq-tokio): format the driver's stopper as quinn does Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/recv_stream.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 2d1ffae81da..50d5ec9696c 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -663,7 +663,9 @@ impl Stopper { impl fmt::Debug for Stopper { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Stopper").field("stream", &self.stream).finish() + f.debug_struct("Stopper") + .field("stream", &self.stream) + .finish() } } From 95af8017e209072879457e16f121c2e441872f17 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 21:46:55 -0700 Subject: [PATCH 43/51] docs(moq-tokio): bound iroh's incomplete messages by its stream limit only iroh's upstream quinn returns connection credit as it reads, so the receive window cannot hold an incomplete message's bytes there and nothing evicts. Say so rather than claim the protection. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 4 +++- rs/moq-tokio/src/quic.rs | 7 ++++++- rs/moq-tokio/src/transport/held.rs | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 07756b48722..aec15bf1f01 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -77,7 +77,9 @@ Raise `receive_window` when a fat, long path idles below the link rate. The data and incomplete messages one peer can make the relay buffer. A message holds its share until it is complete, so the window must fit the largest one, just over 64 MiB, and the relay refuses to start with less. When incomplete messages fill -it, the relay cancels the least urgent stream and carries on. +it, the relay cancels the least urgent stream and carries on. iroh returns credit +as it reads, so there the window bounds only unread data, and incomplete messages +are bounded by `max_streams` streams of up to the largest message each. Stream credit returns as data is read, so a `stream_receive_window` under `receive_window` only adds round trips to a large frame. diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 85257f6372c..223fc64cafb 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -108,7 +108,8 @@ pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Default connection-wide receive window, used when [`Config::receive_window`] is unset. /// /// Bounds how much unread data and incomplete messages one peer can make us buffer across -/// all of its streams, which the backends otherwise leave unlimited. Sized to fit the +/// all of its streams, which the backends otherwise leave unlimited. On iroh, which returns +/// credit as it reads, it bounds only the unread data. Sized to fit the /// largest message with room to spare, so a relay-to-relay session, which shares one /// connection, can still sustain several Gbps at intercontinental RTTs. #[cfg(any(feature = "quic", feature = "iroh"))] @@ -237,6 +238,10 @@ pub struct Config { /// in flight or in incomplete messages across every stream at once. Defaults to /// 128 MiB, and must fit the largest message a session assembles, just over 64 MiB. /// + /// iroh returns credit as it reads, so there this bounds only unread data, and + /// incomplete messages are bounded by [`max_streams`](Self::max_streams) streams + /// of up to the largest message each. + /// /// Raise it when a fat, long path is idling below the link rate: the window /// has to cover the bandwidth-delay product or the peer stalls waiting for /// credit. diff --git a/rs/moq-tokio/src/transport/held.rs b/rs/moq-tokio/src/transport/held.rs index 05f17cafa6c..e3085dd408c 100644 --- a/rs/moq-tokio/src/transport/held.rs +++ b/rs/moq-tokio/src/transport/held.rs @@ -44,6 +44,8 @@ pub(crate) use unheld; #[cfg(any(feature = "tcp", feature = "websocket"))] unheld!(qmux::Session, qmux::RecvStream); +// iroh's upstream quinn has no way to hold credit, so its incomplete messages are bounded +// only by its stream limit, each up to the largest message. #[cfg(feature = "iroh")] unheld!(web_transport_iroh::Session, web_transport_iroh::RecvStream); From 2f5be7e0c96eca76516bfc7591f96c900b71be1e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 00:48:45 -0700 Subject: [PATCH 44/51] fix(moq-net): free a parked reader's buffer with its evicted credit Eviction returned a parked reader's credit but left its read-ahead allocated, so a peer could refill the window and repeat across retained readers until memory passed receive_window. A tracked reader's buffer now lives beside its credit in the Hold, under the lock each read already takes, and evict drops both. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 244 +++++++++++++++++++++++--------- rs/moq-net/src/pressure.rs | 121 +++++++++++----- 2 files changed, 261 insertions(+), 104 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index d6daba37d2a..59a00aad739 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -1,6 +1,7 @@ use std::{ cmp, fmt::Debug, + ops::{Deref, DerefMut}, sync::Arc, task::{Context, Poll, ready}, }; @@ -10,7 +11,7 @@ use bytes::{Buf, Bytes, BytesMut}; use crate::{ Error, StreamError, coding::*, - pressure::{Erase, Hold, Holder, Pressure, Rank}, + pressure::{Erase, Hold, Holder, Locked, Pressure, Rank}, }; /// A reader for decoding messages from a stream. @@ -22,10 +23,13 @@ use crate::{ /// /// A [tracked](Self::track) reader holds the transport's connection credit for its /// buffered bytes and for the unit it is assembling, releasing each unit's bytes once -/// it is decoded and handed off, so read-ahead stays with the next unit. An untracked -/// reader returns credit as soon as it reads. +/// it is decoded and handed off, so read-ahead stays with the next unit. Its buffer sits +/// beside that credit in its [`Hold`], so an eviction frees the memory with the credit +/// even while the reader is parked. An untracked reader returns credit as soon as it +/// reads. pub struct Reader { stream: S, + /// The buffer until tracked, then empty: tracking moves it into the hold. buffer: BytesMut, version: V, /// Bytes of the incomplete frame already taken out of `buffer`. @@ -72,7 +76,9 @@ impl Reader { } self.free = self.buffer.len() as u64 + self.unit; let code = self.version.encode_stream_code(&StreamError::Evicted); - self.hold = Some(pressure.track(rank, self.stream.stopper(), code)); + let hold = pressure.track(rank, self.stream.stopper(), code); + *hold.lock().buffer_mut() = std::mem::take(&mut self.buffer); + self.hold = Some(hold); } /// Re-rank a tracked stream once its header identifies it. @@ -82,44 +88,22 @@ impl Reader { } } - /// Keep the credit for bytes just read, if tracked; otherwise it returns now. - fn take(&mut self, credit: S::Credit) { - if let Some(hold) = &self.hold { - hold.take(credit); - } - } - /// Return the credit for `bytes` that completed a unit. fn release(&mut self, bytes: u64) { - let free = self.free.min(bytes); - self.free -= free; - if let Some(hold) = &self.hold { - hold.release(bytes - free); - } + Buffer::of(&mut self.buffer, &self.hold).release(&mut self.free, bytes); } - /// Drop `used` decoded bytes from the buffer, returning their credit. - fn consume(&mut self, used: usize) { - self.buffer.advance(used); - self.release(used as u64); + /// Whether this stream was evicted, registering `cx` to hear when it is. The eviction + /// already stopped the stream and took its buffer and credit. + fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { + self.evicted |= self.hold.as_ref().is_some_and(|hold| hold.poll_evicted(cx)); + self.evicted } - /// Whether this stream was evicted, registering `cx` when given. The eviction already - /// stopped the stream and took its credit, so what is buffered is all that is left. - fn evicted(&mut self, cx: Option<&Context<'_>>) -> bool { - if self.evicted { - return true; - } - let evicted = match (&self.hold, cx) { - (None, _) => false, - (Some(hold), Some(cx)) => hold.poll_evicted(cx), - (Some(hold), None) => hold.is_evicted(), - }; - if evicted { - self.evicted = true; - self.free = self.buffer.len() as u64 + self.unit; - } - evicted + /// The buffered bytes, for tests. + #[cfg(test)] + fn buffered(&mut self) -> usize { + Buffer::of(&mut self.buffer, &self.hold).len() } /// Poll `decode` against the buffered bytes, reading more until it stops coming up @@ -134,22 +118,25 @@ impl Reader { V: Into
+ Copy, { loop { - let mut r = Decoder::new(&self.buffer, self.version.into()); - match decode(&mut r) { + let mut buffer = Buffer::of(&mut self.buffer, &self.hold); + let mut r = Decoder::new(&buffer, self.version.into()); + let decoded = decode(&mut r); + let used = buffer.len() - r.remaining(); + match decoded { Ok(value) => { if consume { - let used = self.buffer.len() - r.remaining(); - self.consume(used); + buffer.advance(used); + buffer.release(&mut self.free, used as u64); } return Poll::Ready(Ok(value)); } - // Stream closed while we still need more data. - Err(DecodeError::Short) if !ready!(self.poll_read_more(cx))? => { - return Poll::Ready(Err(DecodeError::Short.into())); - } - Err(DecodeError::Short) => {} + Err(DecodeError::Short) => drop(buffer), Err(e) => return Poll::Ready(Err(e.into())), } + // Stream closed while we still need more data. + if !ready!(self.poll_read_more(cx))? { + return Poll::Ready(Err(DecodeError::Short.into())); + } } } @@ -285,24 +272,31 @@ impl Reader { /// The next chunk of the current unit, draining the buffer first. Its credit stays /// held in `unit` until the caller releases it. fn poll_unit_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { - if !self.buffer.is_empty() { - let n = cmp::min(self.buffer.len(), max); + let mut buffer = Buffer::of(&mut self.buffer, &self.hold); + if !buffer.is_empty() { + let n = cmp::min(buffer.len(), max); self.unit += n as u64; - return Poll::Ready(Ok(Some(self.buffer.split_to(n).freeze()))); + return Poll::Ready(Ok(Some(buffer.split_to(n).freeze()))); } - if self.evicted(None) { + if self.evicted || buffer.is_evicted() { + self.evicted = true; return Poll::Ready(Err(Error::Evicted)); } let read = match self.stream.poll_read_chunk(cx, max) { - Poll::Pending if self.evicted(Some(cx)) => return Poll::Ready(Err(Error::Evicted)), - Poll::Pending => return Poll::Pending, + Poll::Pending => { + drop(buffer); + return match self.poll_evicted(cx) { + true => Poll::Ready(Err(Error::Evicted)), + false => Poll::Pending, + }; + } Poll::Ready(read) => read, }; let Some((chunk, credit)) = read.map_err(|err| self.version.transport_error(err))? else { return Poll::Ready(Ok(None)); }; self.unit += chunk.len() as u64; - self.take(credit); + buffer.take(credit); Poll::Ready(Ok(Some(chunk))) } @@ -364,14 +358,18 @@ impl Reader { /// Poll for exactly `size` bytes, accumulating partial reads in the buffer. pub fn poll_read_exact(&mut self, cx: &mut Context<'_>, size: usize) -> Poll> { - while self.buffer.len() < size { + loop { + let mut buffer = Buffer::of(&mut self.buffer, &self.hold); + if buffer.len() >= size { + let bytes = buffer.split_to(size).freeze(); + buffer.release(&mut self.free, size as u64); + return Poll::Ready(Ok(bytes)); + } + drop(buffer); if !ready!(self.poll_read_more(cx))? { return Poll::Ready(Err(DecodeError::Short.into())); } } - let bytes = self.buffer.split_to(size).freeze(); - self.release(size as u64); - Poll::Ready(Ok(bytes)) } /// Poll to discard `remaining` bytes, counting it down as they arrive. Nothing is @@ -379,8 +377,11 @@ impl Reader { /// memory. pub(crate) fn poll_skip(&mut self, cx: &mut Context<'_>, remaining: &mut usize) -> Poll> { loop { - let skipped = self.buffer.len().min(*remaining); - self.consume(skipped); + let mut buffer = Buffer::of(&mut self.buffer, &self.hold); + let skipped = buffer.len().min(*remaining); + buffer.advance(skipped); + buffer.release(&mut self.free, skipped as u64); + drop(buffer); *remaining -= skipped; if *remaining == 0 { return Poll::Ready(Ok(())); @@ -407,7 +408,7 @@ impl Reader { /// Poll for whether data is available in the buffer or stream: `false` once it finishes. pub(crate) fn poll_has_more(&mut self, cx: &mut Context<'_>) -> Poll> { - if !self.buffer.is_empty() { + if !Buffer::of(&mut self.buffer, &self.hold).is_empty() { return Poll::Ready(Ok(true)); } @@ -417,17 +418,24 @@ impl Reader { /// Poll for more data from the stream. `true` if data was read, `false` if the /// stream finished. fn poll_read_more(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.evicted(None) { + let mut buffer = Buffer::of(&mut self.buffer, &self.hold); + if self.evicted || buffer.is_evicted() { + self.evicted = true; return Poll::Ready(Err(Error::Evicted)); } - let read = match self.stream.poll_read_buf(cx, &mut self.buffer) { - Poll::Pending if self.evicted(Some(cx)) => return Poll::Ready(Err(Error::Evicted)), - Poll::Pending => return Poll::Pending, + let read = match self.stream.poll_read_buf(cx, &mut *buffer) { + Poll::Pending => { + drop(buffer); + return match self.poll_evicted(cx) { + true => Poll::Ready(Err(Error::Evicted)), + false => Poll::Pending, + }; + } Poll::Ready(read) => read, }; match read { Ok(Some(credit)) => { - self.take(credit); + buffer.take(credit); Poll::Ready(Ok(true)) } Ok(None) => Poll::Ready(Ok(false)), @@ -445,7 +453,7 @@ impl Reader { self.stream.stop(self.version.encode_stream_code(&err.into())); } // What is buffered stays readable, but nothing more is worth holding for. - self.free = self.buffer.len() as u64 + self.unit; + self.free = Buffer::of(&mut self.buffer, &self.hold).len() as u64 + self.unit; if let Some(hold) = &self.hold { hold.clear(); } @@ -465,6 +473,64 @@ impl Reader { } } +/// A reader's buffered bytes: its own while untracked, else locked in its hold beside +/// their credit, so no eviction runs between a check and a read. +enum Buffer<'a, S: crate::transport::poll::RecvStream> { + Own(&'a mut BytesMut), + Held(Locked<'a, S>), +} + +impl<'a, S: crate::transport::poll::RecvStream> Buffer<'a, S> { + fn of(own: &'a mut BytesMut, hold: &'a Option>>) -> Self { + match hold { + Some(hold) => Self::Held(hold.lock()), + None => Self::Own(own), + } + } + + /// Whether an eviction emptied the buffer, after which nothing may be read into it. + fn is_evicted(&self) -> bool { + matches!(self, Self::Held(held) if held.is_evicted()) + } + + /// Keep the credit for bytes just read, if tracked; otherwise it returns now. + fn take(&mut self, credit: S::Credit) { + if let Self::Held(held) = self { + held.take(credit); + } + } + + /// Return the credit for `bytes` that completed a unit, less the `free` bytes read + /// before tracking began. + fn release(&mut self, free: &mut u64, bytes: u64) { + let skip = (*free).min(bytes); + *free -= skip; + if let Self::Held(held) = self { + held.release(bytes - skip); + } + } +} + +impl Deref for Buffer<'_, S> { + type Target = BytesMut; + + fn deref(&self) -> &BytesMut { + match self { + Self::Own(buffer) => buffer, + Self::Held(held) => held.buffer(), + } + } +} + +impl DerefMut for Buffer<'_, S> { + fn deref_mut(&mut self) -> &mut BytesMut { + match self { + Self::Own(buffer) => buffer, + Self::Held(held) => held.buffer_mut(), + } + } +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -637,7 +703,7 @@ mod tests { let mut remaining = 5; assert!(reader.poll_skip(&mut cx, &mut remaining).is_pending()); assert_eq!(remaining, 1); - assert!(reader.buffer.is_empty(), "nothing skipped is kept"); + assert_eq!(reader.buffered(), 0, "nothing skipped is kept"); } fn refuses_frame + Debug, V: StreamCodes + Into + Copy>( @@ -663,7 +729,7 @@ mod tests { "{result:?}" ); } - assert_eq!(reader.buffer.len(), prefix.len()); + assert_eq!(reader.buffered(), prefix.len()); assert_eq!(reader.stream.0.len(), 1, "read past the complete frame"); } @@ -748,7 +814,7 @@ mod tests { reader.poll_decode::(&mut cx), Poll::Ready(Ok(_)) )); - assert!(reader.buffer.is_empty()); + assert_eq!(reader.buffered(), 0); } /// A consumer parked on the group cannot run until this task yields, so a burst of @@ -1096,5 +1162,47 @@ mod tests { Poll::Ready(Err(memory::Error::Stopped(_))) )); } + + /// Parked readers filled and evicted in turn never buffer more than the window. An + /// eviction that returned the credit but left a parked reader its read-ahead would + /// let the peer refill the window while every evicted reader kept its bytes. + #[test] + fn evicted_parked_readers_free_their_read_ahead() { + const WINDOW: usize = 16; + let (mut a, mut b) = memory::pair(memory::Config { + receive_window: Some(WINDOW), + ..Default::default() + }); + let pressure = Tracked::default(); + // One message, then the start of the next, which the reader keeps as read-ahead. + const MESSAGE: &[u8] = &[0x07, 0x40, 0x01, 0x40, 0x02, 0x40, 0x03, 0x40]; + let mut parked = Vec::new(); + for _ in 0..64 { + let Poll::Ready(Ok(mut send)) = a.poll_open_uni(&mut noop()) else { + panic!("open") + }; + let mut rest = MESSAGE; + while !rest.is_empty() { + match write(&mut send, rest) { + Poll::Ready(n) => rest = &rest[n..], + // Full of parked read-ahead: evict one, as the monitor would. + Poll::Pending => assert!(pressure.evict(), "a full window has a holder"), + } + } + let Poll::Ready(Ok(recv)) = b.poll_accept_uni(&mut noop()) else { + panic!("accept") + }; + let mut reader = Reader::new(recv, Version::Lite05); + reader.track(&pressure, Rank::Unidentified); + assert_eq!(varint(reader.poll_varint(&mut noop())), 7); + parked.push((send, reader)); + + let buffered: usize = parked.iter_mut().map(|(_, reader)| reader.buffered()).sum(); + assert!( + buffered <= WINDOW, + "{buffered} bytes buffered past a {WINDOW}-byte window" + ); + } + } } } diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 6d43998af74..88b11714ca6 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -1,13 +1,13 @@ //! Receive-side recovery for when held connection credit fills the allowance. //! -//! A [`Reader`](crate::coding::Reader) tracked here keeps the connection credit for its -//! buffered bytes and its incomplete unit in a [`Hold`], so incomplete messages are -//! bounded by the transport's receive allowance. When the transport reports that the -//! allowance is spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), +//! A [`Reader`](crate::coding::Reader) tracked here keeps its buffered bytes, and the +//! connection credit for them and for its incomplete unit, in a [`Hold`], so incomplete +//! messages are bounded by the transport's receive allowance. When the transport reports +//! that the allowance is spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), //! nothing can arrive to complete any unit, so the [`Monitor`] evicts the least useful -//! holder: the eviction itself drops the credit and stops the stream with STOP_SENDING, -//! whatever the stream's handler is waiting on, and the handler's next read fails with -//! [`Error::Evicted`](crate::Error::Evicted). +//! holder: the eviction itself drops the buffer and the credit and stops the stream with +//! STOP_SENDING, whatever the stream's handler is waiting on, and the handler's next read +//! fails with [`Error::Evicted`](crate::Error::Evicted). //! //! Victims, first to last: //! - Media, from the broadcast holding the most credit, lowest priority first. Within a @@ -31,6 +31,8 @@ use std::{ task::{Context, Waker}, }; +use bytes::BytesMut; + use crate::{ PathOwned, priority::Priority, @@ -58,60 +60,98 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(PoisonError::into_inner) } -/// A tracked stream's credit, and what evicting it takes, shared by its reader and the -/// registry. +/// A tracked stream's buffer and credit, and what evicting it takes, shared by its reader +/// and the registry. pub(crate) struct Hold { - /// Bytes of credit `credit` covers, read by the registry without the lock. + /// Bytes of credit the slot covers, read by the registry without the lock. held: AtomicU64, - /// The monitor chose this stream: its credit is gone and the stream is stopped. + /// The monitor chose this stream: its buffer and credit are gone and the stream is + /// stopped. evicted: AtomicBool, /// The reader's task, woken on eviction. waker: Mutex>, rank: Mutex, /// Registration order, the last tiebreak: the newest stream goes first. seq: u64, - credit: Mutex>, + slot: Mutex>, stopper: R::Stopper, /// The STOP_SENDING code for an eviction, in the stream's negotiated registry. code: u32, } -impl Hold { +/// A tracked reader's buffered bytes and the credit it holds, under one lock so an +/// eviction frees both: credit returned for bytes still allocated would let a peer refill +/// the window while parked readers keep the memory. +struct Slot { + buffer: BytesMut, + credit: Option, +} + +/// A [`Hold`]'s slot, locked by its reader. +pub(crate) struct Locked<'a, R: RecvStream> { + hold: &'a Hold, + slot: MutexGuard<'a, Slot>, +} + +impl Locked<'_, R> { + /// The reader's buffered bytes. + pub fn buffer(&self) -> &BytesMut { + &self.slot.buffer + } + + /// The reader's buffered bytes, to read into or consume. + pub fn buffer_mut(&mut self) -> &mut BytesMut { + &mut self.slot.buffer + } + + /// Whether the monitor chose this stream. Checked under the lock, a `false` means no + /// eviction can empty the buffer until it is unlocked. + pub fn is_evicted(&self) -> bool { + self.hold.is_evicted() + } + /// Keep the credit for bytes just read, or return it at once if already evicted. - pub fn take(&self, credit: R::Credit) { - let mut held = lock(&self.credit); - // Checked under the lock: an eviction sets the flag before it takes the credit. - if self.evicted.load(Ordering::Acquire) { - drop(held); + pub fn take(&mut self, credit: R::Credit) { + if self.hold.is_evicted() { return drop(credit); } - let len = match held.as_mut() { + let len = match self.slot.credit.as_mut() { Some(held) => { held.merge(credit); held.len() } None => { let len = credit.len(); - *held = Some(credit); + self.slot.credit = Some(credit); len } }; - self.held.store(len, Ordering::Relaxed); + self.hold.held.store(len, Ordering::Relaxed); } /// Return the credit for `bytes` that completed a unit. - pub fn release(&self, bytes: u64) { - let mut held = lock(&self.credit); - let Some(credit) = held.as_mut() else { return }; - let freed = credit.split(bytes); - self.held.store(credit.len(), Ordering::Relaxed); - drop(held); - drop(freed); + pub fn release(&mut self, bytes: u64) { + let Some(credit) = self.slot.credit.as_mut() else { + return; + }; + drop(credit.split(bytes)); + self.hold.held.store(credit.len(), Ordering::Relaxed); + } +} + +impl Hold { + /// Lock the buffer and credit, which an eviction drops together. + pub fn lock(&self) -> Locked<'_, R> { + Locked { + hold: self, + slot: lock(&self.slot), + } } - /// Return everything held: the reader gave up on the stream. + /// Return all the credit: the reader gave up on the stream, and what it buffered + /// stays readable. pub fn clear(&self) { - let credit = lock(&self.credit).take(); + let credit = lock(&self.slot).credit.take(); self.held.store(0, Ordering::Relaxed); drop(credit); } @@ -152,7 +192,7 @@ pub(crate) trait Holder { fn rank(&self) -> Rank; /// Registration order. fn seq(&self) -> u64; - /// Drop the credit, stop the stream, and wake its reader to fail. + /// Drop the buffer and credit, stop the stream, and wake its reader to fail. fn evict(&self); } @@ -174,13 +214,19 @@ impl Holder for Hold { } fn evict(&self) { + // Set before locking, so a reader that locks after it sees the flag and reads + // nothing more into the buffer this empties. self.evicted.store(true, Ordering::Release); - let credit = lock(&self.credit).take(); + let mut slot = lock(&self.slot); + let credit = slot.credit.take(); + let buffer = std::mem::take(&mut slot.buffer); + drop(slot); let held = self.held.swap(0, Ordering::Relaxed); tracing::debug!(rank = ?Holder::rank(self), held, "receive credit exhausted, cancelling a stream"); // Stopped first, so the window the credit frees does not invite more of this stream. self.stopper.stop(self.code); drop(credit); + drop(buffer); let waker = lock(&self.waker).take(); if let Some(waker) = waker { waker.wake(); @@ -278,7 +324,10 @@ impl Pressure { waker: Mutex::new(None), rank: Mutex::new(rank), seq: state.next, - credit: Mutex::new(None), + slot: Mutex::new(Slot { + buffer: BytesMut::new(), + credit: None, + }), stopper, code, }); @@ -463,7 +512,7 @@ mod tests { /// Track a receiver holding `held` bytes. fn hold(pressure: &Probes, rank: Rank, held: u64) -> Arc> { let hold = pressure.track::(rank, Stops::default(), 0x35); - hold.take(Unheld(held)); + hold.lock().take(Unheld(held)); hold } @@ -582,7 +631,7 @@ mod tests { let pressure = Probes::default(); let stops = Stops::default(); let hold = pressure.track::(Rank::Unidentified, stops.clone(), 0x35); - hold.take(Unheld(10)); + hold.lock().take(Unheld(10)); let count = Arc::new(Count::default()); let waker = Waker::from(count.clone()); assert!(!hold.poll_evicted(&Context::from_waker(&waker))); @@ -594,7 +643,7 @@ mod tests { assert!(hold.poll_evicted(&Context::from_waker(&waker))); // A read that raced the eviction returns its credit at once. - hold.take(Unheld(5)); + hold.lock().take(Unheld(5)); assert_eq!(pressure.held(), 0); } From 9c14af159810a8215613f9d48e6d945b4face2a7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 01:14:26 -0700 Subject: [PATCH 45/51] fix(moq-net): never return credit before its buffered bytes are freed An eviction published its flag before taking the slot lock, so a read already under the lock could keep bytes whose credit it then dropped, and it dropped the credit before the buffer. It now sets the flag under the lock and frees the buffer first. Abort discards the buffer with its credit too, since a reader retained after its abort kept memory whose credit had returned. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 14 ++-- rs/moq-net/src/pressure.rs | 115 +++++++++++++++++++++++++++++--- 2 files changed, 113 insertions(+), 16 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 59a00aad739..238c5f8f8ec 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -443,7 +443,8 @@ impl Reader { } } - /// Abort the stream with the given error, returning the credit it holds. + /// Abort the stream with the given error, discarding what it buffered and returning + /// the credit it holds. pub fn abort(&mut self, err: impl Into) { // An evicted stream was stopped with its own code already. if !self.hold.as_ref().is_some_and(|hold| hold.is_evicted()) { @@ -452,8 +453,10 @@ impl Reader { // be read against the wrong table and mean something else. self.stream.stop(self.version.encode_stream_code(&err.into())); } - // What is buffered stays readable, but nothing more is worth holding for. - self.free = Buffer::of(&mut self.buffer, &self.hold).len() as u64 + self.unit; + // Nothing buffered is worth keeping, and a reader retained after its abort must not + // keep memory whose credit returned. The unit's credit returns too. + self.buffer = BytesMut::new(); + self.free = self.unit; if let Some(hold) = &self.hold { hold.clear(); } @@ -1055,7 +1058,7 @@ mod tests { assert!(write(&mut send, &[0x00]).is_pending()); } - /// Aborting lets go of the credit, and what was buffered stays readable. + /// Aborting lets go of the credit and of what was buffered. #[test] fn abort_returns_the_credit() { let (mut send, mut reader, pressure, _sessions) = tracked(None); @@ -1064,8 +1067,7 @@ mod tests { assert_eq!(pressure.held(), 2); reader.abort(&Error::Cancel); assert_eq!(pressure.held(), 0); - assert_eq!(varint(reader.poll_varint(&mut noop())), 7); - assert_eq!(pressure.held(), 0); + assert_eq!(reader.buffered(), 0); } /// Hands out `max`-byte reads of a looping buffer of units, so only the reader costs. diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 88b11714ca6..f6a7ebf338c 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -104,8 +104,9 @@ impl Locked<'_, R> { &mut self.slot.buffer } - /// Whether the monitor chose this stream. Checked under the lock, a `false` means no - /// eviction can empty the buffer until it is unlocked. + /// Whether the monitor chose this stream. An eviction sets it under the lock, so it + /// holds for as long as this does: a `false` means bytes read now keep their credit + /// until an eviction frees both. pub fn is_evicted(&self) -> bool { self.hold.is_evicted() } @@ -148,11 +149,15 @@ impl Hold { } } - /// Return all the credit: the reader gave up on the stream, and what it buffered - /// stays readable. + /// Drop the buffer and return the credit: the reader gave up on the stream. pub fn clear(&self) { - let credit = lock(&self.slot).credit.take(); + let mut slot = lock(&self.slot); + let buffer = std::mem::take(&mut slot.buffer); + let credit = slot.credit.take(); self.held.store(0, Ordering::Relaxed); + drop(slot); + // Freed before the credit returns, as on eviction. + drop(buffer); drop(credit); } @@ -214,19 +219,21 @@ impl Holder for Hold { } fn evict(&self) { - // Set before locking, so a reader that locks after it sees the flag and reads - // nothing more into the buffer this empties. - self.evicted.store(true, Ordering::Release); let mut slot = lock(&self.slot); - let credit = slot.credit.take(); + // Set under the lock, so a reader holding it reads and keeps that credit as one + // step, and one locking after sees the flag and reads nothing more in. + self.evicted.store(true, Ordering::Release); let buffer = std::mem::take(&mut slot.buffer); + let credit = slot.credit.take(); drop(slot); let held = self.held.swap(0, Ordering::Relaxed); tracing::debug!(rank = ?Holder::rank(self), held, "receive credit exhausted, cancelling a stream"); // Stopped first, so the window the credit frees does not invite more of this stream. self.stopper.stop(self.code); - drop(credit); + // The bytes go before their credit: returned credit lets the peer send again at + // once, and the memory it bounds must already be free. drop(buffer); + drop(credit); let waker = lock(&self.waker).take(); if let Some(waker) = waker { waker.wake(); @@ -502,6 +509,94 @@ mod tests { type Probes = Pressure>; + /// Credit that records, as it returns, whether the bytes it covers were freed first: + /// it shares their allocation, so it is the last owner once the buffer is gone. + struct Watched { + len: u64, + bytes: Option, + freed_first: Arc, + } + + impl Credit for Watched { + fn len(&self) -> u64 { + self.len + } + + fn split(&mut self, bytes: u64) -> Self { + self.len -= bytes; + Self { + len: bytes, + bytes: None, + freed_first: self.freed_first.clone(), + } + } + + fn merge(&mut self, other: Self) { + self.len += other.len; + } + } + + impl Drop for Watched { + fn drop(&mut self) { + if let Some(bytes) = &self.bytes { + self.freed_first.store(bytes.is_unique(), Ordering::Relaxed); + } + } + } + + /// A stream whose credit is [`Watched`]. + struct Watch; + + impl RecvStream for Watch { + type Error = crate::lite::test_transport::SinkError; + type Credit = Watched; + type Stopper = Stops; + + fn poll_read(&mut self, _: &mut Context<'_>, _: &mut [u8]) -> Poll, Self::Error>> { + Poll::Pending + } + + fn stop(&mut self, _: u32) {} + + fn stopper(&self) -> Stops { + Stops::default() + } + + fn poll_closed(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + /// Eviction and abort free the buffered bytes before their credit returns, so the + /// window the peer regains never covers memory still allocated. + #[test] + fn buffered_bytes_are_freed_before_their_credit_returns() { + let pressure = Pressure::>::default(); + for evict in [true, false] { + let hold = pressure.track::(Rank::Unidentified, Stops::default(), 0x35); + let freed_first = Arc::new(AtomicBool::new(false)); + { + let mut locked = hold.lock(); + locked.buffer_mut().extend_from_slice(&[7; 64]); + let bytes = locked.buffer_mut().split_off(32).freeze(); + locked.take(Watched { + len: 32, + bytes: Some(bytes), + freed_first: freed_first.clone(), + }); + } + match evict { + true => assert!(pressure.evict()), + false => hold.clear(), + } + assert!( + freed_first.load(Ordering::Relaxed), + "evict={evict}: the credit returned while its bytes were still allocated" + ); + assert!(hold.lock().buffer().is_empty()); + } + } + fn media(broadcast: &str, track: u8, group: u64) -> Rank { Rank::Media { broadcast: Arc::new(Path::new(broadcast).to_owned()), From b9ebaf2df429c37b7958d5d499f08eb9a436731b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 01:33:59 -0700 Subject: [PATCH 46/51] fix(moq-net): keep a partial frame's credit until the frame is given up Bytes a reader moved out of its buffer into a partial frame left their credit in the slot, so an eviction returned it while the frame still held them. The slot now keeps the unit's credit apart from the buffer's: an eviction frees the buffer and its credit, and the unit's returns once the reader, woken by the eviction, gives the frame up. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 86 +++++++++++++++++++++++------ rs/moq-net/src/pressure.rs | 95 ++++++++++++++++++++++++--------- 2 files changed, 140 insertions(+), 41 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 238c5f8f8ec..c0f8845c4c0 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -88,9 +88,10 @@ impl Reader { } } - /// Return the credit for `bytes` that completed a unit. - fn release(&mut self, bytes: u64) { - Buffer::of(&mut self.buffer, &self.hold).release(&mut self.free, bytes); + /// Return the credit for `bytes` of the unit, which completed or was given up. + fn release_unit(&mut self, bytes: u64) { + Buffer::of(&mut self.buffer, &self.hold).release_unit(&mut self.free, bytes); + self.unit -= bytes; } /// Whether this stream was evicted, registering `cx` to hear when it is. The eviction @@ -264,19 +265,20 @@ impl Reader { pub fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { let chunk = ready!(self.poll_unit_chunk(cx, max))?; let size = chunk.as_ref().map_or(0, |chunk| chunk.len() as u64); - self.release(size); - self.unit -= size; + self.release_unit(size); Poll::Ready(Ok(chunk)) } - /// The next chunk of the current unit, draining the buffer first. Its credit stays - /// held in `unit` until the caller releases it. + /// The next chunk of the current unit, draining the buffer first. Its credit moves to + /// the unit, held until the caller releases it: an eviction frees only the buffer. fn poll_unit_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { let mut buffer = Buffer::of(&mut self.buffer, &self.hold); if !buffer.is_empty() { let n = cmp::min(buffer.len(), max); + let chunk = buffer.split_to(n).freeze(); + buffer.advance_unit(self.free, self.unit, n as u64); self.unit += n as u64; - return Poll::Ready(Ok(Some(buffer.split_to(n).freeze()))); + return Poll::Ready(Ok(Some(chunk))); } if self.evicted || buffer.is_evicted() { self.evicted = true; @@ -296,7 +298,7 @@ impl Reader { return Poll::Ready(Ok(None)); }; self.unit += chunk.len() as u64; - buffer.take(credit); + buffer.take_unit(credit); Poll::Ready(Ok(Some(chunk))) } @@ -350,9 +352,8 @@ impl Reader { } }; - // The frame is handed off or discarded either way. - let unit = std::mem::take(&mut self.unit); - self.release(unit); + // The frame is handed off, or the caller discards it on the error. + self.release_unit(self.unit); Poll::Ready(result) } @@ -496,15 +497,23 @@ impl<'a, S: crate::transport::poll::RecvStream> Buffer<'a, S> { matches!(self, Self::Held(held) if held.is_evicted()) } - /// Keep the credit for bytes just read, if tracked; otherwise it returns now. + /// Keep the credit for bytes just read into the buffer, if tracked; otherwise it + /// returns now. fn take(&mut self, credit: S::Credit) { if let Self::Held(held) = self { held.take(credit); } } - /// Return the credit for `bytes` that completed a unit, less the `free` bytes read - /// before tracking began. + /// Keep the credit for bytes just read straight into the unit, if tracked. + fn take_unit(&mut self, credit: S::Credit) { + if let Self::Held(held) = self { + held.take_unit(credit); + } + } + + /// Return the credit for `bytes` consumed from the buffer as a complete unit, less the + /// leading `free` bytes, which hold none. fn release(&mut self, free: &mut u64, bytes: u64) { let skip = (*free).min(bytes); *free -= skip; @@ -512,6 +521,24 @@ impl<'a, S: crate::transport::poll::RecvStream> Buffer<'a, S> { held.release(bytes - skip); } } + + /// Move the credit for `bytes` taken from the buffer into a `unit` already that long. + /// The leading `free` bytes hold no credit, and the unit's come first. + fn advance_unit(&mut self, free: u64, unit: u64, bytes: u64) { + let skip = free.saturating_sub(unit).min(bytes); + if let Self::Held(held) = self { + held.advance_unit(bytes - skip); + } + } + + /// Return the credit for `bytes` of the unit, less the leading `free` bytes. + fn release_unit(&mut self, free: &mut u64, bytes: u64) { + let skip = (*free).min(bytes); + *free -= skip; + if let Self::Held(held) = self { + held.release_unit(bytes - skip); + } + } } impl Deref for Buffer<'_, S> { @@ -1165,6 +1192,35 @@ mod tests { )); } + /// An evicted reader mid-frame keeps the credit for the payload already in its + /// partial frame, which is still allocated, until it gives the frame up. + #[test] + fn a_partial_frame_keeps_its_credit_through_eviction() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + let (mut frame, _payload, _waiter, _wakes) = parked_frame(9); + + // Six bytes read ahead into the buffer, then moved into the frame, and two more + // read straight into it. + assert_eq!(write(&mut send, b"abcdef"), Poll::Ready(6)); + assert!(reader.poll_has_more(&mut noop()).is_ready()); + assert!(reader.poll_read_frame(&mut noop(), &mut frame).is_pending()); + assert_eq!(write(&mut send, b"gh"), Poll::Ready(2)); + assert!(reader.poll_read_frame(&mut noop(), &mut frame).is_pending()); + assert_eq!(pressure.held(), 8); + + assert!(pressure.evict()); + assert_eq!( + pressure.held(), + 8, + "the partial frame's credit returned while it was allocated" + ); + assert!(matches!( + reader.poll_read_frame(&mut noop(), &mut frame), + Poll::Ready(Err(Error::Evicted)) + )); + assert_eq!(pressure.held(), 0); + } + /// Parked readers filled and evicted in turn never buffer more than the window. An /// eviction that returned the credit but left a parked reader its read-ahead would /// let the peer refill the window while every evicted reader kept its bytes. diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index f6a7ebf338c..20022699732 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -79,12 +79,36 @@ pub(crate) struct Hold { code: u32, } -/// A tracked reader's buffered bytes and the credit it holds, under one lock so an -/// eviction frees both: credit returned for bytes still allocated would let a peer refill -/// the window while parked readers keep the memory. +/// A tracked reader's bytes and their credit, under one lock. Credit travels with the +/// bytes it covers and returns only once they are freed: credit returned for bytes still +/// allocated would let a peer refill the window while readers keep the memory. struct Slot { buffer: BytesMut, + /// The credit for `buffer`, which an eviction frees with it. credit: Option, + /// The credit for bytes moved out of `buffer` into the unit being assembled, which the + /// reader returns once it completes or gives the unit up: an eviction cannot free a + /// partial frame it does not own. + unit: Option, +} + +impl Slot { + fn held(&self) -> u64 { + self.credit.as_ref().map_or(0, C::len) + self.unit.as_ref().map_or(0, C::len) + } +} + +/// Add `credit` to `into`. +fn merge(into: &mut Option, credit: C) { + match into { + Some(held) => held.merge(credit), + None => *into = Some(credit), + } +} + +/// Split the credit for `bytes` off `from`, if it holds any. +fn split(from: &mut Option, bytes: u64) -> Option { + from.as_mut().map(|credit| credit.split(bytes)) } /// A [`Hold`]'s slot, locked by its reader. @@ -111,32 +135,46 @@ impl Locked<'_, R> { self.hold.is_evicted() } - /// Keep the credit for bytes just read, or return it at once if already evicted. + /// Keep the credit for bytes just read into the buffer, or return it at once if + /// already evicted. pub fn take(&mut self, credit: R::Credit) { if self.hold.is_evicted() { return drop(credit); } - let len = match self.slot.credit.as_mut() { - Some(held) => { - held.merge(credit); - held.len() - } - None => { - let len = credit.len(); - self.slot.credit = Some(credit); - len - } - }; - self.hold.held.store(len, Ordering::Relaxed); + merge(&mut self.slot.credit, credit); + self.store(); + } + + /// Keep the credit for bytes just read straight into the unit. + pub fn take_unit(&mut self, credit: R::Credit) { + if self.hold.is_evicted() { + return drop(credit); + } + merge(&mut self.slot.unit, credit); + self.store(); } - /// Return the credit for `bytes` that completed a unit. + /// Return the credit for `bytes` consumed from the buffer as a complete unit. pub fn release(&mut self, bytes: u64) { - let Some(credit) = self.slot.credit.as_mut() else { - return; - }; - drop(credit.split(bytes)); - self.hold.held.store(credit.len(), Ordering::Relaxed); + drop(split(&mut self.slot.credit, bytes)); + self.store(); + } + + /// Move the credit for `bytes` taken from the buffer into the unit. + pub fn advance_unit(&mut self, bytes: u64) { + if let Some(credit) = split(&mut self.slot.credit, bytes) { + merge(&mut self.slot.unit, credit); + } + } + + /// Return the credit for `bytes` of the unit, which completed or was given up. + pub fn release_unit(&mut self, bytes: u64) { + drop(split(&mut self.slot.unit, bytes)); + self.store(); + } + + fn store(&self) { + self.hold.held.store(self.slot.held(), Ordering::Relaxed); } } @@ -149,11 +187,12 @@ impl Hold { } } - /// Drop the buffer and return the credit: the reader gave up on the stream. + /// Drop the buffer and return all the credit: the reader gave up on the stream, and its + /// caller gives up any partial unit with it. pub fn clear(&self) { let mut slot = lock(&self.slot); let buffer = std::mem::take(&mut slot.buffer); - let credit = slot.credit.take(); + let credit = (slot.credit.take(), slot.unit.take()); self.held.store(0, Ordering::Relaxed); drop(slot); // Freed before the credit returns, as on eviction. @@ -225,9 +264,12 @@ impl Holder for Hold { self.evicted.store(true, Ordering::Release); let buffer = std::mem::take(&mut slot.buffer); let credit = slot.credit.take(); + // A partial unit's credit stays counted until the reader, woken below, gives up the + // frame that holds its bytes. + self.held.store(slot.held(), Ordering::Relaxed); drop(slot); - let held = self.held.swap(0, Ordering::Relaxed); - tracing::debug!(rank = ?Holder::rank(self), held, "receive credit exhausted, cancelling a stream"); + let freed = credit.as_ref().map_or(0, Credit::len); + tracing::debug!(rank = ?Holder::rank(self), freed, "receive credit exhausted, cancelling a stream"); // Stopped first, so the window the credit frees does not invite more of this stream. self.stopper.stop(self.code); // The bytes go before their credit: returned credit lets the peer send again at @@ -334,6 +376,7 @@ impl Pressure { slot: Mutex::new(Slot { buffer: BytesMut::new(), credit: None, + unit: None, }), stopper, code, From b9611415398babe9eef0a9937d3cf32cd4cf4896 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 01:40:05 -0700 Subject: [PATCH 47/51] fix(moq-net): wait out an evicted partial unit before cancelling another An evicted stream's partial unit now returns its credit only when its reader wakes, so a fill reported in between would cancel a second stream for credit already on its way back. Eviction answers such a fill with the release in flight instead. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/pressure.rs | 42 +++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index 20022699732..c2912241a4f 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -391,8 +391,23 @@ impl Pressure { } /// Evict the least useful holder of credit, returning whether there was one. + /// + /// An evicted stream still holding a partial unit is one: its reader, woken by the + /// eviction, gives the unit up and frees what this fill needs, so cancelling another + /// stream meanwhile would cost it for nothing. pub fn evict(&self) -> bool { - let victim = victim(&self.state.lock().entries); + let victim = { + let state = self.state.lock(); + let releasing = state + .entries + .iter() + .filter_map(Weak::upgrade) + .any(|hold| hold.is_evicted() && hold.held() > 0); + if releasing { + return true; + } + victim(&state.entries) + }; // Evicted without the registry lock: dropping credit and stopping call into the // transport, and readers track and drop meanwhile. match victim { @@ -610,6 +625,31 @@ mod tests { } } + /// An evicted stream still holding a partial unit answers the next fill: its reader + /// frees that credit once it wakes, and another stream goes only if the window is still + /// full after. + #[test] + fn a_release_in_flight_spares_the_next_stream() { + let pressure = Probes::default(); + let partial = pressure.track::(Rank::Unidentified, Stops::default(), 0x35); + partial.lock().take_unit(Unheld(10)); + let other = hold(&pressure, Rank::Unidentified, 5); + + assert!(pressure.evict()); + assert!(Holder::is_evicted(&*partial)); + assert_eq!(pressure.held(), 15, "the partial unit stays counted"); + + assert!(pressure.evict()); + assert!( + !Holder::is_evicted(&*other), + "cancelled while the partial unit was letting go" + ); + + partial.lock().release_unit(10); + assert!(pressure.evict()); + assert!(Holder::is_evicted(&*other)); + } + /// Eviction and abort free the buffered bytes before their credit returns, so the /// window the peer regains never covers memory still allocated. #[test] From 01ab75d051b1c475a0e584c210a603e83731fc6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 01:47:13 -0700 Subject: [PATCH 48/51] fix(moq-tokio,moq-uring): wake a parked read when a Stopper stops it A stop produces no readable event, so a read parked on the stream never heard of an eviction's stop. Both stoppers now wake it, and the uring read fails rather than taking the discarded tail as the stream's end. uring's Stopper gains a public stop to match tokio's. IETF track ranks now share the broadcast's Arc instead of cloning its path, and the quest's test paths point at the integration directory. Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/receive-memory-coverage.md | 4 +- rs/moq-net/src/ietf/subscriber.rs | 9 ++- rs/moq-tokio/src/quic/driver/recv_stream.rs | 7 +- rs/moq-tokio/src/quic/driver/tests.rs | 42 ++++++++++ rs/moq-uring/src/quic/driver/connection.rs | 22 +++++- rs/moq-uring/src/quic/driver/stream.rs | 23 +++++- rs/moq-uring/tests/integration/credit.rs | 86 +++++++++++++++++++++ 7 files changed, 181 insertions(+), 12 deletions(-) diff --git a/quest/m1/quic/receive-memory-coverage.md b/quest/m1/quic/receive-memory-coverage.md index 81e8835d7f1..784ba357ffd 100644 --- a/quest/m1/quic/receive-memory-coverage.md +++ b/quest/m1/quic/receive-memory-coverage.md @@ -24,10 +24,10 @@ to configure a window below `MIN_RECEIVE_WINDOW`. The sender is a Bun script using `@moq/web-transport`, pinned to the test server's certificate hash with WebSocket disabled. It mirrors the Rust tests -in `rs/moq-tokio/tests/receive_credit.rs`. A low-priority track sends a frame +in `rs/moq-tokio/tests/integration/receive_credit.rs`. A low-priority track sends a frame larger than the window. A high-priority frame then has to complete, and a later low-priority group has to arrive after the low stream is stopped. Run -it from Rust the way `rs/moq-tokio/tests/max_age.rs` drives its Bun client, +it from Rust the way `rs/moq-tokio/tests/integration/max_age.rs` drives its Bun client, and wire it into the interop workflow, since normal CI has no Bun. The benchmark sweeps concurrent streams and frame sizes, and compares a diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index e5ad246c543..4c20faf21d5 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -502,8 +502,9 @@ struct TrackState { // The broadcast this track was subscribed from. With the track name it forms the full // track name, which is what decides whether a repeated alias is the fatal collision - // (one alias, two tracks) or the legal sharing of an alias across subscriptions. - broadcast: PathOwned, + // (one alias, two tracks) or the legal sharing of an alias across subscriptions. Shared + // with the rank of every stream it carries, so victim choice compares one pointer. + broadcast: std::sync::Arc, // Units for this track's object Timestamps, from the TIMESCALE Track Property in // SUBSCRIBE_OK. `None` until it arrives, and for a track that declares none: its @@ -552,7 +553,7 @@ impl TrackState { pending: None, name, alias: None, - broadcast, + broadcast: std::sync::Arc::new(broadcast), timescale: None, fill, largest: None, @@ -568,7 +569,7 @@ impl TrackState { /// The rank of this subscription's data stream for `group`, starting at `start`. fn rank(&self, subscribe: RequestId, group: u64, start: u64) -> Rank { Rank::Media { - broadcast: std::sync::Arc::new(self.broadcast.clone()), + broadcast: self.broadcast.clone(), priority: crate::priority::Priority::new(self.priority, subscribe.0, group) .with_order(self.order) .with_start(start), diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 50d5ec9696c..f9d89289ade 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -648,7 +648,8 @@ pub struct Stopper { impl Stopper { /// Stop accepting data, as [`RecvStream::stop`] does. /// - /// The stream's own handle learns of it on its next read, which yields `ClosedStream`. + /// The stream's own handle learns of it on its next read, which yields `ClosedStream`; + /// a read parked on the stream is woken for it. pub fn stop(&self, error_code: VarInt) -> Result<(), ClosedStream> { let mut conn = self.conn.state.lock("Stopper::stop"); if self.is_0rtt && conn.check_0rtt().is_err() { @@ -656,6 +657,10 @@ impl Stopper { } conn.inner.recv_stream(self.stream).stop(error_code)?; conn.check_credit(); + // A stop produces no readable event, so the parked read would never hear of it. + if let Some(waker) = conn.blocked_readers.remove(&self.stream) { + waker.wake(); + } conn.wake(); Ok(()) } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 2cd043eeff2..409908d2f55 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1770,6 +1770,48 @@ async fn stream_drop_removes_blocked_reader() { } } +/// A read parked on a stream wakes when a `Stopper` stops it, and fails as closed. +#[tokio::test] +async fn stopper_wakes_a_parked_read() { + let _guard = subscribe(); + let endpoint_factory = EndpointFactory::new(); + let server = endpoint_factory.endpoint(); + let server_address = server.local_addr().unwrap(); + let client = endpoint_factory.endpoint(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + let mut buf = [0u8; 5]; + stream.read_exact(&mut buf).await.unwrap(); + + let stopper = stream.stopper(); + let (waker, wake_counter) = new_count_waker(); + let mut cx = Context::from_waker(&waker); + let mut buf = [0u8; 64]; + let read_fut = stream.read(&mut buf); + tokio::pin!(read_fut); + assert!(matches!(read_fut.as_mut().poll(&mut cx), Poll::Pending)); + + stopper.stop(0u32.into()).unwrap(); + assert_eq!(wake_counter.wakes(), 1, "the parked read was not woken"); + assert!(matches!( + read_fut.as_mut().poll(&mut cx), + Poll::Ready(Err(crate::quic::driver::ReadError::ClosedStream)) + )); + conn.close(0u32.into(), b"done"); + }); + + let conn = client + .connect(server_address, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + stream.write_all(b"hello").await.unwrap(); + server_task.await.unwrap(); +} + /// Test that dropping a `RecvStream` after cancelling a read and then /// explicitly `stop`ing it doesn't panic. #[tokio::test] diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 221be9ada39..01c3a0894f8 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -8,7 +8,7 @@ use std::time::Instant; use bytes::Bytes; use moq_quic::{ConnectionHandle, Dir, StreamId, VarInt}; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use super::super::{Error, SEGMENT}; use super::endpoint; @@ -58,6 +58,9 @@ pub(crate) struct State { /// driver's per-event path, and moq-quic assigns the ids. readable: FxHashMap, writable: FxHashMap, + /// Receive streams a [`Stopper`](super::Stopper) stopped, so their handle's next read + /// fails rather than reading the discarded tail as the stream's end. + stopped: FxHashSet, /// Send streams waiting for their end: a FIN the peer acknowledged, or a /// `STOP_SENDING`. finishing: FxHashMap, @@ -104,6 +107,7 @@ impl State { open_waiters: kio::WaiterList::new(), readable: FxHashMap::default(), writable: FxHashMap::default(), + stopped: FxHashSet::default(), finishing: FxHashMap::default(), sends: FxHashMap::default(), datagram_recv_waiters: kio::WaiterList::new(), @@ -128,6 +132,7 @@ impl State { /// be parked. fn forget_recv(&mut self, id: StreamId) { self.readable.remove(&id); + self.stopped.remove(&id); } /// Terminate with `err` (the first one wins) and wake absolutely everyone, @@ -255,6 +260,21 @@ impl Inner { self.state.borrow_mut().forget_recv(id); } + /// Record that a [`Stopper`](super::Stopper) stopped receive stream `id`, waking a read + /// parked on it: a stop produces no readable event. + pub(crate) fn stop_recv(&self, id: StreamId) { + let mut state = self.state.borrow_mut(); + state.stopped.insert(id); + if let Some(mut waiters) = state.readable.remove(&id) { + waiters.wake(); + } + } + + /// Whether a [`Stopper`](super::Stopper) stopped receive stream `id`. + pub(crate) fn recv_stopped(&self, id: StreamId) -> bool { + self.state.borrow().stopped.contains(&id) + } + /// Wake anyone parked on stream `id` being readable. pub(crate) fn wake_readable(&self, id: StreamId) { let mut state = self.state.borrow_mut(); diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index 4f9e9706d2b..3f9f1e2962f 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -318,6 +318,11 @@ impl RecvStream { match Self::read(&self.shared, self.id, max, held) { // moq-quic hands out at most `max`. Read::Chunk(bytes) => Poll::Ready(Ok(Some(bytes))), + // What a stop discarded is not the stream's end. + Read::Finished if self.shared.recv_stopped(self.id) => { + self.stopped = true; + Poll::Ready(Err(Error::Quic("stream stopped".to_string()))) + } Read::Finished => { self.finished = true; Poll::Ready(Ok(None)) @@ -479,8 +484,9 @@ impl web_transport_trait::poll::RecvStream for RecvStream { /// Stops a [`RecvStream`] without borrowing it. /// -/// The stream's own handle learns of it on its next read, which fails as stopped. What its -/// closed-watch read ahead stays with that handle until it stops or drops. +/// The stream's own handle learns of it on its next read, which fails as stopped, and a +/// read parked on the stream is woken for it. What its closed-watch read ahead stays with +/// that handle until it stops or drops. #[derive(Clone)] pub struct Stopper { shared: Shared, @@ -488,15 +494,24 @@ pub struct Stopper { } impl Stopper { + /// Stop the stream with `code`, as [`stop`](web_transport_trait::poll::RecvStream::stop) + /// does. + pub fn stop(&self, code: u32) { + self.stop_code(code.into()); + } + /// [`RecvStream::stop_code`] from outside the stream. pub(crate) fn stop_code(&self, code: u64) { - // Err means the stream is already gone, which is what we wanted. - let _ = self + let stopped = self .shared .conn .borrow_mut() .recv_stream(self.id) .stop(VarInt::from_u64(code).unwrap_or(VarInt::MAX)); + // Err means the stream is already gone, which is what we wanted. + if stopped.is_ok() { + self.shared.stop_recv(self.id); + } self.shared.check_credit(); self.shared.kick(); } diff --git a/rs/moq-uring/tests/integration/credit.rs b/rs/moq-uring/tests/integration/credit.rs index f1157a8f9f1..0379718f105 100644 --- a/rs/moq-uring/tests/integration/credit.rs +++ b/rs/moq-uring/tests/integration/credit.rs @@ -196,6 +196,92 @@ fn held_reads_spend_the_connection_window() { .expect("worker"); } +/// A read parked on a stream wakes when its `Stopper` stops it, and fails rather than +/// reading what the stop discarded as the stream's end. +#[test] +fn a_stopper_wakes_and_fails_a_parked_read() { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + #[derive(Default)] + struct Wakes(AtomicUsize); + + impl std::task::Wake for Wakes { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + let Some(mut worker) = worker() else { return }; + let handle = worker.handle(); + let certs = support::certs().expect("certificates"); + + let socket = |handle: &moq_uring::Handle| { + handle + .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) + .expect("socket") + }; + let server_config = quic::server::Config::new( + quic::Identity::open(&certs.cert, &certs.key) + .expect("identity") + .server(&[ALPN], quic::server::ClientAuth::None) + .expect("tls"), + ); + let server = quic::Endpoint::new( + socket(&handle), + quic::endpoint::Config::default().with_server(server_config), + ) + .expect("server endpoint"); + let client = quic::Endpoint::new(socket(&handle), quic::endpoint::Config::default()).expect("dial-only endpoint"); + let mut dial = quic::client::Config::new(server.local_addr(), "localhost"); + dial.alpn = vec![ALPN.to_string()]; + dial.verify = false; + + worker + .block_on(async move { + let mut dialed = client.connect(&dial).await.expect("dial"); + let mut accepted = server.accept().await.expect("accept"); + + // Held open, so the stream never finishes on its own. + let mut send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) + .await + .expect("open_uni"); + let written = within( + &handle, + "the write", + std::future::poll_fn(|cx| send.poll_write(cx, b"hello")), + ) + .await + .expect("write"); + let mut recv = within( + &handle, + "the stream", + std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)), + ) + .await + .expect("accept_uni"); + read(&handle, &mut recv, written).await; + + let stopper = recv.stopper(); + let wakes = Arc::new(Wakes::default()); + let waker = Waker::from(wakes.clone()); + let mut cx = Context::from_waker(&waker); + let mut buf = [0u8; 64]; + assert!(recv.poll_read(&mut cx, &mut buf).is_pending()); + + stopper.stop(0); + assert_eq!(wakes.0.load(Ordering::Relaxed), 1, "the parked read was not woken"); + assert!( + matches!(recv.poll_read(&mut cx, &mut buf), Poll::Ready(Err(_))), + "a stopped stream read as finished" + ); + drop(send); + }) + .expect("worker"); +} + #[test] fn chunked_writes_stop_at_the_window() { const STREAM_WINDOW: usize = 4096; From 4dcd59cadedae1e5457a953ac7170f902f330a52 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 02:49:27 -0700 Subject: [PATCH 49/51] refactor(moq-net): give a partial frame's credit to the frame that owns its bytes A partial frame's credit was released by hand at the end of `poll_read_frame`, before the caller aborted the frame still holding those bytes, so an eviction, a premature FIN or a transport error returned credit for memory that was still allocated. Each exit needed its own fix. The credit now travels as a `Lent` inside a `HeldFrame`, which pairs it with the `ProducerOwned` filling it. The frame is declared first, so finishing, aborting or dropping the frame frees the payload and then returns its credit, on every path alike. `release_unit` and the slot's unit credit are gone. The slot frees what it owns through `consume`, `hand_off`, `clear` and `evict`, each dropping the bytes before their credit. `benchmark_tracked_reads` gains frame columns, untracked and tracked. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/coding/reader.rs | 260 ++++++++++++++++++------------ rs/moq-net/src/fuzz.rs | 10 +- rs/moq-net/src/ietf/subscriber.rs | 20 +-- rs/moq-net/src/lite/subscriber.rs | 26 +-- rs/moq-net/src/pressure.rs | 162 +++++++++++++------ 5 files changed, 296 insertions(+), 182 deletions(-) diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index c0f8845c4c0..9e2e6fdee82 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -6,12 +6,12 @@ use std::{ task::{Context, Poll, ready}, }; -use bytes::{Buf, Bytes, BytesMut}; +use bytes::{Bytes, BytesMut}; use crate::{ Error, StreamError, coding::*, - pressure::{Erase, Hold, Holder, Locked, Pressure, Rank}, + pressure::{Erase, Hold, Holder, Lent, Locked, Pressure, Rank}, }; /// A reader for decoding messages from a stream. @@ -22,26 +22,25 @@ use crate::{ /// `Pending` (or a cancelled wrapper) never desynchronizes the stream. /// /// A [tracked](Self::track) reader holds the transport's connection credit for its -/// buffered bytes and for the unit it is assembling, releasing each unit's bytes once -/// it is decoded and handed off, so read-ahead stays with the next unit. Its buffer sits -/// beside that credit in its [`Hold`], so an eviction frees the memory with the credit -/// even while the reader is parked. An untracked reader returns credit as soon as it -/// reads. +/// buffered bytes, returning each unit's as it is decoded and handed off, so read-ahead +/// stays with the next unit. Its buffer sits beside that credit in its [`Hold`], so an +/// eviction frees the memory with the credit even while the reader is parked. A partial +/// frame's payload carries its own credit in a [`HeldFrame`], which returns it when the +/// frame goes. An untracked reader returns credit as soon as it reads. pub struct Reader { stream: S, /// The buffer until tracked, then empty: tracking moves it into the hold. buffer: BytesMut, version: V, - /// Bytes of the incomplete frame already taken out of `buffer`. - unit: u64, - /// Leading bytes read before tracking began, whose credit already returned. - free: u64, - /// While tracked, the credit for `buffer` plus `unit`, less `free`. + /// While tracked, the buffer and its credit. hold: Option>>, /// The receive allowance filled and this stream was chosen to free it. evicted: bool, } +/// A chunk of a unit, with the credit lent to it if tracked. +type LentChunk = (Bytes, Option>); + /// How much payload may pile up behind a parked consumer before /// [`Reader::poll_read_frame`] wakes it, even while the transport keeps handing over /// ready chunks. @@ -59,8 +58,6 @@ impl Reader { stream, buffer: Default::default(), version, - unit: 0, - free: 0, hold: None, evicted: false, } @@ -74,10 +71,9 @@ impl Reader { if self.hold.is_some() { return self.rank(rank); } - self.free = self.buffer.len() as u64 + self.unit; let code = self.version.encode_stream_code(&StreamError::Evicted); let hold = pressure.track(rank, self.stream.stopper(), code); - *hold.lock().buffer_mut() = std::mem::take(&mut self.buffer); + hold.lock().adopt(std::mem::take(&mut self.buffer)); self.hold = Some(hold); } @@ -88,12 +84,6 @@ impl Reader { } } - /// Return the credit for `bytes` of the unit, which completed or was given up. - fn release_unit(&mut self, bytes: u64) { - Buffer::of(&mut self.buffer, &self.hold).release_unit(&mut self.free, bytes); - self.unit -= bytes; - } - /// Whether this stream was evicted, registering `cx` to hear when it is. The eviction /// already stopped the stream and took its buffer and credit. fn poll_evicted(&mut self, cx: &Context<'_>) -> bool { @@ -126,8 +116,7 @@ impl Reader { match decoded { Ok(value) => { if consume { - buffer.advance(used); - buffer.release(&mut self.free, used as u64); + buffer.consume(used); } return Poll::Ready(Ok(value)); } @@ -264,21 +253,16 @@ impl Reader { /// a complete unit: its credit returns as it is handed over. pub fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { let chunk = ready!(self.poll_unit_chunk(cx, max))?; - let size = chunk.as_ref().map_or(0, |chunk| chunk.len() as u64); - self.release_unit(size); - Poll::Ready(Ok(chunk)) + Poll::Ready(Ok(chunk.map(|(chunk, _credit)| chunk))) } - /// The next chunk of the current unit, draining the buffer first. Its credit moves to - /// the unit, held until the caller releases it: an eviction frees only the buffer. - fn poll_unit_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { + /// The next chunk of a unit, draining the buffer first, with the credit lent to it: + /// whoever keeps the bytes keeps the credit, so an eviction frees only the buffer. + fn poll_unit_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll>, Error>> { let mut buffer = Buffer::of(&mut self.buffer, &self.hold); if !buffer.is_empty() { let n = cmp::min(buffer.len(), max); - let chunk = buffer.split_to(n).freeze(); - buffer.advance_unit(self.free, self.unit, n as u64); - self.unit += n as u64; - return Poll::Ready(Ok(Some(chunk))); + return Poll::Ready(Ok(Some(buffer.lend_buffered(n)))); } if self.evicted || buffer.is_evicted() { self.evicted = true; @@ -297,17 +281,16 @@ impl Reader { let Some((chunk, credit)) = read.map_err(|err| self.version.transport_error(err))? else { return Poll::Ready(Ok(None)); }; - self.unit += chunk.len() as u64; - buffer.take_unit(credit); - Poll::Ready(Ok(Some(chunk))) + let credit = buffer.lend(credit); + Poll::Ready(Ok(Some((chunk, credit)))) } /// Fill a frame's payload from the stream, returning `Pending` once it would block. /// /// `Ready(Ok(()))` means the frame is full and the caller should commit it; an error /// means the caller should abort it. Either way the frame is untouched otherwise, so - /// the caller keeps ownership of the commit/abort decision. The payload's credit is - /// held until then, whichever way the frame ends. + /// the caller keeps ownership of the commit/abort decision. The payload's credit moves + /// into the frame, so it returns only as the frame goes, whichever way that is. /// /// Consumers are woken where the loop yields rather than once per chunk. The wire /// hands over everything it has already buffered in one turn, and each wake pays a @@ -320,41 +303,36 @@ impl Reader { pub(crate) fn poll_read_frame( &mut self, cx: &mut Context<'_>, - frame: &mut crate::frame::ProducerOwned, + frame: &mut HeldFrame, ) -> Poll> { let mut owed = 0; - let result = loop { - if frame.remaining() == 0 { - break Ok(()); + loop { + if frame.frame.remaining() == 0 { + return Poll::Ready(Ok(())); } - match self.poll_unit_chunk(cx, frame.remaining()) { + match self.poll_unit_chunk(cx, frame.frame.remaining()) { Poll::Pending => { if owed > 0 { - frame.notify(); + frame.frame.notify(); } return Poll::Pending; } - Poll::Ready(Ok(Some(chunk))) if !chunk.is_empty() => { + Poll::Ready(Ok(Some((chunk, credit)))) if !chunk.is_empty() => { + frame.keep(credit); owed += chunk.len(); - if let Err(err) = frame.write(chunk) { - break Err(err); - } + frame.frame.write(chunk)?; if owed >= WAKE_BUDGET { - frame.notify(); + frame.frame.notify(); owed = 0; } } // A FIN mid-payload: the declared size never arrived. - Poll::Ready(Ok(_)) => break Err(Error::WrongSize), - Poll::Ready(Err(err)) => break Err(err), + Poll::Ready(Ok(_)) => return Poll::Ready(Err(Error::WrongSize)), + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), } - }; - - // The frame is handed off, or the caller discards it on the error. - self.release_unit(self.unit); - Poll::Ready(result) + } } /// Poll for exactly `size` bytes, accumulating partial reads in the buffer. @@ -362,9 +340,7 @@ impl Reader { loop { let mut buffer = Buffer::of(&mut self.buffer, &self.hold); if buffer.len() >= size { - let bytes = buffer.split_to(size).freeze(); - buffer.release(&mut self.free, size as u64); - return Poll::Ready(Ok(bytes)); + return Poll::Ready(Ok(buffer.hand_off(size))); } drop(buffer); if !ready!(self.poll_read_more(cx))? { @@ -380,8 +356,7 @@ impl Reader { loop { let mut buffer = Buffer::of(&mut self.buffer, &self.hold); let skipped = buffer.len().min(*remaining); - buffer.advance(skipped); - buffer.release(&mut self.free, skipped as u64); + buffer.consume(skipped); drop(buffer); *remaining -= skipped; if *remaining == 0 { @@ -455,9 +430,9 @@ impl Reader { self.stream.stop(self.version.encode_stream_code(&err.into())); } // Nothing buffered is worth keeping, and a reader retained after its abort must not - // keep memory whose credit returned. The unit's credit returns too. + // keep memory whose credit returned. A partial frame's credit is its own, returned + // as its owner aborts it. self.buffer = BytesMut::new(); - self.free = self.unit; if let Some(hold) = &self.hold { hold.clear(); } @@ -469,8 +444,6 @@ impl Reader { stream: self.stream, buffer: self.buffer, version, - unit: self.unit, - free: self.free, hold: self.hold, evicted: self.evicted, } @@ -505,42 +478,78 @@ impl<'a, S: crate::transport::poll::RecvStream> Buffer<'a, S> { } } - /// Keep the credit for bytes just read straight into the unit, if tracked. - fn take_unit(&mut self, credit: S::Credit) { - if let Self::Held(held) = self { - held.take_unit(credit); + /// Discard the first `bytes`, a unit consumed whole, then return their credit. + fn consume(&mut self, bytes: usize) { + match self { + Self::Own(buffer) => bytes::Buf::advance(buffer, bytes), + Self::Held(held) => held.consume(bytes), } } - /// Return the credit for `bytes` consumed from the buffer as a complete unit, less the - /// leading `free` bytes, which hold none. - fn release(&mut self, free: &mut u64, bytes: u64) { - let skip = (*free).min(bytes); - *free -= skip; - if let Self::Held(held) = self { - held.release(bytes - skip); + /// Hand off the first `bytes` as a complete unit, whose credit returns as it leaves. + fn hand_off(&mut self, bytes: usize) -> Bytes { + match self { + Self::Own(buffer) => buffer.split_to(bytes).freeze(), + Self::Held(held) => held.hand_off(bytes), } } - /// Move the credit for `bytes` taken from the buffer into a `unit` already that long. - /// The leading `free` bytes hold no credit, and the unit's come first. - fn advance_unit(&mut self, free: u64, unit: u64, bytes: u64) { - let skip = free.saturating_sub(unit).min(bytes); - if let Self::Held(held) = self { - held.advance_unit(bytes - skip); + /// Move the first `bytes` into a partial unit, with their credit if tracked. + fn lend_buffered(&mut self, bytes: usize) -> LentChunk { + match self { + Self::Own(buffer) => (buffer.split_to(bytes).freeze(), None), + Self::Held(held) => held.lend_buffered(bytes), } } - /// Return the credit for `bytes` of the unit, less the leading `free` bytes. - fn release_unit(&mut self, free: &mut u64, bytes: u64) { - let skip = (*free).min(bytes); - *free -= skip; - if let Self::Held(held) = self { - held.release_unit(bytes - skip); + /// The credit for bytes read straight into a partial unit, if tracked; otherwise it + /// returns now. + fn lend(&self, credit: S::Credit) -> Option> { + match self { + Self::Own(_) => None, + Self::Held(held) => Some(held.lend(credit)), } } } +/// A frame filled from a stream by [`Reader::poll_read_frame`], with the connection credit +/// for the payload it holds so far. The credit is the frame's own, so it returns only as +/// the frame finishes, aborts or drops, on every exit alike. +pub(crate) struct HeldFrame { + /// Declared before `credit`: fields drop in order, so a dropped frame frees its payload + /// before the credit returns and lets the peer send more. + frame: crate::frame::ProducerOwned, + credit: Option>, +} + +impl HeldFrame { + pub fn new(frame: crate::frame::ProducerOwned) -> Self { + Self { frame, credit: None } + } + + /// Keep the credit for payload just written into the frame. + fn keep(&mut self, credit: Option>) { + match (&mut self.credit, credit) { + (Some(held), Some(credit)) => held.merge(credit), + (held, credit @ Some(_)) => *held = credit, + (_, None) => {} + } + } + + /// Publish the complete frame, returning its credit once it is handed off. + pub fn finish(self) -> crate::Result<()> { + // `_credit` lives to the end of the scope, past the frame `finish` consumes. + let Self { frame, credit: _credit } = self; + frame.finish() + } + + /// Fail the frame with `err`, freeing its payload before its credit returns. + pub fn abort(self, err: Error) -> crate::Result<()> { + let Self { frame, credit: _credit } = self; + frame.abort(err) + } +} + impl Deref for Buffer<'_, S> { type Target = BytesMut; @@ -653,14 +662,9 @@ mod tests { } /// A group with one open frame, plus a consumer parked on its payload. - fn parked_frame( + fn parked_frame( size: usize, - ) -> ( - crate::frame::ProducerOwned, - crate::frame::Consumer, - kio::Waiter, - Arc, - ) { + ) -> (HeldFrame, crate::frame::Consumer, kio::Waiter, Arc) { let mut group = crate::group::Info { sequence: 0 }.produce(); let mut consumer = group.consume(); let frame = group @@ -677,7 +681,7 @@ mod tests { let wakes = Arc::new(CountWaker::default()); let waiter = kio::Waiter::new(wakes.clone().into()); - (frame, payload, waiter, wakes) + (HeldFrame::new(frame), payload, waiter, wakes) } /// Hands back one queued chunk per poll, then blocks: a stream whose socket has @@ -1128,8 +1132,8 @@ mod tests { } } - /// Decode size-prefixed units, untracked and tracked: the per-read and per-unit cost - /// of holding credit in the reader. Run by nightly. + /// Decode size-prefixed units, untracked and tracked, as messages and as frames: the + /// per-read and per-unit cost of holding credit in the reader. Run by nightly. #[test] #[ignore = "release benchmark, run by nightly"] fn benchmark_tracked_reads() { @@ -1143,11 +1147,12 @@ mod tests { unit.resize(unit.len() + UNIT, 7); let data = Bytes::from(unit.repeat(1024)); - println!("max_read,untracked_ns_per_unit,tracked_ns_per_unit"); + println!("max_read,untracked_ns_per_unit,tracked_ns_per_unit,untracked_ns_per_frame,tracked_ns_per_frame"); for max in [1200usize, 16 * 1024, 64 * 1024] { - let mut ns = [f64::MAX; 2]; + let mut ns = [f64::MAX; 4]; for _ in 0..RUNS { - for (mode, tracked) in [false, true].into_iter().enumerate() { + let modes = [(false, false), (true, false), (false, true), (true, true)]; + for (mode, (tracked, framed)) in modes.into_iter().enumerate() { let stream = Endless { data: data.clone(), pos: 0, @@ -1158,18 +1163,37 @@ mod tests { if tracked { reader.track(&pressure, Rank::Unidentified); } + let mut group = crate::group::Info { sequence: 0 }.produce(); let start = std::time::Instant::now(); - for _ in 0..UNITS { + for i in 0..UNITS { let size = varint(reader.poll_varint(&mut noop())); - let Poll::Ready(Ok(payload)) = reader.poll_read_exact(&mut noop(), size as usize) else { - panic!("short unit"); + if !framed { + let Poll::Ready(Ok(payload)) = reader.poll_read_exact(&mut noop(), size as usize) + else { + panic!("short unit"); + }; + std::hint::black_box(payload); + continue; + } + // A fresh group now and then, so the cache stays small. + if i % 64 == 0 { + group = crate::group::Info { sequence: i as u64 }.produce(); + } + let info = crate::frame::Info { + size, + timestamp: Some(crate::Timestamp::ZERO), }; - std::hint::black_box(payload); + let frame = group.create_frame_owned(info, &Default::default()).unwrap(); + let mut frame = HeldFrame::new(frame); + let Poll::Ready(Ok(())) = reader.poll_read_frame(&mut noop(), &mut frame) else { + panic!("short frame"); + }; + frame.finish().unwrap(); } ns[mode] = ns[mode].min(start.elapsed().as_nanos() as f64 / UNITS as f64); } } - println!("{max},{:.1},{:.1}", ns[0], ns[1]); + println!("{max},{:.1},{:.1},{:.1},{:.1}", ns[0], ns[1], ns[2], ns[3]); } } @@ -1218,6 +1242,28 @@ mod tests { reader.poll_read_frame(&mut noop(), &mut frame), Poll::Ready(Err(Error::Evicted)) )); + reader.abort(&Error::Evicted); + assert_eq!(pressure.held(), 8, "the reader returned its frame's credit"); + frame.abort(Error::Evicted).unwrap(); + assert_eq!(pressure.held(), 0); + } + + /// A FIN mid-payload fails the frame, whose credit returns only once its owner drops + /// it: the payload is still allocated until then. + #[test] + fn a_truncated_frame_keeps_its_credit_until_dropped() { + let (mut send, mut reader, pressure, _sessions) = tracked(None); + let (mut frame, _payload, _waiter, _wakes) = parked_frame(9); + + assert_eq!(write(&mut send, b"abc"), Poll::Ready(3)); + assert!(reader.poll_read_frame(&mut noop(), &mut frame).is_pending()); + send.finish().expect("finish"); + assert!(matches!( + reader.poll_read_frame(&mut noop(), &mut frame), + Poll::Ready(Err(Error::WrongSize)) + )); + assert_eq!(pressure.held(), 3, "the credit returned before its frame dropped"); + drop(frame); assert_eq!(pressure.held(), 0); } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index a8d969395bb..89ad792a018 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -1080,6 +1080,9 @@ pub fn seeds() -> Vec { seeds } +/// A stream's reader and the frame it fills. +type Receiving = (coding::Reader, coding::HeldFrame); + /// Frames arriving on concurrent group streams, as the frame bench times them: one /// frame per stream, read a chunk per stream per poll turn so the streams interleave /// the way a session's do. @@ -1088,7 +1091,7 @@ pub fn seeds() -> Vec { /// every declared size when the first byte lands and `Some(0)` grows every buffer with /// the bytes received. pub struct FrameRecv { - streams: Vec, frame::ProducerOwned)>>, + streams: Vec>, groups: Vec, // Keeps the groups' track alive. _track: track::Producer, @@ -1127,7 +1130,10 @@ impl FrameRecv { chunk, ready: true, }; - Some((coding::Reader::new(stream, lite::Version::Lite05), frame)) + Some(( + coding::Reader::new(stream, lite::Version::Lite05), + coding::HeldFrame::new(frame), + )) }) .collect(); diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 4c20faf21d5..378653368df 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -4133,7 +4133,7 @@ where /// Pumps moq-transport subgroup objects from a reader into a group producer: /// the id delta, the extension headers (carrying the timestamp), the size, the /// status for empty objects, and the streamed payload. -struct GroupIngest { +struct GroupIngest { has_extensions: bool, timescale: Option, version: Version, @@ -4143,11 +4143,11 @@ struct GroupIngest { gap: Option, /// The object being read is one the group started past, so it is read and dropped. dropping: bool, - phase: IngestPhase, + phase: IngestPhase, budget: frame::Budget, } -enum IngestPhase { +enum IngestPhase { /// Reading the object id delta. Stream end here ends the group. Delta, /// Reading the extension block's size. @@ -4159,14 +4159,14 @@ enum IngestPhase { /// Reading the status of an empty object. Status { timestamp: Option }, /// Streaming the object payload. - Payload { frame: frame::ProducerOwned }, + Payload { frame: crate::coding::HeldFrame }, /// Discarding a dropped object's payload: the bytes of it still to read. Skip { size: usize }, /// An explicit end-of-group or end-of-track status arrived. Finished(Ended), } -impl GroupIngest { +impl GroupIngest { fn new( subscriber: &Subscriber, group: &ietf::GroupHeader, @@ -4566,7 +4566,7 @@ where info: frame::Info { size, timestamp }, fields, }; - let mut frame = producer.create_frame_owned(header, &self.frames)?; + let mut frame = crate::coding::HeldFrame::new(producer.create_frame_owned(header, &self.frames)?); if let Err(err) = std::future::poll_fn(|cx| stream.poll_read_frame(cx, &mut frame)).await { let _ = frame.abort(err.clone()); return Err(err); @@ -5281,7 +5281,7 @@ fn object_time( } } -impl GroupIngest { +impl GroupIngest { fn header(&mut self, group: &group::Producer, info: frame::Info) -> Result { let id = self.prior_object.expect("payload follows object ID"); group.mark_gap(self.prior_object.expect("properties follow Object ID"), self.gap)?; @@ -5300,7 +5300,7 @@ impl GroupIngest { /// `Ready(Ok(_))` once the stream FINs on an object boundary, or an explicit /// end-of-group or end-of-track status arrives. The caller finishes or aborts the /// group; an object cut short mid-payload was already aborted here with the reason. - fn poll( + fn poll( &mut self, reader: &mut Reader, group: &mut group::Producer, @@ -5352,7 +5352,9 @@ impl GroupIngest { let timestamp = object_time(self.timescale, *timestamp)?; let header = self.header(group, frame::Info { size, timestamp })?; let frame = group.create_frame_owned(header, &self.budget)?; - self.phase = IngestPhase::Payload { frame }; + self.phase = IngestPhase::Payload { + frame: crate::coding::HeldFrame::new(frame), + }; } IngestPhase::Status { timestamp } => { let status = ready!(reader.poll_varint(&mut cx))?; diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 333b28c2498..5cf38923494 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -858,13 +858,13 @@ impl UniServe { struct GroupRecv { subscriber: Subscriber, reader: Reader, - state: GroupRecvState, + state: GroupRecvState, } // A state machine's enum is its storage: one transient instance per stream, so the // big variant is the working state, not padding held in bulk. #[allow(clippy::large_enum_variant)] -enum GroupRecvState { +enum GroupRecvState { /// Reading the GROUP header. Header, /// Filling the group, bailing if the track or group dies first. @@ -872,7 +872,7 @@ enum GroupRecvState { /// Guarded: dropping this machine mid-group is a cancellation, not a clean end. group: crate::recv::Group, track: track::Producer, - ingest: FrameIngest, + ingest: FrameIngest, /// Whether readers see the group yet; see [`track::Producer::receive_group`]. shown: bool, _reading: Reading, @@ -1027,20 +1027,20 @@ impl GroupRecv { } } -/// Pumps bare FRAME messages from a reader into a group producer: the wire +/// Pumps bare FRAME messages from a reader of `R` into a group producer: the wire /// format shared by GROUP streams and FETCH responses. -struct FrameIngest { +struct FrameIngest { /// `Some` decodes the lite-05 zigzag-delta timestamp prefix; `None` leaves frames /// untimed (pre-lite-05). timescale: Option, /// Previous frame's raw timestamp value (in `timescale` units), for the /// zigzag-delta decode. The first frame's delta is absolute (prev = 0). prev_ts: u64, - phase: IngestPhase, + phase: IngestPhase, budget: frame::Budget, } -enum IngestPhase { +enum IngestPhase { /// Reading the timestamp delta (skipped without a timescale). Stream end here /// means the group has no more frames. Timing, @@ -1048,10 +1048,10 @@ enum IngestPhase { /// where there is no timing prefix to act as the sentinel). Size { timestamp: Option }, /// Streaming the frame payload. - Payload { frame: frame::ProducerOwned }, + Payload { frame: crate::coding::HeldFrame }, } -impl FrameIngest { +impl FrameIngest { fn new(subscriber: &Subscriber, timescale: Option) -> Self { Self { timescale, @@ -1064,7 +1064,7 @@ impl FrameIngest { /// `Ready(Ok(()))` once the stream FINs on a frame boundary. The caller /// finishes or aborts the group; a frame cut short mid-payload was already /// aborted here with the reason. - fn poll( + fn poll( &mut self, reader: &mut Reader, group: &mut group::Producer, @@ -1107,7 +1107,9 @@ impl FrameIngest { }, &self.budget, )?; - self.phase = IngestPhase::Payload { frame }; + self.phase = IngestPhase::Payload { + frame: crate::coding::HeldFrame::new(frame), + }; } IngestPhase::Payload { frame } => { let failed = ready!(reader.poll_read_frame(&mut cx, frame)).err(); @@ -5970,7 +5972,7 @@ enum FetchRunState { // picked the group up from the cache. joined: kio::Producer, // Boxed so the other states stay small; one allocation per fetch. - ingest: Box, + ingest: Box>, }, Done, } diff --git a/rs/moq-net/src/pressure.rs b/rs/moq-net/src/pressure.rs index c2912241a4f..fdb6dc8e3b1 100644 --- a/rs/moq-net/src/pressure.rs +++ b/rs/moq-net/src/pressure.rs @@ -1,8 +1,9 @@ //! Receive-side recovery for when held connection credit fills the allowance. //! -//! A [`Reader`](crate::coding::Reader) tracked here keeps its buffered bytes, and the -//! connection credit for them and for its incomplete unit, in a [`Hold`], so incomplete -//! messages are bounded by the transport's receive allowance. When the transport reports +//! A [`Reader`](crate::coding::Reader) tracked here keeps its buffered bytes and their +//! connection credit in a [`Hold`], and lends the credit for a partial unit's bytes to +//! whatever owns them as a [`Lent`], so incomplete messages are bounded by the +//! transport's receive allowance. When the transport reports //! that the allowance is spent ([`poll_credit_exhausted`](crate::transport::poll::Session::poll_credit_exhausted)), //! nothing can arrive to complete any unit, so the [`Monitor`] evicts the least useful //! holder: the eviction itself drops the buffer and the credit and stops the stream with @@ -31,7 +32,7 @@ use std::{ task::{Context, Waker}, }; -use bytes::BytesMut; +use bytes::{Buf, Bytes, BytesMut}; use crate::{ PathOwned, @@ -65,6 +66,8 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { pub(crate) struct Hold { /// Bytes of credit the slot covers, read by the registry without the lock. held: AtomicU64, + /// Bytes of credit lent out with partial units, until each [`Lent`] drops. + lent: AtomicU64, /// The monitor chose this stream: its buffer and credit are gone and the stream is /// stopped. evicted: AtomicBool, @@ -83,18 +86,31 @@ pub(crate) struct Hold { /// bytes it covers and returns only once they are freed: credit returned for bytes still /// allocated would let a peer refill the window while readers keep the memory. struct Slot { + /// Declared before `credit`: fields drop in order, so a dropped slot frees the bytes + /// before their credit returns. buffer: BytesMut, - /// The credit for `buffer`, which an eviction frees with it. + /// Leading bytes of `buffer` read before tracking began, which hold no credit. + free: u64, + /// The credit for `buffer` past `free`, which an eviction frees with it. credit: Option, - /// The credit for bytes moved out of `buffer` into the unit being assembled, which the - /// reader returns once it completes or gives the unit up: an eviction cannot free a - /// partial frame it does not own. - unit: Option, } impl Slot { fn held(&self) -> u64 { - self.credit.as_ref().map_or(0, C::len) + self.unit.as_ref().map_or(0, C::len) + self.credit.as_ref().map_or(0, C::len) + } + + /// Split off the credit for the first `bytes` of the buffer, which are leaving it. + fn split(&mut self, bytes: u64) -> Option { + let free = self.free.min(bytes); + self.free -= free; + split(&mut self.credit, bytes - free) + } + + /// Empty the buffer, handing back its bytes and their credit to drop in that order. + fn empty(&mut self) -> (BytesMut, Option) { + self.free = 0; + (std::mem::take(&mut self.buffer), self.credit.take()) } } @@ -113,7 +129,7 @@ fn split(from: &mut Option, bytes: u64) -> Option { /// A [`Hold`]'s slot, locked by its reader. pub(crate) struct Locked<'a, R: RecvStream> { - hold: &'a Hold, + hold: &'a Arc>, slot: MutexGuard<'a, Slot>, } @@ -135,42 +151,51 @@ impl Locked<'_, R> { self.hold.is_evicted() } - /// Keep the credit for bytes just read into the buffer, or return it at once if - /// already evicted. + /// Take over bytes buffered before tracking began, whose credit already returned. + pub fn adopt(&mut self, buffer: BytesMut) { + self.slot.free = buffer.len() as u64; + self.slot.buffer = buffer; + } + + /// Keep the credit for bytes just read into the buffer. An eviction sets its flag under + /// this lock, so a reader that checked it first cannot be reading into an evicted slot. pub fn take(&mut self, credit: R::Credit) { - if self.hold.is_evicted() { - return drop(credit); - } + debug_assert!(!self.hold.is_evicted(), "read into an evicted buffer"); merge(&mut self.slot.credit, credit); self.store(); } - /// Keep the credit for bytes just read straight into the unit. - pub fn take_unit(&mut self, credit: R::Credit) { - if self.hold.is_evicted() { - return drop(credit); - } - merge(&mut self.slot.unit, credit); + /// Discard the first `bytes` of the buffer, a unit consumed whole, then return their + /// credit. + pub fn consume(&mut self, bytes: usize) { + self.slot.buffer.advance(bytes); + let credit = self.slot.split(bytes as u64); self.store(); + drop(credit); } - /// Return the credit for `bytes` consumed from the buffer as a complete unit. - pub fn release(&mut self, bytes: u64) { - drop(split(&mut self.slot.credit, bytes)); + /// Hand off the first `bytes` of the buffer as a complete unit, whose credit returns as + /// it leaves. + pub fn hand_off(&mut self, bytes: usize) -> Bytes { + let unit = self.slot.buffer.split_to(bytes).freeze(); + let credit = self.slot.split(bytes as u64); self.store(); + drop(credit); + unit } - /// Move the credit for `bytes` taken from the buffer into the unit. - pub fn advance_unit(&mut self, bytes: u64) { - if let Some(credit) = split(&mut self.slot.credit, bytes) { - merge(&mut self.slot.unit, credit); - } + /// Move the first `bytes` of the buffer into a partial unit, lending their credit to + /// whatever owns them next. + pub fn lend_buffered(&mut self, bytes: usize) -> (Bytes, Option>) { + let unit = self.slot.buffer.split_to(bytes).freeze(); + let lent = self.slot.split(bytes as u64).map(|credit| self.hold.lend(credit)); + self.store(); + (unit, lent) } - /// Return the credit for `bytes` of the unit, which completed or was given up. - pub fn release_unit(&mut self, bytes: u64) { - drop(split(&mut self.slot.unit, bytes)); - self.store(); + /// Lend the credit for bytes read straight into a partial unit to whatever owns them. + pub fn lend(&self, credit: R::Credit) -> Lent { + self.hold.lend(credit) } fn store(&self) { @@ -178,21 +203,58 @@ impl Locked<'_, R> { } } +/// Credit lent out of a [`Hold`] with the bytes of a partial unit: still counted toward +/// the hold, and returned when dropped. Whatever owns those bytes owns this beside them, so +/// the credit returns only once they are given up. +pub(crate) struct Lent { + credit: Option, + hold: Arc>, +} + +impl Lent { + /// Take over `other`'s credit, lent by the same hold for the same unit. + pub fn merge(&mut self, mut other: Self) { + if let Some(credit) = other.credit.take() { + merge(&mut self.credit, credit); + } + } +} + +impl Drop for Lent { + fn drop(&mut self) { + let Some(credit) = self.credit.take() else { + return; + }; + let len = credit.len(); + // Returned before it stops counting, so a fill reported in between still sees an + // eviction's release in flight rather than cancelling another stream. + drop(credit); + self.hold.lent.fetch_sub(len, Ordering::Relaxed); + } +} + impl Hold { /// Lock the buffer and credit, which an eviction drops together. - pub fn lock(&self) -> Locked<'_, R> { + pub fn lock(self: &Arc) -> Locked<'_, R> { Locked { hold: self, slot: lock(&self.slot), } } - /// Drop the buffer and return all the credit: the reader gave up on the stream, and its - /// caller gives up any partial unit with it. + fn lend(self: &Arc, credit: R::Credit) -> Lent { + self.lent.fetch_add(credit.len(), Ordering::Relaxed); + Lent { + credit: Some(credit), + hold: self.clone(), + } + } + + /// Drop the buffer and return its credit: the reader gave up on the stream. Credit lent + /// to a partial unit returns when its owner gives that up. pub fn clear(&self) { let mut slot = lock(&self.slot); - let buffer = std::mem::take(&mut slot.buffer); - let credit = (slot.credit.take(), slot.unit.take()); + let (buffer, credit) = slot.empty(); self.held.store(0, Ordering::Relaxed); drop(slot); // Freed before the credit returns, as on eviction. @@ -242,7 +304,7 @@ pub(crate) trait Holder { impl Holder for Hold { fn held(&self) -> u64 { - self.held.load(Ordering::Relaxed) + self.held.load(Ordering::Relaxed) + self.lent.load(Ordering::Relaxed) } fn is_evicted(&self) -> bool { @@ -262,11 +324,10 @@ impl Holder for Hold { // Set under the lock, so a reader holding it reads and keeps that credit as one // step, and one locking after sees the flag and reads nothing more in. self.evicted.store(true, Ordering::Release); - let buffer = std::mem::take(&mut slot.buffer); - let credit = slot.credit.take(); - // A partial unit's credit stays counted until the reader, woken below, gives up the - // frame that holds its bytes. - self.held.store(slot.held(), Ordering::Relaxed); + let (buffer, credit) = slot.empty(); + // Credit lent to a partial unit stays counted until the owner of its bytes, woken + // below through the reader, gives the unit up. + self.held.store(0, Ordering::Relaxed); drop(slot); let freed = credit.as_ref().map_or(0, Credit::len); tracing::debug!(rank = ?Holder::rank(self), freed, "receive credit exhausted, cancelling a stream"); @@ -369,14 +430,15 @@ impl Pressure { let mut state = self.state.lock(); let hold = Arc::new(Hold { held: AtomicU64::new(0), + lent: AtomicU64::new(0), evicted: AtomicBool::new(false), waker: Mutex::new(None), rank: Mutex::new(rank), seq: state.next, slot: Mutex::new(Slot { buffer: BytesMut::new(), + free: 0, credit: None, - unit: None, }), stopper, code, @@ -632,7 +694,7 @@ mod tests { fn a_release_in_flight_spares_the_next_stream() { let pressure = Probes::default(); let partial = pressure.track::(Rank::Unidentified, Stops::default(), 0x35); - partial.lock().take_unit(Unheld(10)); + let lent = partial.lock().lend(Unheld(10)); let other = hold(&pressure, Rank::Unidentified, 5); assert!(pressure.evict()); @@ -645,7 +707,7 @@ mod tests { "cancelled while the partial unit was letting go" ); - partial.lock().release_unit(10); + drop(lent); assert!(pressure.evict()); assert!(Holder::is_evicted(&*other)); } @@ -819,10 +881,6 @@ mod tests { assert_eq!(*lock(&stops.0), [0x35]); assert_eq!(count.0.load(Ordering::Relaxed), 1); assert!(hold.poll_evicted(&Context::from_waker(&waker))); - - // A read that raced the eviction returns its credit at once. - hold.lock().take(Unheld(5)); - assert_eq!(pressure.held(), 0); } /// Choose a victim among streams spread across broadcasts. The cost per stream must stay From 0b0fcd70983daf331f1d234a0f0cb4fb13e8250c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 03:50:31 -0700 Subject: [PATCH 50/51] fix(moq-uring): keep a stop from reading as the stream's end in poll_closed The closed-watch read ahead past a Stopper's stop marked the stream finished, so the next read returned a clean end instead of failing as stopped. It now records the stop, as a read does. Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/src/quic/driver/stream.rs | 5 +++++ rs/moq-uring/tests/integration/credit.rs | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs index 3f9f1e2962f..43291be9d3c 100644 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -465,6 +465,11 @@ impl web_transport_trait::poll::RecvStream for RecvStream { } match Self::read(&self.shared, self.id, READ_CHUNK, Some(&mut self.backlog_credit)) { Read::Chunk(bytes) => self.backlog.extend_from_slice(&bytes), + // What a stop discarded is not the stream's end, so the next read fails. + Read::Finished if self.shared.recv_stopped(self.id) => { + self.stopped = true; + return Poll::Ready(Ok(())); + } Read::Finished => { self.finished = true; return Poll::Ready(Ok(())); diff --git a/rs/moq-uring/tests/integration/credit.rs b/rs/moq-uring/tests/integration/credit.rs index 0379718f105..4ff0167e326 100644 --- a/rs/moq-uring/tests/integration/credit.rs +++ b/rs/moq-uring/tests/integration/credit.rs @@ -197,7 +197,8 @@ fn held_reads_spend_the_connection_window() { } /// A read parked on a stream wakes when its `Stopper` stops it, and fails rather than -/// reading what the stop discarded as the stream's end. +/// reading what the stop discarded as the stream's end, even after the closed-watch saw +/// the stop first. #[test] fn a_stopper_wakes_and_fails_a_parked_read() { use std::sync::{ @@ -273,6 +274,7 @@ fn a_stopper_wakes_and_fails_a_parked_read() { stopper.stop(0); assert_eq!(wakes.0.load(Ordering::Relaxed), 1, "the parked read was not woken"); + assert!(recv.poll_closed(&mut cx).is_ready(), "the closed-watch missed the stop"); assert!( matches!(recv.poll_read(&mut cx, &mut buf), Poll::Ready(Err(_))), "a stopped stream read as finished" From 76660fe162ca5decaf8194bf74d14e38a04768fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 04:14:20 -0700 Subject: [PATCH 51/51] test: move this branch's tests onto moq-clock #5681 replaced moq_net_sim and crate::time::Clock with moq-clock and Deadlines, and connect/accept now take a moq_net::time::Instant. The receive-credit tests added here still used the old names. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 6 ++--- rs/moq-net/src/lite/subscriber.rs | 6 ++--- .../tests/integration/receive_credit.rs | 22 +++++++++---------- rs/moq-net/tests/integration/support/mock.rs | 4 ++-- .../tests/integration/receive_credit.rs | 2 +- .../tests/integration/receive_credit.rs | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index c70044270f9..90b50c78e5f 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -11086,7 +11086,7 @@ mod stitch_tests { /// pressure monitor chooses it: the eviction itself returns the credit, which the head may /// need to arrive, and stops the stream, without the tail being polled. The head still /// lands, and the tail finishes from what it had already buffered. - #[moq_net_sim::test] + #[moq_clock::test] async fn an_evicted_tail_waiting_on_its_fill_lets_go() { use crate::coding::StreamCodes as _; @@ -11104,7 +11104,7 @@ mod stitch_tests { let mut serve_tail = h.subscriber.clone(); let mut parked = std::pin::pin!(serve_tail.recv_group(&mut tail)); - let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), parked.as_mut()).await; + let waited = moq_clock::timeout(std::time::Duration::from_secs(1), parked.as_mut()).await; assert!(waited.is_err(), "the tail did not wait for its head: {waited:?}"); assert!(pressure.held() > 0, "the parked tail holds its read-ahead"); @@ -11114,7 +11114,7 @@ mod stitch_tests { let mut fill = h.stream().await; h.subscriber.clone().recv_fill(&mut fill).await.expect("the head lands"); - moq_net_sim::timeout(std::time::Duration::from_secs(1), parked) + moq_clock::timeout(std::time::Duration::from_secs(1), parked) .await .expect("the tail never took its head") .expect("the tail finishes from its buffer"); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index a4feff0f2c0..64a6c34d36d 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -4269,7 +4269,7 @@ mod tests { /// A prefix stream waiting on the peer's SETUP for its link cost holds what it read /// past ANNOUNCE_OK. The pressure monitor choosing it returns that credit and stops the /// stream itself, without the prefix being polled. - #[moq_net_sim::test] + #[moq_clock::test] async fn an_evicted_prefix_waiting_on_its_link_cost_lets_go() { use crate::coding::StreamCodes as _; const VERSION: Version = Version::Lite06; @@ -4293,7 +4293,7 @@ mod tests { let session = crate::lite::test_transport::ScriptedSession::new(script); let log = session.log.clone(); let subscriber = Subscriber::new(SubscriberConfig { - runtime: crate::time::Clock::sim(), + runtime: crate::time::sim(), session, origin, recv_bandwidth: None, @@ -4309,7 +4309,7 @@ mod tests { let mut prefix = AnnouncePrefix::new(subscriber, Path::new("").to_owned()); let mut run = std::pin::pin!(kio::wait(|waiter| prefix.poll(waiter))); - let waited = moq_net_sim::timeout(std::time::Duration::from_secs(1), run.as_mut()).await; + let waited = moq_clock::timeout(std::time::Duration::from_secs(1), run.as_mut()).await; assert!(waited.is_err(), "the prefix did not wait for its link cost: {waited:?}"); assert!(pressure.held() > 0, "the parked prefix holds its read-ahead"); diff --git a/rs/moq-net/tests/integration/receive_credit.rs b/rs/moq-net/tests/integration/receive_credit.rs index 8b4c7e9d594..c784e1ac310 100644 --- a/rs/moq-net/tests/integration/receive_credit.rs +++ b/rs/moq-net/tests/integration/receive_credit.rs @@ -61,11 +61,11 @@ async fn setup(version: Version) -> Setup { let pair = connect_mock(options).await; let consumer = subscriber.consume(); - moq_net_sim::timeout(TIMEOUT, consumer.routed("room")) + moq_clock::timeout(TIMEOUT, consumer.routed("room")) .await .expect("announce timeout") .expect("routed"); - let remote = moq_net_sim::timeout(TIMEOUT, consumer.request_broadcast("room", None)) + let remote = moq_clock::timeout(TIMEOUT, consumer.request_broadcast("room", None)) .await .expect("resolve timeout") .expect("broadcast resolves"); @@ -81,7 +81,7 @@ impl Setup { /// Subscribe to a new track at `priority`, once the publisher sees the demand. async fn subscribe(&self, name: &str, priority: u8) -> (moq_net::track::Producer, moq_net::track::Subscriber) { let track = self.broadcast.create_track(name, timed()).unwrap(); - let sub = moq_net_sim::timeout( + let sub = moq_clock::timeout( TIMEOUT, self.remote .track(name) @@ -98,11 +98,11 @@ impl Setup { /// The next frame's payload, which must arrive in time. async fn next_frame(sub: &mut moq_net::track::Subscriber) -> moq_net::Result { - let mut group = moq_net_sim::timeout(TIMEOUT, sub.recv_group()) + let mut group = moq_clock::timeout(TIMEOUT, sub.recv_group()) .await .expect("group timed out")? .expect("track ended"); - let frame = moq_net_sim::timeout(TIMEOUT, group.read_frame()) + let frame = moq_clock::timeout(TIMEOUT, group.read_frame()) .await .expect("frame timed out")? .expect("group ended"); @@ -111,7 +111,7 @@ async fn next_frame(sub: &mut moq_net::track::Subscriber) -> moq_net::Result) { let mut session = pair.client_transport.clone(); - moq_net_sim::spawn(async move { + moq_clock::sim::spawn(async move { let Ok(mut send) = session.open_uni().await else { return }; let _ = send.write_chunk(Bytes::from(bytes)).await; // Hold the stream open, so only the receiver can end it. @@ -174,7 +174,7 @@ fn partial_group_header(size: u32, body: usize) -> Vec { /// Headers that never complete fill the allowance: one is cancelled, and a subscription /// that arrives afterwards still delivers. -#[moq_net_sim::test] +#[moq_clock::test] async fn unidentified_headers_are_cancelled() { let setup = setup("moq-lite-06".parse().unwrap()).await; @@ -191,13 +191,13 @@ async fn unidentified_headers_are_cancelled() { /// Control requests that never complete fill the allowance: one is cancelled with its /// stream, and the session carries on. -#[moq_net_sim::test] +#[moq_clock::test] async fn partial_control_requests_are_cancelled() { let setup = setup("moq-lite-06".parse().unwrap()).await; for _ in 0..2 { let mut session = setup.pair.client_transport.clone(); - moq_net_sim::spawn(async move { + moq_clock::sim::spawn(async move { let Ok((mut send, _recv)) = session.open_bi().await else { return; }; @@ -220,7 +220,7 @@ async fn partial_control_requests_are_cancelled() { /// A frame as large as the allowance allows completes while partial frames of a less /// urgent track keep arriving, each cancelled as it gets in the way. -#[moq_net_sim::test] +#[moq_clock::test] async fn the_largest_unit_completes_under_pressure() { for version in VERSIONS { let setup = setup(version.parse().unwrap()).await; diff --git a/rs/moq-net/tests/integration/support/mock.rs b/rs/moq-net/tests/integration/support/mock.rs index 2c3b6691bc3..2e53c03c882 100644 --- a/rs/moq-net/tests/integration/support/mock.rs +++ b/rs/moq-net/tests/integration/support/mock.rs @@ -399,8 +399,8 @@ impl moq_net::transport::Stopper for MockStopper { return moq_net::transport::Stopper::stop(&self.inner, code); } let inner = self.inner.clone(); - drop(moq_net_sim::spawn(async move { - moq_net_sim::sleep(latency).await; + drop(moq_clock::sim::spawn(async move { + moq_clock::sleep(latency).await; moq_net::transport::Stopper::stop(&inner, code); })); } diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 4bd54161e7d..d9c146d81c8 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -83,7 +83,7 @@ async fn setup(version: &str) -> Setup { let (client, driver) = moq_net::Client::new() .with_versions(version.into()) .with_subscriber(subscriber.clone()) - .connect(std::time::Instant::now(), session) + .connect(moq_net::time::now(), session) .await .expect("client handshake"); tokio::spawn(moq_net::time::run(driver)); diff --git a/rs/moq-uring/tests/integration/receive_credit.rs b/rs/moq-uring/tests/integration/receive_credit.rs index e0d952f71a6..085ed546aff 100644 --- a/rs/moq-uring/tests/integration/receive_credit.rs +++ b/rs/moq-uring/tests/integration/receive_credit.rs @@ -119,7 +119,7 @@ fn partial_frame_gives_way() { let (session, driver) = moq_net::Server::new() .with_publisher(&pub_origin) .accept_lite( - std::time::Instant::now(), + moq_net::time::now(), moq_uring::transport::Session::new(quic::web::Session::raw(conn), moq_net::transport::Binding::InBand), ) .await @@ -135,7 +135,7 @@ fn partial_frame_gives_way() { let (session, driver) = moq_net::Client::new() .with_subscriber(sub.clone()) .connect_lite( - std::time::Instant::now(), + moq_net::time::now(), moq_uring::transport::Session::new( quic::web::Session::raw(conn), moq_net::transport::Binding::InBand,