From a62b26dc8ed4917362737e7d9ebc81145b77dac3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:26:58 -0700 Subject: [PATCH 01/17] quest: claim quest/m1/quic/fork/switch Co-Authored-By: Claude Opus 5.5 From 590d323102f2dd9b7630077175b05ae164aa9b6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:32:13 -0700 Subject: [PATCH 02/17] feat(moq-tokio): import quinn's async layer and web-transport-moq verbatim Unmodified copies, not yet compiled, so a reviewer can diff the adaptation that follows against upstream: - quinn-rs/quinn 7616e6b2782722f3ce4a1b181ef08817d7f545e4 quinn/src -> rs/moq-tokio/src/quic/driver (lib.rs -> mod.rs), the commit moq-quic forked from. - moq-dev/noq e27c0ce00741d81c8620b98e345d4f2c88e0768c web-transport-moq/src -> rs/moq-tokio/src/quic/web (lib.rs -> mod.rs). Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1467 +++++++++++++ rs/moq-tokio/src/quic/driver/endpoint.rs | 1020 +++++++++ rs/moq-tokio/src/quic/driver/incoming.rs | 152 ++ rs/moq-tokio/src/quic/driver/mod.rs | 134 ++ rs/moq-tokio/src/quic/driver/mutex.rs | 163 ++ rs/moq-tokio/src/quic/driver/recv_stream.rs | 823 ++++++++ rs/moq-tokio/src/quic/driver/runtime/mod.rs | 330 +++ rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 + rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 111 + rs/moq-tokio/src/quic/driver/send_stream.rs | 454 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 1871 +++++++++++++++++ rs/moq-tokio/src/quic/driver/work_limiter.rs | 224 ++ rs/moq-tokio/src/quic/web/client.rs | 378 ++++ rs/moq-tokio/src/quic/web/connect.rs | 158 ++ rs/moq-tokio/src/quic/web/crypto.rs | 67 + rs/moq-tokio/src/quic/web/error.rs | 405 ++++ rs/moq-tokio/src/quic/web/mod.rs | 71 + rs/moq-tokio/src/quic/web/recv.rs | 159 ++ rs/moq-tokio/src/quic/web/send.rs | 197 ++ rs/moq-tokio/src/quic/web/server.rs | 200 ++ rs/moq-tokio/src/quic/web/session.rs | 1331 ++++++++++++ rs/moq-tokio/src/quic/web/settings.rs | 70 + 22 files changed, 9890 insertions(+) create mode 100644 rs/moq-tokio/src/quic/driver/connection.rs create mode 100644 rs/moq-tokio/src/quic/driver/endpoint.rs create mode 100644 rs/moq-tokio/src/quic/driver/incoming.rs create mode 100644 rs/moq-tokio/src/quic/driver/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/mutex.rs create mode 100644 rs/moq-tokio/src/quic/driver/recv_stream.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/mod.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/driver/runtime/tokio.rs create mode 100644 rs/moq-tokio/src/quic/driver/send_stream.rs create mode 100755 rs/moq-tokio/src/quic/driver/tests.rs create mode 100644 rs/moq-tokio/src/quic/driver/work_limiter.rs create mode 100644 rs/moq-tokio/src/quic/web/client.rs create mode 100644 rs/moq-tokio/src/quic/web/connect.rs create mode 100644 rs/moq-tokio/src/quic/web/crypto.rs create mode 100644 rs/moq-tokio/src/quic/web/error.rs create mode 100644 rs/moq-tokio/src/quic/web/mod.rs create mode 100644 rs/moq-tokio/src/quic/web/recv.rs create mode 100644 rs/moq-tokio/src/quic/web/send.rs create mode 100644 rs/moq-tokio/src/quic/web/server.rs create mode 100644 rs/moq-tokio/src/quic/web/session.rs create mode 100644 rs/moq-tokio/src/quic/web/settings.rs diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs new file mode 100644 index 0000000000..6ce968db47 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -0,0 +1,1467 @@ +use std::{ + any::Any, + fmt, + future::Future, + io, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, Waker, ready}, +}; + +use bytes::Bytes; +use pin_project_lite::pin_project; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tokio::sync::{ + Notify, + futures::{Notified, OwnedNotified}, + mpsc, oneshot, watch, +}; +use tracing::{Instrument, Span, debug_span}; + +use crate::{ + ConnectionEvent, Duration, Instant, VarInt, + mutex::Mutex, + recv_stream::RecvStream, + runtime::{AsyncTimer, Runtime, UdpSender}, + send_stream::SendStream, + udp_transmit, +}; +use proto::{ + ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, + StreamId, TransportError, TransportErrorCode, congestion::Controller, +}; + +/// In-progress connection attempt future +/// +/// Resolves into a [`Connection`] once the TLS handshake completes. In TLS 1.3, peer +/// certificates arrive with handshake completion, so they are only available via +/// [`Connection::peer_identity()`]. [`Connection`] is a lightweight handle, so completing the +/// handshake imposes no extra cost; use [`Connection::close()`] to reject peers that fail +/// custom validation. +#[derive(Debug)] +pub struct Connecting { + conn: Option, + connected: Pin>, + handshake_data_ready: Option>, +} + +impl Connecting { + pub(crate) fn new( + handle: ConnectionHandle, + conn: proto::Connection, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + sender: Pin>, + runtime: Arc, + ) -> Self { + let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); + + let conn = ConnectionRef(Arc::new(ConnectionInner { + state: Mutex::new(State::new( + conn, + handle, + endpoint_events, + conn_events, + on_handshake_data_send, + sender, + runtime.clone(), + )), + shared: Shared::default(), + })); + let connected = Box::pin(conn.shared.connected.clone().notified_owned()); + + let driver = ConnectionDriver::new(conn.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {e}"); + } + } + .instrument(Span::current()), + )); + + Self { + conn: Some(conn), + connected, + handshake_data_ready: Some(on_handshake_data_recv), + } + } + + /// Convert into a 0-RTT or 0.5-RTT connection at the cost of weakened security + /// + /// Returns `Ok` immediately if the local endpoint is able to attempt sending 0/0.5-RTT data. + /// If so, the returned [`Connection`] can be used to send application data without waiting for + /// the rest of the handshake to complete, at the cost of weakened cryptographic security + /// guarantees. The [`Connection::authenticated`] future resolves when the handshake does + /// complete, at which point subsequently opened streams and written data will have full + /// cryptographic protection. + /// + /// ## Outgoing + /// + /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually + /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This + /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may + /// arise when the handshake completes. If 0-RTT was rejected, the existence of streams opened + /// and other application data sent by a client prior to the handshake completing will not be + /// conveyed to the remote application. Clients which need to e.g. re-transmit rejected stream + /// data must take care to detect `ZeroRttRejected` errors even on finished streams opened + /// during 0-RTT, e.g. by awaiting an application-layer response on bidirectional streams, or + /// awaiting [`SendStream::stopped`] on unidirectional streams. + /// + /// A server may reject 0-RTT data at its discretion, but accepting 0-RTT data requires the + /// relevant resumption state to be stored in the server, which servers may limit or lose for + /// various reasons including not persisting resumption state across server restarts. + /// + /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Incoming + /// + /// For incoming connections, conversion to 0.5-RTT will always succeed. + /// + /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// implementation's docs for 0-RTT pitfalls. + /// + /// ## Security + /// + /// On outgoing connections, this enables transmission of 0-RTT data, which is vulnerable to + /// replay attacks, and should therefore never invoke non-idempotent operations. + /// + /// On incoming connections, this enables transmission of 0.5-RTT data, which may be sent + /// before TLS client authentication has occurred, and should therefore not be used to send + /// data for which client authentication is being used. + pub fn into_0rtt(mut self) -> Result { + // This lock borrows `self` and would normally be dropped at the end of this scope, so we'll + // have to release it explicitly before returning `self` by value. + let conn = (self.conn.as_mut().unwrap()).state.lock("into_0rtt"); + + let is_ok = conn.inner.has_0rtt() || conn.inner.side().is_server(); + drop(conn); + + if is_ok { + let conn = self.conn.take().unwrap(); + Ok(Connection(conn)) + } else { + Err(self) + } + } + + /// Parameters negotiated during the handshake + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a + /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// + /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. + /// + /// This operation is cancel-safe. + pub async fn handshake_data(&mut self) -> Result, ConnectionError> { + // Taking &mut self allows us to use a single oneshot channel rather than dealing with + // potentially many tasks waiting on the same event. It's a bit of a hack, but keeps things + // simple. + if let Some(x) = self.handshake_data_ready.as_mut() { + let _ = x.await; + // Once the data is ready, apply state changes. This prevents a panic due to + // inconsistent state when retrying a call to `handshake_data`. + self.handshake_data_ready = None; + } + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("handshake"); + inner + .inner + .crypto_session() + .handshake_data() + .ok_or_else(|| { + inner + .error + .clone() + .expect("spurious handshake data ready notification") + }) + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn local_ip(&self) -> Option { + let conn = self.conn.as_ref().unwrap(); + let inner = conn.state.lock("local_ip"); + + inner.inner.local_ip() + } + + /// The peer's UDP address + /// + /// Will panic if called after `poll` has returned `Ready`. + pub fn remote_address(&self) -> SocketAddr { + let conn_ref: &ConnectionRef = self.conn.as_ref().expect("used after yielding Ready"); + conn_ref.state.lock("remote_address").inner.remote_address() + } +} + +impl Future for Connecting { + type Output = Result; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.connected).poll(cx).map(|_| { + let conn = self.conn.take().unwrap(); + let inner = conn.state.lock("connecting"); + if inner.connected { + drop(inner); + Ok(Connection(conn)) + } else { + Err(inner + .error + .clone() + .expect("connected signaled without connection success or error")) + } + }) + } +} + +/// A future that drives protocol logic for a connection +/// +/// This future handles the protocol logic for a single connection, routing events from the +/// `Connection` API object to the `Endpoint` task and the related stream-related interfaces. +/// It also keeps track of outstanding timeouts for the `Connection`. +/// +/// If the connection encounters an error condition, this future will yield an error. It will +/// terminate (yielding `Ok(())`) if the connection was closed without error. Unlike other +/// connection-related futures, this waits for the draining period to complete to ensure that +/// packets still in flight from the peer are handled gracefully. +#[must_use = "connection drivers must be spawned for their connections to function"] +#[derive(Debug)] +struct ConnectionDriver { + conn: ConnectionRef, + span: Span, +} + +impl ConnectionDriver { + fn new(conn: ConnectionRef) -> Self { + let span = { + let conn = &mut conn.state.lock("poll"); + debug_span!("drive", id = conn.handle.0) + }; + + Self { conn, span } + } +} + +impl Future for ConnectionDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let conn = &mut *self.conn.state.lock("poll"); + let _guard = self.span.enter(); + + if let Err(e) = conn.process_conn_events(&self.conn.shared, cx) { + conn.terminate(e, &self.conn.shared); + return Poll::Ready(Ok(())); + } + let mut keep_going = conn.drive_transmit(cx).inspect_err(|_| { + // Transmit failed, so close the connection and clean state before returning the error. + if !conn.inner.is_closed() { + conn.implicit_close(&self.conn.shared); + } + })?; + // If a timer expires, there might be more to transmit. When we transmit something, we + // might need to reset a timer. Hence, we must loop until neither happens. + keep_going |= conn.drive_timer(cx); + conn.forward_endpoint_events(); + conn.forward_app_events(&self.conn.shared); + + if !conn.inner.is_drained() { + if keep_going { + // If the connection hasn't processed all tasks, schedule it again + cx.waker().wake_by_ref(); + } else { + match conn.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => conn.driver = Some(cx.waker().clone()), + }; + } + return Poll::Pending; + } + if conn.error.is_none() { + unreachable!("drained connections always have an error"); + } + Poll::Ready(Ok(())) + } +} + +/// A QUIC connection. +/// +/// If all references to a connection (including every clone of the `Connection` handle, streams of +/// incoming streams, and the various stream types) have been dropped, then the connection will be +/// automatically closed with an `error_code` of 0 and an empty `reason`. You can also close the +/// connection explicitly by calling [`Connection::close()`]. +/// +/// Closing the connection immediately abandons efforts to deliver data to the peer. Upon +/// receiving CONNECTION_CLOSE the peer *may* drop any stream data not yet delivered to the +/// application. [`Connection::close()`] describes in more detail how to gracefully close a +/// connection without losing application data. +/// +/// May be cloned to obtain another handle to the same connection. +/// +/// [`Connection::close()`]: Connection::close +#[derive(Debug, Clone)] +pub struct Connection(ConnectionRef); + +impl Connection { + /// Initiate a new outgoing unidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. + pub fn open_uni(&self) -> OpenUni<'_> { + OpenUni { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Uni as usize].notified(), + } + } + + /// Initiate a new outgoing bidirectional stream. + /// + /// Streams are cheap and instantaneous to open unless blocked by flow control. As a + /// consequence, the peer won't be notified that a stream has been opened until the stream is + /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing + /// anything to [`SendStream`] will never succeed. + /// + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + pub fn open_bi(&self) -> OpenBi<'_> { + OpenBi { + conn: &self.0, + notify: self.0.shared.stream_budget_available[Dir::Bi as usize].notified(), + } + } + + /// Accept the next incoming uni-directional stream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_uni(&self) -> AcceptUni<'_> { + AcceptUni { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Uni as usize].notified(), + } + } + + /// Accept the next incoming bidirectional stream + /// + /// **Important Note**: The `Connection` that calls [`open_bi()`] must write to its [`SendStream`] + /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then + /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. + /// + /// [`accept_bi()`]: crate::Connection::accept_bi + /// [`open_bi()`]: crate::Connection::open_bi + /// [`SendStream`]: crate::SendStream + /// [`RecvStream`]: crate::RecvStream + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no streams were consumed. + pub fn accept_bi(&self) -> AcceptBi<'_> { + AcceptBi { + conn: &self.0, + notify: self.0.shared.stream_incoming[Dir::Bi as usize].notified(), + } + } + + /// Receive an application datagram + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no datagrams were consumed. + pub fn read_datagram(&self) -> ReadDatagram<'_> { + ReadDatagram { + conn: &self.0, + notify: self.0.shared.datagram_received.notified(), + } + } + + /// Wait for the connection to be closed for any reason + /// + /// Despite the return type's name, closed connections are often not an error condition at the + /// application layer. Cases that might be routine include [`ConnectionError::LocallyClosed`] + /// and [`ConnectionError::ApplicationClosed`]. + pub async fn closed(&self) -> ConnectionError { + { + let conn = self.0.state.lock("closed"); + if let Some(error) = conn.error.as_ref() { + return error.clone(); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.closed.notified() + } + .await; + self.0 + .state + .lock("closed") + .error + .as_ref() + .expect("closed without an error") + .clone() + } + + /// Whether the connection is closed, and why. + /// + /// The close_reason is always set to `Some(ConnectionError)` when a socket is + /// closed; whether it was closed manually by calling [`Connection::close()`] or due to + /// an internal error (such as an idle timeout or the peer closing the + /// connection). + /// + /// Note: when the connection is closed, `connection.close_reason().is_some()` will always be true. + pub fn close_reason(&self) -> Option { + self.0.state.lock("close_reason").error.clone() + } + + /// Close the connection immediately. + /// + /// Pending operations will fail immediately with [`ConnectionError::LocallyClosed`]. No + /// more data is sent to the peer and the peer may drop buffered data upon receiving + /// the CONNECTION_CLOSE frame. + /// + /// `error_code` and `reason` are not interpreted, and are provided directly to the peer. + /// + /// `reason` will be truncated to fit in a single packet with overhead; to improve odds that it + /// is preserved in full, it should be kept under 1KiB. + /// + /// # Gracefully closing a connection + /// + /// Only the peer last receiving application data can be certain that all data is + /// delivered. The only reliable action it can then take is to close the connection, + /// potentially with a custom error code. The delivery of the final CONNECTION_CLOSE + /// frame is very likely if both endpoints stay online long enough, and + /// [`Endpoint::wait_idle()`] can be used to provide sufficient time. Otherwise, the + /// remote peer will time out the connection, provided that the idle timeout is not + /// disabled. + /// + /// The sending side can not guarantee all stream data is delivered to the remote + /// application. It only knows the data is delivered to the QUIC stack of the remote + /// endpoint. Once the local side sends a CONNECTION_CLOSE frame in response to calling + /// [`close()`] the remote endpoint may drop any data it received but is as yet + /// undelivered to the application, including data that was acknowledged as received to + /// the local endpoint. + /// + /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`close()`]: Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let conn = &mut *self.0.state.lock("close"); + conn.close(error_code, Bytes::copy_from_slice(reason), &self.0.shared); + } + + /// Wait for the handshake to be confirmed. + /// + /// As a server, who must be authenticated by clients, + /// this happens when the handshake completes + /// upon receiving a TLS Finished message from the client. + /// In return, the server send a HANDSHAKE_DONE frame. + /// + /// As a client, this happens when receiving a HANDSHAKE_DONE frame. + /// At this point, the server has either accepted our authentication, + /// or, if client authentication is not required, accepted our lack of authentication. + pub async fn handshake_confirmed(&self) -> Result<(), ConnectionError> { + { + let conn = self.0.state.lock("handshake_confirmed"); + if let Some(error) = conn.error.as_ref() { + return Err(error.clone()); + } + if conn.handshake_confirmed { + return Ok(()); + } + // Construct the future while the lock is held to ensure we can't miss a wakeup if + // the `Notify` is signaled immediately after we release the lock. `await` it after + // the lock guard is out of scope. + self.0.shared.handshake_confirmed.notified() + } + .await; + if let Some(error) = self.0.state.lock("handshake_confirmed").error.as_ref() { + Err(error.clone()) + } else { + Ok(()) + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Application datagrams are a low-level primitive. They may be lost or delivered out of order, + /// and `data` must both fit inside a single QUIC packet and be smaller than the maximum + /// dictated by the peer. + /// + /// Previously queued datagrams which are still unsent may be discarded to make space for this + /// datagram, in order of oldest to newest. + pub fn send_datagram(&self, data: Bytes) -> Result<(), SendDatagramError> { + let conn = &mut *self.0.state.lock("send_datagram"); + if let Some(x) = &conn.error { + return Err(SendDatagramError::ConnectionLost(x.clone())); + } + use proto::SendDatagramError::*; + match conn.inner.datagrams().send(data, true) { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(e) => Err(match e { + Blocked(..) => unreachable!(), + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + }), + } + } + + /// Transmit `data` as an unreliable, unordered application datagram + /// + /// Unlike [`send_datagram()`], this method will wait for buffer space during congestion + /// conditions, which effectively prioritizes old datagrams over new datagrams. + /// + /// See [`send_datagram()`] for details. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn send_datagram_wait(&self, data: Bytes) -> SendDatagram<'_> { + SendDatagram { + conn: &self.0, + data: Some(data), + notify: self.0.shared.datagrams_unblocked.notified(), + } + } + + /// Compute the maximum size of datagrams that may be passed to [`send_datagram()`]. + /// + /// Returns `None` if datagrams are unsupported by the peer or disabled locally. + /// + /// This may change over the lifetime of a connection according to variation in the path MTU + /// estimate. The peer can also enforce an arbitrarily small fixed limit, but if the peer's + /// limit is large this is guaranteed to be a little over a kilobyte at minimum. + /// + /// Not necessarily the maximum size of received datagrams. + /// + /// [`send_datagram()`]: Connection::send_datagram + pub fn max_datagram_size(&self) -> Option { + self.0 + .state + .lock("max_datagram_size") + .inner + .datagrams() + .max_size() + } + + /// Bytes available in the outgoing datagram buffer + /// + /// When greater than zero, calling [`send_datagram()`](Self::send_datagram) with a datagram of + /// at most this size is guaranteed not to cause older datagrams to be dropped. + pub fn datagram_send_buffer_space(&self) -> usize { + self.0 + .state + .lock("datagram_send_buffer_space") + .inner + .datagrams() + .send_buffer_space() + } + + /// The side of the connection (client or server) + pub fn side(&self) -> Side { + self.0.state.lock("side").inner.side() + } + + /// The peer's UDP address + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn remote_address(&self) -> SocketAddr { + self.0.state.lock("remote_address").inner.remote_address() + } + + /// Watch the peer's UDP address. + /// + /// If `ServerConfig::migration` is `true`, clients may change addresses at will, e.g. when + /// switching to a cellular internet connection. + pub fn watch_remote_address(&self) -> RemoteAddressWatcher { + let receiver = self + .0 + .state + .lock("watch_remote_address") + .remote_address + .subscribe(); + + RemoteAddressWatcher { receiver } + } + + /// The local IP address which was used when the peer established + /// the connection + /// + /// This can be different from the address the endpoint is bound to, in case + /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. + /// + /// This will return `None` for clients, or when the platform does not expose this + /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. + pub fn local_ip(&self) -> Option { + self.0.state.lock("local_ip").inner.local_ip() + } + + /// Current best estimate of this connection's latency (round-trip-time) + pub fn rtt(&self) -> Duration { + self.0.state.lock("rtt").inner.rtt() + } + + /// Minimum RTT seen on this path, ignoring ack delay + pub fn min_rtt(&self) -> Duration { + self.0.state.lock("min_rtt").inner.min_rtt() + } + + /// Returns connection statistics + pub fn stats(&self) -> ConnectionStats { + self.0.state.lock("stats").inner.stats() + } + + /// Current state of the congestion control algorithm, for debugging purposes + pub fn congestion_state(&self) -> Box { + self.0 + .state + .lock("congestion_state") + .inner + .congestion_state() + .clone_box() + } + + /// Succeeds when an incoming connection is proven not to be a replay attack. + /// + /// Only interesting for `Connection`s obtained from [`Connecting::into_0rtt`]. On 1-RTT + /// connections, always completes immediately. Contrast + /// [`handshake_confirmed`](Self::handshake_confirmed), which waits longer on clients e.g. to + /// confirm client authentication. + /// + /// For incoming connections, reads from [`RecvStream`]s are guaranteed not to arise from replay + /// attacks after this succeeds, even for streams accepted or read during 0-RTT. For outgoing + /// connections, streams opened after this succeeds will never be discarded by the server due to + /// 0-RTT rejection. + pub async fn authenticated(&self) -> Result<(), ConnectionError> { + let notified = { + let conn = self.0.state.lock("connected"); + if let Some(e) = &conn.error { + return Err(e.clone()); + } + if conn.connected { + return Ok(()); + } + self.0.shared.connected.notified() + }; + notified.await; + let conn = self.0.state.lock("connected"); + conn.error.clone().map_or(Ok(()), Err) + } + + /// Parameters negotiated during the handshake + /// + /// Guaranteed to return `Some` on fully established connections or after + /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on + /// the returned value. + /// + /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + pub fn handshake_data(&self) -> Option> { + self.0 + .state + .lock("handshake_data") + .inner + .crypto_session() + .handshake_data() + } + + /// Cryptographic identity of the peer + /// + /// The dynamic type returned is determined by the configured + /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. + /// + /// For custom asynchronous validation (e.g. checking a certificate against an external + /// database), call this after awaiting [`Connecting`] and use [`close()`][Self::close] to + /// reject untrusted peers. + pub fn peer_identity(&self) -> Option> { + self.0 + .state + .lock("peer_identity") + .inner + .crypto_session() + .peer_identity() + } + + /// A stable identifier for this connection + /// + /// Peer addresses and connection IDs can change, but this value will remain + /// fixed for the lifetime of the connection. + pub fn stable_id(&self) -> usize { + self.0.stable_id() + } + + /// Update traffic keys spontaneously + /// + /// This primarily exists for testing purposes. + pub fn force_key_update(&self) { + self.0 + .state + .lock("force_key_update") + .inner + .force_key_update() + } + + /// Derive keying material from this connection's TLS session secrets. + /// + /// When both peers call this method with the same `label` and `context` + /// arguments and `output` buffers of equal length, they will get the + /// same sequence of bytes in `output`. These bytes are cryptographically + /// strong and pseudorandom, and are suitable for use as keying material. + /// + /// See [RFC5705](https://tools.ietf.org/html/rfc5705) for more information. + pub fn export_keying_material( + &self, + output: &mut [u8], + label: &[u8], + context: &[u8], + ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + self.0 + .state + .lock("export_keying_material") + .inner + .crypto_session() + .export_keying_material(output, label, context) + } + + /// Modify the number of remotely initiated unidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_uni_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_uni_streams"); + conn.inner.set_max_concurrent_streams(Dir::Uni, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } + + /// See [`proto::TransportConfig::send_window()`] + pub fn set_send_window(&self, send_window: u64) { + let mut conn = self.0.state.lock("set_send_window"); + conn.inner.set_send_window(send_window); + conn.wake(); + } + + /// See [`proto::TransportConfig::receive_window()`] + pub fn set_receive_window(&self, receive_window: VarInt) { + let mut conn = self.0.state.lock("set_receive_window"); + conn.inner.set_receive_window(receive_window); + conn.wake(); + } + + /// Modify the number of remotely initiated bidirectional streams that may be concurrently open + /// + /// No streams may be opened by the peer unless fewer than `count` are already open. Large + /// `count`s increase both minimum and worst-case memory consumption. + pub fn set_max_concurrent_bi_streams(&self, count: VarInt) { + let mut conn = self.0.state.lock("set_max_concurrent_bi_streams"); + conn.inner.set_max_concurrent_streams(Dir::Bi, count); + // May need to send MAX_STREAMS to make progress + conn.wake(); + } +} + +pin_project! { + /// Future produced by [`Connection::open_uni`] + pub struct OpenUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenUni<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(SendStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::open_bi`] + pub struct OpenBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for OpenBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_open(ctx, this.conn, this.notify, Dir::Bi))?; + + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_open<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_open"); + if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } else if let Some(id) = state.inner.streams().open(dir) { + let is_0rtt = state.inner.side().is_client() && state.inner.is_handshaking(); + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => { + notify.set(conn.shared.stream_budget_available[dir as usize].notified()) + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::accept_uni`] + pub struct AcceptUni<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptUni<'_> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Uni))?; + Poll::Ready(Ok(RecvStream::new(conn, id, is_0rtt))) + } +} + +pin_project! { + /// Future produced by [`Connection::accept_bi`] + pub struct AcceptBi<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for AcceptBi<'_> { + type Output = Result<(SendStream, RecvStream), ConnectionError>; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let this = self.project(); + let (conn, id, is_0rtt) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Bi))?; + Poll::Ready(Ok(( + SendStream::new(conn.clone(), id, is_0rtt), + RecvStream::new(conn, id, is_0rtt), + ))) + } +} + +fn poll_accept<'a>( + ctx: &mut Context<'_>, + conn: &'a ConnectionRef, + mut notify: Pin<&mut Notified<'a>>, + dir: Dir, +) -> Poll> { + let mut state = conn.state.lock("poll_accept"); + // Check for incoming streams before checking `state.error` so that already-received streams, + // which are necessarily finite, can be drained from a closed connection. + if let Some(id) = state.inner.streams().accept(dir) { + let is_0rtt = state.inner.is_handshaking(); + state.wake(); // To send additional stream ID credit + drop(state); // Release the lock so clone can take it + return Poll::Ready(Ok((conn.clone(), id, is_0rtt))); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => notify.set(conn.shared.stream_incoming[dir as usize].notified()), + } + } +} + +pin_project! { + /// Future produced by [`Connection::read_datagram`] + pub struct ReadDatagram<'a> { + conn: &'a ConnectionRef, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for ReadDatagram<'_> { + type Output = Result; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("ReadDatagram::poll"); + // Check for buffered datagrams before checking `state.error` so that already-received + // datagrams, which are necessarily finite, can be drained from a closed connection. + if let Some(x) = state.inner.datagrams().recv() { + return Poll::Ready(Ok(x)); + } else if let Some(e) = &state.error { + return Poll::Ready(Err(e.clone())); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagram_received.notified()), + } + } + } +} + +pin_project! { + /// Future produced by [`Connection::send_datagram_wait`] + pub struct SendDatagram<'a> { + conn: &'a ConnectionRef, + data: Option, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for SendDatagram<'_> { + type Output = Result<(), SendDatagramError>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut state = this.conn.state.lock("SendDatagram::poll"); + if let Some(e) = &state.error { + return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); + } + use proto::SendDatagramError::*; + match state + .inner + .datagrams() + .send(this.data.take().unwrap(), false) + { + Ok(()) => { + state.wake(); + Poll::Ready(Ok(())) + } + Err(e) => Poll::Ready(Err(match e { + Blocked(data) => { + this.data.replace(data); + loop { + match this.notify.as_mut().poll(ctx) { + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.conn.shared.datagrams_unblocked.notified()), + } + } + } + UnsupportedByPeer => SendDatagramError::UnsupportedByPeer, + Disabled => SendDatagramError::Disabled, + TooLarge => SendDatagramError::TooLarge, + })), + } + } +} + +/// Watcher produced by [`Connection::watch_remote_address`] +pub struct RemoteAddressWatcher { + receiver: watch::Receiver, +} + +impl RemoteAddressWatcher { + /// Wait for an address change, mark it as seen and return the address. + /// + /// If multiple migrations occur between calls to `wait` or polls of the resulting future, + /// the last observed address will be returned, skipping intermediate updates. + pub async fn wait(&mut self) -> Option { + if self.receiver.changed().await.is_err() { + return None; + } + + Some(*self.receiver.borrow()) + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionRef(Arc); + +impl ConnectionRef { + fn stable_id(&self) -> usize { + &*self.0 as *const _ as usize + } +} + +impl Clone for ConnectionRef { + fn clone(&self) -> Self { + self.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for ConnectionRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let conn = &mut *self.state.lock("drop"); + + if !conn.inner.is_closed() { + // If the driver is alive, it's just it and us, so we'd better shut it down. If it's + // not, we can't do any harm. If there were any streams being opened, then either + // the connection will be closed for an unrelated reason or a fresh reference will + // be constructed for the newly opened stream. + conn.implicit_close(&self.shared); + } + } +} + +impl std::ops::Deref for ConnectionRef { + type Target = ConnectionInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct ConnectionInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +#[derive(Debug, Default)] +pub(crate) struct Shared { + handshake_confirmed: Notify, + /// Notified when new streams may be locally initiated due to an increase in stream ID flow + /// control budget + stream_budget_available: [Notify; 2], + /// Notified when the peer has initiated a new stream + stream_incoming: [Notify; 2], + datagram_received: Notify, + datagrams_unblocked: Notify, + closed: Notify, + connected: Arc, + /// Number of live handles that can used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +pub(crate) struct State { + pub(crate) inner: proto::Connection, + driver: Option, + handle: ConnectionHandle, + on_handshake_data: Option>, + connected: bool, + handshake_confirmed: bool, + timer: Option>>, + timer_deadline: Option, + conn_events: mpsc::UnboundedReceiver, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + pub(crate) blocked_writers: FxHashMap, + pub(crate) blocked_readers: FxHashMap, + pub(crate) stopped: FxHashMap>, + /// Always set to Some before the connection becomes drained + pub(crate) error: Option, + sender: Pin>, + runtime: Arc, + send_buffer: Vec, + /// We buffer a transmit when the underlying I/O would block + buffered_transmit: Option, + remote_address: watch::Sender, +} + +impl State { + #[allow(clippy::too_many_arguments)] + fn new( + inner: proto::Connection, + handle: ConnectionHandle, + endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + conn_events: mpsc::UnboundedReceiver, + on_handshake_data: oneshot::Sender<()>, + sender: Pin>, + runtime: Arc, + ) -> Self { + let remote_address = inner.remote_address(); + + Self { + inner, + driver: None, + handle, + on_handshake_data: Some(on_handshake_data), + connected: false, + handshake_confirmed: false, + timer: None, + timer_deadline: None, + conn_events, + endpoint_events, + blocked_writers: FxHashMap::default(), + blocked_readers: FxHashMap::default(), + stopped: FxHashMap::default(), + error: None, + sender, + runtime, + send_buffer: Vec::new(), + buffered_transmit: None, + remote_address: watch::Sender::new(remote_address), + } + } + + fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { + let now = self.runtime.now(); + let mut transmits = 0; + + let max_datagrams = self + .sender + .max_transmit_segments() + .min(MAX_TRANSMIT_SEGMENTS); + + loop { + // Retry the last transmit, or get a new one. + let t = match self.buffered_transmit.take() { + Some(t) => t, + None => { + self.send_buffer.clear(); + self.send_buffer.reserve(self.inner.current_mtu() as usize); + match self + .inner + .poll_transmit(now, max_datagrams, &mut self.send_buffer) + { + Some(t) => { + transmits += match t.segment_size { + None => 1, + Some(s) => t.size.div_ceil(s), // round up + }; + t + } + None => break, + } + } + }; + + let len = t.size; + match self + .sender + .as_mut() + .poll_send(&udp_transmit(&t, &self.send_buffer[..len]), cx) + { + Poll::Pending => { + self.buffered_transmit = Some(t); + return Ok(false); + } + Poll::Ready(Err(e)) => return Err(e), + Poll::Ready(Ok(())) => {} + } + + if transmits >= MAX_TRANSMIT_DATAGRAMS { + // TODO: What isn't ideal here yet is that if we don't poll all + // datagrams that could be sent we don't go into the `app_limited` + // state and CWND continues to grow until we get here the next time. + // See https://github.com/quinn-rs/quinn/issues/1126 + return Ok(true); + } + } + + Ok(false) + } + + fn forward_endpoint_events(&mut self) { + while let Some(event) = self.inner.poll_endpoint_events() { + // If the endpoint driver is gone, noop. + let _ = self.endpoint_events.send((self.handle, event)); + } + } + + /// If this returns `Err`, the endpoint is dead, so the driver should exit immediately. + fn process_conn_events( + &mut self, + shared: &Shared, + cx: &mut Context<'_>, + ) -> Result<(), ConnectionError> { + loop { + match self.conn_events.poll_recv(cx) { + Poll::Ready(Some(ConnectionEvent::Rebind(sender))) => { + self.sender = sender; + self.inner.local_address_changed(); + } + Poll::Ready(Some(ConnectionEvent::Proto(event))) => { + self.inner.handle_event(event); + } + Poll::Ready(Some(ConnectionEvent::Close { reason, error_code })) => { + self.close(error_code, reason, shared); + } + Poll::Ready(None) => { + return Err(ConnectionError::TransportError(TransportError::new( + TransportErrorCode::INTERNAL_ERROR, + "endpoint driver future was dropped".to_string(), + ))); + } + Poll::Pending => { + return Ok(()); + } + } + } + } + + fn forward_app_events(&mut self, shared: &Shared) { + while let Some(event) = self.inner.poll() { + use proto::Event::*; + match event { + HandshakeDataReady => { + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + } + Connected => { + self.connected = true; + shared.connected.notify_waiters(); + if self.inner.side().is_client() && !self.inner.accepted_0rtt() { + // Wake up rejected 0-RTT streams so they can fail immediately with + // `ZeroRttRejected` errors. + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + wake_all_notify(&mut self.stopped); + } + } + HandshakeConfirmed => { + self.handshake_confirmed = true; + shared.handshake_confirmed.notify_waiters(); + } + ConnectionLost { reason } => { + self.terminate(reason, shared); + } + Stream(StreamEvent::Writable { id }) => wake_stream(id, &mut self.blocked_writers), + Stream(StreamEvent::Opened { dir: Dir::Uni }) => { + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + } + Stream(StreamEvent::Opened { dir: Dir::Bi }) => { + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + } + DatagramReceived => { + shared.datagram_received.notify_waiters(); + } + DatagramsUnblocked => { + shared.datagrams_unblocked.notify_waiters(); + } + Stream(StreamEvent::Readable { id }) => wake_stream(id, &mut self.blocked_readers), + Stream(StreamEvent::Available { dir }) => { + // Might mean any number of streams are ready, so we wake up everyone + shared.stream_budget_available[dir as usize].notify_waiters(); + } + Stream(StreamEvent::Finished { id }) => wake_stream_notify(id, &mut self.stopped), + Stream(StreamEvent::Stopped { id, .. }) => { + wake_stream_notify(id, &mut self.stopped); + wake_stream(id, &mut self.blocked_writers); + } + PathUpdated => { + // Using `send_replace` since `send` does not update the value + // when there are no subscribed receivers. + let _ = self + .remote_address + .send_replace(self.inner.remote_address()); + } + _ => {} + } + } + } + + fn drive_timer(&mut self, cx: &mut Context<'_>) -> bool { + let Some(deadline) = self.inner.poll_timeout() else { + self.timer_deadline = None; + return false; + }; + + // Use the clock rather than the async timer to detect expiry: Sleep::poll + // respects Tokio's cooperative budget and can return Pending for elapsed + // deadlines. + let now = self.runtime.now(); + if now >= deadline { + self.inner.handle_timeout(now); + self.timer_deadline = None; + return true; + } + + match &mut self.timer { + // Avoid resetting the timer when the deadline is unchanged. + Some(delay) if self.timer_deadline != Some(deadline) => { + delay.as_mut().reset(deadline); + } + None => { + self.timer = Some(self.runtime.new_timer(deadline)); + } + _ => {} + } + self.timer_deadline = Some(deadline); + + let delay = self + .timer + .as_mut() + .expect("timer must exist in this state") + .as_mut(); + if delay.poll(cx).is_pending() { + return false; + } + + // The deadline elapsed in the window between the clock check and poll. + self.inner.handle_timeout(self.runtime.now()); + self.timer_deadline = None; + true + } + + /// Wake up a blocked `Driver` task to process I/O + pub(crate) fn wake(&mut self) { + if let Some(x) = self.driver.take() { + x.wake(); + } + } + + /// Used to wake up all blocked futures when the connection becomes closed for any reason + fn terminate(&mut self, reason: ConnectionError, shared: &Shared) { + self.error = Some(reason.clone()); + if let Some(x) = self.on_handshake_data.take() { + let _ = x.send(()); + } + wake_all(&mut self.blocked_writers); + wake_all(&mut self.blocked_readers); + shared.stream_budget_available[Dir::Uni as usize].notify_waiters(); + shared.stream_budget_available[Dir::Bi as usize].notify_waiters(); + shared.stream_incoming[Dir::Uni as usize].notify_waiters(); + shared.stream_incoming[Dir::Bi as usize].notify_waiters(); + shared.datagram_received.notify_waiters(); + shared.datagrams_unblocked.notify_waiters(); + shared.handshake_confirmed.notify_waiters(); + wake_all_notify(&mut self.stopped); + shared.closed.notify_waiters(); + shared.connected.notify_waiters(); + } + + fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { + self.inner.close(self.runtime.now(), error_code, reason); + self.terminate(ConnectionError::LocallyClosed, shared); + self.wake(); + } + + /// Close for a reason other than the application's explicit request + pub(crate) fn implicit_close(&mut self, shared: &Shared) { + self.close(0u32.into(), Bytes::new(), shared); + } + + pub(crate) fn check_0rtt(&self) -> Result<(), ()> { + if self.inner.is_handshaking() + || self.inner.accepted_0rtt() + || self.inner.side().is_server() + { + Ok(()) + } else { + Err(()) + } + } +} + +impl Drop for State { + fn drop(&mut self) { + if !self.inner.is_drained() { + // Ensure the endpoint can tidy up + let _ = self + .endpoint_events + .send((self.handle, EndpointEvent::drained())); + } + } +} + +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("State").field("inner", &self.inner).finish() + } +} + +fn wake_stream(stream_id: StreamId, wakers: &mut FxHashMap) { + if let Some(waker) = wakers.remove(&stream_id) { + waker.wake(); + } +} + +fn wake_all(wakers: &mut FxHashMap) { + wakers.drain().for_each(|(_, waker)| waker.wake()) +} + +fn wake_stream_notify(stream_id: StreamId, wakers: &mut FxHashMap>) { + if let Some(notify) = wakers.remove(&stream_id) { + notify.notify_waiters() + } +} + +fn wake_all_notify(wakers: &mut FxHashMap>) { + wakers + .drain() + .for_each(|(_, notify)| notify.notify_waiters()) +} + +/// Errors that can arise when sending a datagram +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub enum SendDatagramError { + /// The peer does not support receiving datagram frames + #[error("datagrams not supported by peer")] + UnsupportedByPeer, + /// Datagram support is disabled locally + #[error("datagram support disabled")] + Disabled, + /// The datagram is larger than the connection can currently accommodate + /// + /// Indicates that the path MTU minus overhead or the limit advertised by the peer has been + /// exceeded. + #[error("datagram too large")] + TooLarge, + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), +} + +/// The maximum amount of datagrams which will be produced in a single `drive_transmit` call +/// +/// This limits the amount of CPU resources consumed by datagram generation, +/// and allows other tasks (like receiving ACKs) to run in between. +const MAX_TRANSMIT_DATAGRAMS: usize = 20; + +/// The maximum amount of datagrams that are sent in a single transmit +/// +/// This can be lower than the maximum platform capabilities, to avoid excessive +/// memory allocations when calling `poll_transmit()`. Benchmarks have shown +/// that numbers around 10 are a good compromise. +const MAX_TRANSMIT_SEGMENTS: usize = 10; diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs new file mode 100644 index 0000000000..2012e415ff --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -0,0 +1,1020 @@ +use std::{ + collections::VecDeque, + fmt, + future::Future, + io::{self, IoSliceMut}, + mem, + net::{SocketAddr, SocketAddrV6}, + pin::Pin, + str, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use crate::runtime::default_runtime; +use crate::{ + Instant, + runtime::{AsyncUdpSocket, Runtime, UdpSender}, + udp_transmit, +}; +use bytes::{Bytes, BytesMut}; +use pin_project_lite::pin_project; +use proto::{ + self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, + EndpointEvent, ServerConfig, +}; +use rustc_hash::FxHashMap; +#[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), +))] +use socket2::{Domain, Protocol, Socket, Type}; +use tokio::sync::{Notify, futures::Notified, mpsc}; +use tracing::{Instrument, Span}; +use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; + +use crate::{ + ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, + connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, +}; + +/// A QUIC endpoint. +/// +/// An endpoint corresponds to a single UDP socket, may host many connections, and may act as both +/// client and server for different connections. +/// +/// May be cloned to obtain another handle to the same endpoint. +#[derive(Debug, Clone)] +pub struct Endpoint { + pub(crate) inner: EndpointRef, + runtime: Arc, +} + +impl Endpoint { + /// Helper to construct an endpoint for use with outgoing connections only + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 addresses. As such, calling `Endpoint::client` with + /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other + /// address. For example: + /// + /// ``` + /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ``` + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn client(addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + None, + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Returns relevant stats from this Endpoint + pub fn stats(&self) -> EndpointStats { + self.inner.state.lock().unwrap().stats + } + + /// Helper to construct an endpoint for use with both incoming and outgoing connections + /// + /// Note that `addr` is the *local* address to bind to, which should usually be a wildcard + /// address like `0.0.0.0:0` or `[::]:0`, which allow communication with any reachable IPv4 or + /// IPv6 address respectively from an OS-assigned port. + /// + /// If an IPv6 address is provided, attempts to make the socket dual-stack so as to allow + /// communication with both IPv4 and IPv6 clients. As such, calling `Endpoint::server` with + /// the address `[::]:0` is a reasonable default to maximize the ability to accept connections + /// from any address. + /// + /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 + /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never + /// dual-stack. + #[cfg(all( + not(wasm_browser), + any(feature = "runtime-tokio", feature = "runtime-smol"), + any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these + ))] + pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { + let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; + if addr.is_ipv6() + && let Err(e) = socket.set_only_v6(false) + { + tracing::debug!(%e, "unable to make socket dual-stack"); + } + socket.bind(&addr.into())?; + let runtime = + default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; + Self::new_with_abstract_socket( + EndpointConfig::default(), + Some(config), + runtime.wrap_udp_socket(socket.into())?, + runtime, + ) + } + + /// Construct an endpoint with arbitrary configuration and socket + #[cfg(not(wasm_browser))] + pub fn new( + config: EndpointConfig, + server_config: Option, + socket: std::net::UdpSocket, + runtime: Arc, + ) -> io::Result { + let socket = runtime.wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket, runtime) + } + + /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket + /// + /// Useful when `socket` has additional state (e.g. sidechannels) attached for which shared + /// ownership is needed. + pub fn new_with_abstract_socket( + config: EndpointConfig, + server_config: Option, + socket: Box, + runtime: Arc, + ) -> io::Result { + let addr = socket.local_addr()?; + let allow_mtud = !socket.may_fragment(); + let rc = EndpointRef::new( + socket, + proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + addr.is_ipv6(), + runtime.clone(), + ); + let driver = EndpointDriver(rc.clone()); + runtime.spawn(Box::pin( + async { + if let Err(e) = driver.await { + tracing::error!("I/O error: {}", e); + } + } + .instrument(Span::current()), + )); + Ok(Self { inner: rc, runtime }) + } + + /// Get the next incoming connection attempt from a client + /// + /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] + /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// filter connection attempts or force address validation, or converted into an intermediate + /// `Connecting` future which can be used to e.g. send 0.5-RTT data. + pub fn accept(&self) -> Accept<'_> { + Accept { + endpoint: self, + notify: self.inner.shared.incoming.notified(), + } + } + + /// Set the client configuration used by `connect` + pub fn set_default_client_config(&self, config: ClientConfig) { + self.inner.0.state.lock().unwrap().default_client_config = Some(config); + } + + /// Connect to a remote endpoint + /// + /// `server_name` must be covered by the certificate presented by the server. This prevents a + /// connection from being intercepted by an attacker with a valid certificate for some other + /// server. + /// + /// May fail immediately due to configuration errors, or in the future if the connection could + /// not be established. + pub fn connect(&self, addr: SocketAddr, server_name: &str) -> Result { + let Some(config) = self + .inner + .0 + .state + .lock() + .unwrap() + .default_client_config + .clone() + else { + return Err(ConnectError::NoDefaultClientConfig); + }; + + self.connect_with(config, addr, server_name) + } + + /// Connect to a remote endpoint using a custom configuration. + /// + /// See [`connect()`] for details. + /// + /// [`connect()`]: Endpoint::connect + pub fn connect_with( + &self, + config: ClientConfig, + addr: SocketAddr, + server_name: &str, + ) -> Result { + let mut endpoint = self.inner.state.lock().unwrap(); + if endpoint.driver_lost || endpoint.recv_state.connections.close.is_some() { + return Err(ConnectError::EndpointStopping); + } + if addr.is_ipv6() && !endpoint.ipv6 { + return Err(ConnectError::InvalidRemoteAddress(addr)); + } + let addr = if endpoint.ipv6 { + SocketAddr::V6(ensure_ipv6(addr)) + } else { + addr + }; + + let (ch, conn) = endpoint + .inner + .connect(self.runtime.now(), config, addr, server_name)?; + + let sender = endpoint.socket.create_sender(); + endpoint.stats.outgoing_handshakes += 1; + Ok(endpoint + .recv_state + .connections + .insert(ch, conn, sender, false, self.runtime.clone())) + } + + /// Switch to a new UDP socket + /// + /// See [`Endpoint::rebind_abstract()`] for details. + #[cfg(not(wasm_browser))] + pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { + self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + } + + /// Switch to a new UDP socket + /// + /// Allows the endpoint's address to be updated live, affecting all active connections. Incoming + /// connections and connections to servers unreachable from the new address will be lost. + /// + /// On error, the old UDP socket is retained. + pub fn rebind_abstract(&self, socket: Box) -> io::Result<()> { + let addr = socket.local_addr()?; + let mut inner = self.inner.state.lock().unwrap(); + inner.prev_socket = Some(mem::replace(&mut inner.socket, socket)); + inner.sender = inner.socket.create_sender(); + inner.ipv6 = addr.is_ipv6(); + + // Update connection socket references + for sender in inner.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Rebind(inner.socket.create_sender())); + } + if let Some(driver) = inner.driver.take() { + // Ensure the driver can register for wake-ups from the new socket + driver.wake(); + } + + Ok(()) + } + + /// Replace the server configuration, affecting new incoming connections only + /// + /// Useful for e.g. refreshing TLS certificates without disrupting existing connections. + pub fn set_server_config(&self, server_config: Option) { + self.inner + .state + .lock() + .unwrap() + .inner + .set_server_config(server_config.map(Arc::new)) + } + + /// Get the local `SocketAddr` the underlying socket is bound to + pub fn local_addr(&self) -> io::Result { + self.inner.state.lock().unwrap().socket.local_addr() + } + + /// Get the number of connections that are currently open + pub fn open_connections(&self) -> usize { + self.inner.state.lock().unwrap().inner.open_connections() + } + + /// Close all of this endpoint's connections immediately and cease accepting new connections. + /// + /// See [`Connection::close()`] for details. + /// + /// [`Connection::close()`]: crate::Connection::close + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + let reason = Bytes::copy_from_slice(reason); + let mut endpoint = self.inner.state.lock().unwrap(); + endpoint.recv_state.connections.close = Some((error_code, reason.clone())); + for sender in endpoint.recv_state.connections.senders.values() { + // Ignoring errors from dropped connections + let _ = sender.send(ConnectionEvent::Close { + error_code, + reason: reason.clone(), + }); + } + self.inner.shared.incoming.notify_waiters(); + } + + /// Wait for all connections on the endpoint to be cleanly shut down + /// + /// Waiting for this condition before exiting ensures that a good-faith effort is made to notify + /// peers of recent connection closes, whereas exiting immediately could force them to wait out + /// the idle timeout period. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + pub async fn wait_idle(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.is_idle() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.idle.notified() + } + .await; + } + } +} + +/// Statistics on [Endpoint] activity +#[non_exhaustive] +#[derive(Debug, Default, Copy, Clone)] +pub struct EndpointStats { + /// Cumulative number of Quic handshakes accepted by this [Endpoint] + pub accepted_handshakes: u64, + /// Cumulative number of Quic handshakes sent from this [Endpoint] + pub outgoing_handshakes: u64, + /// Cumulative number of Quic handshakes refused on this [Endpoint] + pub refused_handshakes: u64, + /// Cumulative number of Quic handshakes ignored on this [Endpoint] + pub ignored_handshakes: u64, +} + +/// A future that drives IO on an endpoint +/// +/// This task functions as the switch point between the UDP socket object and the +/// `Endpoint` responsible for routing datagrams to their owning `Connection`. +/// In order to do so, it also facilitates the exchange of different types of events +/// flowing between the `Endpoint` and the tasks managing `Connection`s. As such, +/// running this task is necessary to keep the endpoint's connections running. +/// +/// `EndpointDriver` futures terminate when all clones of the `Endpoint` have been dropped, or when +/// an I/O error occurs. +#[must_use = "endpoint drivers must be spawned for I/O to occur"] +#[derive(Debug)] +pub(crate) struct EndpointDriver(pub(crate) EndpointRef); + +impl Future for EndpointDriver { + type Output = Result<(), io::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut endpoint = self.0.state.lock().unwrap(); + match endpoint.driver.as_mut() { + Some(old_waker) => old_waker.clone_from(cx.waker()), + None => endpoint.driver = Some(cx.waker().clone()), + } + + let now = endpoint.runtime.now(); + let mut keep_going = false; + keep_going |= endpoint.drive_recv(cx, now)?; + keep_going |= endpoint.handle_events(cx, &self.0.shared); + + if !endpoint.recv_state.incoming.is_empty() { + self.0.shared.incoming.notify_waiters(); + } + + if self.0.shared.ref_count.load(Ordering::Relaxed) == 0 && endpoint.is_idle() { + Poll::Ready(Ok(())) + } else { + drop(endpoint); + // If there is more work to do schedule the endpoint task again. + // `wake_by_ref()` is called outside the lock to minimize + // lock contention on a multithreaded runtime. + if keep_going { + cx.waker().wake_by_ref(); + } + Poll::Pending + } + } +} + +impl Drop for EndpointDriver { + fn drop(&mut self) { + let mut endpoint = self.0.state.lock().unwrap(); + endpoint.driver_lost = true; + self.0.shared.incoming.notify_waiters(); + // Drop all outgoing channels, signaling the termination of the endpoint to the associated + // connections. + endpoint.recv_state.connections.senders.clear(); + // Clearing the senders is what makes the endpoint idle, but a waiter already parked on + // `idle` only re-reads that after being notified, and with the driver gone nothing else + // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. + if endpoint.is_idle() { + self.0.shared.idle.notify_waiters(); + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointInner { + pub(crate) state: Mutex, + pub(crate) shared: Shared, +} + +impl EndpointInner { + pub(crate) fn accept( + &self, + incoming: proto::Incoming, + server_config: Option>, + ) -> Result { + let mut response_buffer = Vec::new(); + + // Phase 1: reserve endpoint state for the connection under the lock. + let accepting = { + let mut state = self.state.lock().unwrap(); + let now = state.runtime.now(); + match state + .inner + .start_accept(incoming, now, &mut response_buffer, server_config) + { + Ok(accepting) => { + state.pending_accepts += 1; + accepting + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + return Err(error.cause); + } + } + }; + + // Phase 2: TLS session setup, connection construction, and first-packet handling, + // without holding the lock. + let accepted = accepting.accept(); + + // Phase 3: register the connection, or release the reservation, under the lock. + let mut state = self.state.lock().unwrap(); + state.pending_accepts -= 1; + let result = match state.inner.finish_accept(accepted, &mut response_buffer) { + Ok((handle, conn)) => { + state.stats.accepted_handshakes += 1; + let sender = state.socket.create_sender(); + let runtime = state.runtime.clone(); + let driver_lost = state.driver_lost; + Ok(state + .recv_state + .connections + .insert(handle, conn, sender, driver_lost, runtime)) + } + Err(error) => { + if let Some(transmit) = error.response { + respond(transmit, &response_buffer, &mut state.sender); + } + Err(error.cause) + } + }; + // Failed accepts and accepts completed after driver loss cannot rely on a Drained event + // being processed to wake idle waiters. + if state.is_idle() { + self.shared.idle.notify_waiters(); + } + result + } + + pub(crate) fn refuse(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.refused_handshakes += 1; + let mut response_buffer = Vec::new(); + let transmit = state.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut state.sender); + } + + pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + let mut state = self.state.lock().unwrap(); + let mut response_buffer = Vec::new(); + let transmit = state.inner.retry(incoming, &mut response_buffer)?; + respond(transmit, &response_buffer, &mut state.sender); + Ok(()) + } + + pub(crate) fn ignore(&self, incoming: proto::Incoming) { + let mut state = self.state.lock().unwrap(); + state.stats.ignored_handshakes += 1; + state.inner.ignore(incoming); + } +} + +#[derive(Debug)] +pub(crate) struct State { + socket: Box, + sender: Pin>, + /// During an active migration, abandoned_socket receives traffic + /// until the first packet arrives on the new socket. + prev_socket: Option>, + inner: proto::Endpoint, + recv_state: RecvState, + driver: Option, + ipv6: bool, + events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, + driver_lost: bool, + runtime: Arc, + stats: EndpointStats, + default_client_config: Option, + /// Connections in the process of being accepted + pending_accepts: usize, +} + +#[derive(Debug)] +pub(crate) struct Shared { + incoming: Notify, + idle: Notify, + /// Number of live handles that can be used to initiate or handle I/O; excludes the driver + ref_count: AtomicUsize, +} + +impl State { + fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { + let get_time = || self.runtime.now(); + self.recv_state.recv_limiter.start_cycle(get_time); + if let Some(socket) = &mut self.prev_socket { + // We don't care about the `PollProgress` from old sockets. + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut **socket, + &mut self.sender, + &*self.runtime, + now, + ); + if poll_res.is_err() { + self.prev_socket = None; + } + }; + let poll_res = self.recv_state.poll_socket( + cx, + &mut self.inner, + &mut *self.socket, + &mut self.sender, + &*self.runtime, + now, + ); + self.recv_state.recv_limiter.finish_cycle(get_time); + let poll_res = poll_res?; + if poll_res.received_connection_packet { + // Traffic has arrived on self.socket, therefore there is no need for the abandoned + // one anymore. TODO: Account for multiple outgoing connections. + self.prev_socket = None; + } + Ok(poll_res.keep_going) + } + + fn handle_events(&mut self, cx: &mut Context<'_>, shared: &Shared) -> bool { + for _ in 0..IO_LOOP_BOUND { + let (ch, event) = match self.events.poll_recv(cx) { + Poll::Ready(Some(x)) => x, + Poll::Ready(None) => unreachable!("EndpointInner owns one sender"), + Poll::Pending => { + return false; + } + }; + + if event.is_drained() { + self.recv_state.connections.senders.remove(&ch); + if self.is_idle() { + shared.idle.notify_waiters(); + } + } + let Some(event) = self.inner.handle_event(ch, event) else { + continue; + }; + // Ignoring errors from dropped connections that haven't yet been cleaned up + let _ = self + .recv_state + .connections + .senders + .get_mut(&ch) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + + true + } + + fn is_idle(&self) -> bool { + self.recv_state.connections.is_empty() && self.pending_accepts == 0 + } +} + +impl Drop for State { + fn drop(&mut self) { + for incoming in self.recv_state.incoming.drain(..) { + self.inner.ignore(incoming); + } + } +} + +fn respond( + transmit: proto::Transmit, + response_buffer: &[u8], + sender: &mut Pin>, +) { + // Send if there's kernel buffer space; otherwise, drop it + // + // As an endpoint-generated packet, we know this is an + // immediate, stateless response to an unconnected peer, + // one of: + // + // - A version negotiation response due to an unknown version + // - A `CLOSE` due to a malformed or unwanted connection attempt + // - A stateless reset due to an unrecognized connection + // - A `Retry` packet due to a connection attempt when + // `use_retry` is set + // + // In each case, a well-behaved peer can be trusted to retry a + // few times, which is guaranteed to produce the same response + // from us. Repeated failures might at worst cause a peer's new + // connection attempt to time out, which is acceptable if we're + // under such heavy load that there's never room for this code + // to transmit. This is morally equivalent to the packet getting + // lost due to congestion further along the link, which + // similarly relies on peer retries for recovery. + + // Copied from rust 1.85's std::task::Waker::noop() implementation for backwards compatibility + const NOOP: RawWaker = { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + // Cloning just returns a new no-op raw waker + |_| NOOP, + // `wake` does nothing + |_| {}, + // `wake_by_ref` does nothing + |_| {}, + // Dropping does nothing as we don't allocate anything + |_| {}, + ); + RawWaker::new(std::ptr::null(), &VTABLE) + }; + // SAFETY: Copied from rust stdlib, the NOOP waker is thread-safe and doesn't violate the RawWakerVTable contract, + // it doesn't access the data pointer at all. + let waker = unsafe { Waker::from_raw(NOOP) }; + let mut cx = Context::from_waker(&waker); + _ = sender.as_mut().poll_send( + &udp_transmit(&transmit, &response_buffer[..transmit.size]), + &mut cx, + ); +} + +#[inline] +fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { + match ecn { + udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, + udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, + udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + } +} + +#[derive(Debug)] +struct ConnectionSet { + /// Senders for communicating with the endpoint's connections + senders: FxHashMap>, + /// Stored to give out clones to new ConnectionInners + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + /// Set if the endpoint has been manually closed + close: Option<(VarInt, Bytes)>, +} + +impl ConnectionSet { + fn insert( + &mut self, + handle: ConnectionHandle, + conn: proto::Connection, + sender: Pin>, + driver_lost: bool, + runtime: Arc, + ) -> Connecting { + let (send, recv) = mpsc::unbounded_channel(); + if let Some((error_code, reason)) = &self.close { + send.send(ConnectionEvent::Close { + error_code: *error_code, + reason: reason.clone(), + }) + .unwrap(); + } + match driver_lost { + // Close the event channel before spawning the connection driver, so it cannot + // transmit a handshake after the endpoint driver has stopped. + true => drop(send), + false => { + self.senders.insert(handle, send); + } + } + Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + } + + fn is_empty(&self) -> bool { + self.senders.is_empty() + } +} + +fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { + match x { + SocketAddr::V6(x) => x, + SocketAddr::V4(x) => SocketAddrV6::new(x.ip().to_ipv6_mapped(), x.port(), 0, 0), + } +} + +pin_project! { + /// Future produced by [`Endpoint::accept`] + pub struct Accept<'a> { + endpoint: &'a Endpoint, + #[pin] + notify: Notified<'a>, + } +} + +impl Future for Accept<'_> { + type Output = Option; + fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + let mut endpoint = this.endpoint.inner.state.lock().unwrap(); + if endpoint.driver_lost { + return Poll::Ready(None); + } + if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // Release the mutex lock on endpoint so cloning it doesn't deadlock + drop(endpoint); + let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); + return Poll::Ready(Some(incoming)); + } + if endpoint.recv_state.connections.close.is_some() { + return Poll::Ready(None); + } + loop { + match this.notify.as_mut().poll(ctx) { + // `state` lock ensures we didn't race with readiness + Poll::Pending => return Poll::Pending, + // Spurious wakeup, get a new future + Poll::Ready(()) => this + .notify + .set(this.endpoint.inner.shared.incoming.notified()), + } + } + } +} + +#[derive(Debug)] +pub(crate) struct EndpointRef(Arc); + +impl EndpointRef { + pub(crate) fn new( + socket: Box, + inner: proto::Endpoint, + ipv6: bool, + runtime: Arc, + ) -> Self { + let (sender, events) = mpsc::unbounded_channel(); + let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); + let sender = socket.create_sender(); + Self(Arc::new(EndpointInner { + shared: Shared { + incoming: Notify::new(), + idle: Notify::new(), + ref_count: AtomicUsize::new(0), + }, + state: Mutex::new(State { + socket, + sender, + prev_socket: None, + inner, + ipv6, + events, + driver: None, + driver_lost: false, + recv_state, + runtime, + stats: EndpointStats::default(), + default_client_config: None, + pending_accepts: 0, + }), + })) + } +} + +impl Clone for EndpointRef { + fn clone(&self) -> Self { + self.0.shared.ref_count.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } +} + +impl Drop for EndpointRef { + fn drop(&mut self) { + if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 { + return; + } + + let endpoint = &mut *self.0.state.lock().unwrap(); + // If the driver is about to be on its own, ensure it can shut down if the last + // connection is gone. + if let Some(task) = endpoint.driver.take() { + task.wake(); + } + } +} + +impl std::ops::Deref for EndpointRef { + type Target = EndpointInner; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// State directly involved in handling incoming packets +struct RecvState { + incoming: VecDeque, + connections: ConnectionSet, + recv_buf: Box<[u8]>, + /// Reused buffer that received datagrams are copied into and split into owned slices. + datagrams: BytesMut, + recv_limiter: WorkLimiter, +} + +impl RecvState { + fn new( + sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, + max_receive_segments: usize, + endpoint: &proto::Endpoint, + ) -> Self { + let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) + as usize + * max_receive_segments; + let recv_buf = vec![0; datagram_capacity * BATCH_SIZE]; + Self { + connections: ConnectionSet { + senders: FxHashMap::default(), + sender, + close: None, + }, + incoming: VecDeque::new(), + recv_buf: recv_buf.into(), + datagrams: BytesMut::with_capacity(datagram_capacity), + recv_limiter: WorkLimiter::new(RECV_TIME_BOUND), + } + } + + fn poll_socket( + &mut self, + cx: &mut Context<'_>, + endpoint: &mut proto::Endpoint, + socket: &mut dyn AsyncUdpSocket, + sender: &mut Pin>, + runtime: &dyn Runtime, + now: Instant, + ) -> Result { + let mut received_connection_packet = false; + let mut metas = [RecvMeta::default(); BATCH_SIZE]; + let mut iovs: [IoSliceMut<'_>; BATCH_SIZE] = { + let mut bufs = self + .recv_buf + .chunks_mut(self.recv_buf.len() / BATCH_SIZE) + .map(IoSliceMut::new); + + // expect() safe as self.recv_buf is chunked into BATCH_SIZE items + // and iovs will be of size BATCH_SIZE, thus from_fn is called + // exactly BATCH_SIZE times. + std::array::from_fn(|_| bufs.next().expect("BATCH_SIZE elements")) + }; + loop { + match socket.poll_recv(cx, &mut iovs, &mut metas) { + Poll::Ready(Ok(msgs)) => { + self.recv_limiter.record_work(msgs); + // Copy the received batch into `datagrams` and split each datagram off as + // an owned slice. We can't read directly into `datagrams` and drop + // `recv_buf`: once a slice has been handed to a connection the allocation + // is shared, so the next `reserve` must allocate fresh backing memory. + // Reading in place would therefore mean reserving the worst-case iovec + // envelope before every poll and reallocating it whenever a prior datagram + // is still in flight. `recv_buf` stays an allocate-once scratch buffer, and + // we reserve only the bytes actually received. + let batch_len = metas.iter().take(msgs).map(|meta| meta.len).sum(); + self.datagrams.reserve(batch_len); + for (meta, buf) in metas.iter().zip(iovs.iter()).take(msgs) { + self.datagrams.extend_from_slice(&buf[..meta.len]); + } + for meta in metas.iter().take(msgs) { + let mut data = self.datagrams.split_to(meta.len); + while !data.is_empty() { + let buf = data.split_to(meta.stride.min(data.len())); + let mut response_buffer = Vec::new(); + match endpoint.handle( + now, + meta.addr, + meta.dst_ip, + meta.ecn.map(proto_ecn), + buf, + &mut response_buffer, + ) { + Some(DatagramEvent::NewConnection(incoming)) => { + if self.connections.close.is_none() { + self.incoming.push_back(incoming); + } else { + let transmit = + endpoint.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, sender); + } + } + Some(DatagramEvent::ConnectionEvent(handle, event)) => { + // Ignoring errors from dropped connections that haven't yet been cleaned up + received_connection_packet = true; + let _ = self + .connections + .senders + .get_mut(&handle) + .unwrap() + .send(ConnectionEvent::Proto(event)); + } + Some(DatagramEvent::Response(transmit)) => { + respond(transmit, &response_buffer, sender); + } + None => {} + } + } + } + } + Poll::Pending => { + return Ok(PollProgress { + received_connection_packet, + keep_going: false, + }); + } + // Ignore ECONNRESET as it's undefined in QUIC and may be injected by an + // attacker + Poll::Ready(Err(e)) if e.kind() == io::ErrorKind::ConnectionReset => { + continue; + } + // Ignore EMSGSIZE as we're currently not handling ICMPv4 Fragmentation Needed + // and ICMPv6 Packet Too Big (PTB) messages since they cannot be authenticated, + // and Datagram Packetization Layer Path MTU Discovery (DPLPMTUD) works without + // it anyways. + Poll::Ready(Err(e)) if is_msg_size_err(&e) => { + continue; + } + Poll::Ready(Err(e)) => { + return Err(e); + } + } + if !self.recv_limiter.allow_work(|| runtime.now()) { + return Ok(PollProgress { + received_connection_packet, + keep_going: true, + }); + } + } + } +} + +impl fmt::Debug for RecvState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RecvState") + .field("incoming", &self.incoming) + .field("connections", &self.connections) + // recv_buf too large + .field("recv_limiter", &self.recv_limiter) + .finish_non_exhaustive() + } +} + +#[derive(Default)] +struct PollProgress { + /// Whether a datagram was routed to an existing connection + received_connection_packet: bool, + /// Whether datagram handling was interrupted early by the work limiter for fairness + keep_going: bool, +} diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs new file mode 100644 index 0000000000..b5d60a08ec --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -0,0 +1,152 @@ +use std::{ + future::{Future, IntoFuture}, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use proto::{ConnectionError, ConnectionId, ServerConfig}; +use thiserror::Error; + +use crate::{ + connection::{Connecting, Connection}, + endpoint::EndpointRef, +}; + +/// An incoming connection for which the server has not yet begun its part of the handshake +#[derive(Debug)] +pub struct Incoming(Option); + +impl Incoming { + pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + Self(Some(State { inner, endpoint })) + } + + /// Attempt to accept this incoming connection (an error may still occur) + pub fn accept(mut self) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, None) + } + + /// Accept this incoming connection using a custom configuration + /// + /// See [`accept()`][Incoming::accept] for more details. + pub fn accept_with( + mut self, + server_config: Arc, + ) -> Result { + let state = self.0.take().unwrap(); + state.endpoint.accept(state.inner, Some(server_config)) + } + + /// Reject this incoming connection attempt + pub fn refuse(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.refuse(state.inner); + } + + /// Respond with a retry packet, requiring the client to retry with address validation + /// + /// Errors if `may_retry()` is false. + pub fn retry(mut self) -> Result<(), RetryError> { + let state = self.0.take().unwrap(); + state.endpoint.retry(state.inner).map_err(|e| { + RetryError(Box::new(Self(Some(State { + inner: e.into_incoming(), + endpoint: state.endpoint, + })))) + }) + } + + /// Ignore this incoming connection attempt, not sending any packet in response + pub fn ignore(mut self) { + let state = self.0.take().unwrap(); + state.endpoint.ignore(state.inner); + } + + /// The local IP address which was used when the peer established the connection + pub fn local_ip(&self) -> Option { + self.0.as_ref().unwrap().inner.local_ip() + } + + /// The peer's UDP address + pub fn remote_address(&self) -> SocketAddr { + self.0.as_ref().unwrap().inner.remote_address() + } + + /// Whether the socket address that is initiating this connection has been validated + /// + /// This means that the sender of the initial packet has proved that they can receive traffic + /// sent to `self.remote_address()`. + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn remote_address_validated(&self) -> bool { + self.0.as_ref().unwrap().inner.remote_address_validated() + } + + /// Whether it is legal to respond with a retry packet + /// + /// If `self.remote_address_validated()` is false, `self.may_retry()` is guaranteed to be true. + /// The inverse is not guaranteed. + pub fn may_retry(&self) -> bool { + self.0.as_ref().unwrap().inner.may_retry() + } + + /// The original destination CID when initiating the connection + pub fn orig_dst_cid(&self) -> ConnectionId { + self.0.as_ref().unwrap().inner.orig_dst_cid() + } +} + +impl Drop for Incoming { + fn drop(&mut self) { + // Implicit reject, similar to Connection's implicit close + if let Some(state) = self.0.take() { + state.endpoint.refuse(state.inner); + } + } +} + +#[derive(Debug)] +struct State { + inner: proto::Incoming, + endpoint: EndpointRef, +} + +/// Error for attempting to retry an [`Incoming`] which already bears a token from a previous retry +#[derive(Debug, Error)] +#[error("retry() with validated Incoming")] +pub struct RetryError(Box); + +impl RetryError { + /// Get the [`Incoming`] + pub fn into_incoming(self) -> Incoming { + *self.0 + } +} + +/// Basic adapter to let [`Incoming`] be `await`-ed like a [`Connecting`] +#[derive(Debug)] +pub struct IncomingFuture(Result); + +impl Future for IncomingFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match &mut self.0 { + Ok(connecting) => Pin::new(connecting).poll(cx), + Err(e) => Poll::Ready(Err(e.clone())), + } + } +} + +impl IntoFuture for Incoming { + type Output = Result; + type IntoFuture = IncomingFuture; + + fn into_future(self) -> Self::IntoFuture { + IncomingFuture(self.accept()) + } +} diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs new file mode 100644 index 0000000000..649801fff8 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -0,0 +1,134 @@ +//! QUIC transport protocol implementation +//! +//! [QUIC](https://en.wikipedia.org/wiki/QUIC) is a modern transport protocol addressing +//! shortcomings of TCP, such as head-of-line blocking, poor security, slow handshakes, and +//! inefficient congestion control. This crate provides a portable userspace implementation. It +//! builds on top of quinn-proto, which implements protocol logic independent of any particular +//! runtime. +//! +//! The entry point of this crate is the [`Endpoint`]. +//! +//! # About QUIC +//! +//! A QUIC connection is an association between two endpoints. The endpoint which initiates the +//! connection is termed the client, and the endpoint which accepts it is termed the server. A +//! single endpoint may function as both client and server for different connections, for example +//! in a peer-to-peer application. To communicate application data, each endpoint may open streams +//! up to a limit dictated by its peer. Typically, that limit is increased as old streams are +//! finished. +//! +//! Streams may be unidirectional or bidirectional, and are cheap to create and disposable. For +//! example, a traditionally datagram-oriented application could use a new stream for every +//! message it wants to send, no longer needing to worry about MTUs. Bidirectional streams behave +//! much like a traditional TCP connection, and are useful for sending messages that have an +//! immediate response, such as an HTTP request. Stream data is delivered reliably, and there is no +//! ordering enforced between data on different streams. +//! +//! By avoiding head-of-line blocking and providing unified congestion control across all streams +//! of a connection, QUIC is able to provide higher throughput and lower latency than one or +//! multiple TCP connections between the same two hosts, while providing more useful behavior than +//! raw UDP sockets. +//! +//! Quinn also exposes unreliable datagrams, which are a low-level primitive preferred when +//! automatic fragmentation and retransmission of certain data is not desired. +//! +//! QUIC uses encryption and identity verification built directly on TLS 1.3. Just as with a TLS +//! server, it is useful for a QUIC server to be identified by a certificate signed by a trusted +//! authority. If this is infeasible--for example, if servers are short-lived or not associated +//! with a domain name--then as with TLS, self-signed certificates can be used to provide +//! encryption alone. +#![warn(missing_docs)] + +use std::pin::Pin; + +mod connection; +mod endpoint; +mod incoming; +mod mutex; +mod recv_stream; +mod runtime; +mod send_stream; +mod work_limiter; + +#[cfg(not(wasm_browser))] +pub(crate) use std::time::{Duration, Instant}; +#[cfg(wasm_browser)] +pub(crate) use web_time::{Duration, Instant}; + +#[cfg(feature = "bloom")] +pub use proto::BloomTokenLog; +pub use proto::{ + AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, + ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, + ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, + InvalidCid, MtuDiscoveryConfig, NoneTokenLog, NoneTokenStore, PathStats, ServerConfig, Side, + StdSystemTime, StreamId, TimeSource, TokenLog, TokenMemoryCache, TokenReuseError, TokenStore, + Transmit, TransportConfig, TransportErrorCode, UdpStats, ValidationTokenConfig, VarInt, + VarIntBoundsExceeded, Written, congestion, crypto, +}; +#[cfg(feature = "qlog")] +pub use proto::{QlogConfig, QlogStream}; +#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +pub use rustls; +pub use udp; + +pub use crate::connection::{ + AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, + RemoteAddressWatcher, SendDatagram, SendDatagramError, +}; +pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; +#[cfg(feature = "runtime-smol")] +pub use crate::runtime::SmolRuntime; +#[cfg(feature = "runtime-tokio")] +pub use crate::runtime::TokioRuntime; +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +pub use crate::runtime::default_runtime; +pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; +pub use crate::send_stream::{SendStream, StoppedError, WriteError}; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +enum ConnectionEvent { + Close { + error_code: VarInt, + reason: bytes::Bytes, + }, + Proto(proto::ConnectionEvent), + Rebind(Pin>), +} + +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { + udp::Transmit { + destination: t.destination, + ecn: t.ecn.map(udp_ecn), + contents: buffer, + segment_size: t.segment_size, + src_ip: t.src_ip, + } +} + +fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { + match ecn { + EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + } +} + +/// Maximum number of datagrams processed in send/recv calls to make before moving on to other processing +/// +/// This helps ensure we don't starve anything when the CPU is slower than the link. +/// Value is selected by picking a low number which didn't degrade throughput in benchmarks. +const IO_LOOP_BOUND: usize = 160; + +/// The maximum amount of time that should be spent in `recvmsg()` calls per endpoint iteration +/// +/// 50us are chosen so that an endpoint iteration with a 50us sendmsg limit blocks +/// the runtime for a maximum of about 100us. +/// Going much lower does not yield any noticeable difference, since a single `recvmmsg` +/// batch of size 32 was observed to take 30us on some systems. +const RECV_TIME_BOUND: Duration = Duration::from_micros(50); diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs new file mode 100644 index 0000000000..91962cb072 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -0,0 +1,163 @@ +use std::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +#[cfg(feature = "lock_tracking")] +mod tracking { + use super::*; + use crate::{Duration, Instant}; + use std::collections::VecDeque; + use tracing::warn; + + #[derive(Debug)] + struct Inner { + last_lock_owner: VecDeque<(&'static str, Duration)>, + value: T, + } + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + pub(crate) struct Mutex { + inner: std::sync::Mutex>, + } + + impl Debug for Mutex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Debug::fmt(&self.inner, f) + } + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(Inner { + last_lock_owner: VecDeque::new(), + value, + }), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { + // We don't bother dispatching through Runtime::now because they're pure performance + // diagnostics. + let now = Instant::now(); + let guard = self.inner.lock().unwrap(); + + let lock_time = Instant::now(); + let elapsed = lock_time.duration_since(now); + + if elapsed > Duration::from_millis(1) { + warn!( + "Locking the connection for {} took {:?}. Last owners: {:?}", + purpose, elapsed, guard.last_lock_owner + ); + } + + MutexGuard { + guard, + start_time: lock_time, + purpose, + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, Inner>, + start_time: Instant, + purpose: &'static str, + } + + impl Drop for MutexGuard<'_, T> { + fn drop(&mut self) { + if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { + self.guard.last_lock_owner.pop_back(); + } + + let duration = self.start_time.elapsed(); + + if duration > Duration::from_millis(1) { + warn!( + "Utilizing the connection for {} took {:?}", + self.purpose, duration + ); + } + + self.guard + .last_lock_owner + .push_front((self.purpose, duration)); + } + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard.value + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard.value + } + } + + const MAX_LOCK_OWNERS: usize = 20; +} + +#[cfg(feature = "lock_tracking")] +pub(crate) use tracking::Mutex; + +#[cfg(not(feature = "lock_tracking"))] +mod non_tracking { + use super::*; + + /// A Mutex which optionally allows to track the time a lock was held and + /// emit warnings in case of excessive lock times + #[derive(Debug)] + pub(crate) struct Mutex { + inner: std::sync::Mutex, + } + + impl Mutex { + pub(crate) fn new(value: T) -> Self { + Self { + inner: std::sync::Mutex::new(value), + } + } + + /// Acquires the lock for a certain purpose + /// + /// The purpose will be recorded in the list of last lock owners + pub(crate) fn lock(&self, _purpose: &'static str) -> MutexGuard<'_, T> { + MutexGuard { + guard: self.inner.lock().unwrap(), + } + } + } + + pub(crate) struct MutexGuard<'a, T> { + guard: std::sync::MutexGuard<'a, T>, + } + + impl Deref for MutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.guard.deref() + } + } + + impl DerefMut for MutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.deref_mut() + } + } +} + +#[cfg(not(feature = "lock_tracking"))] +pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs new file mode 100644 index 0000000000..d3548a1a33 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -0,0 +1,823 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use bytes::Bytes; +use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use thiserror::Error; +use tokio::io::ReadBuf; + +use crate::{VarInt, connection::ConnectionRef}; + +/// A stream that can only be used to receive data +/// +/// `stop(0)` is implicitly called on drop unless: +/// - A variant of [`ReadError`] has been yielded by a read call +/// - [`stop()`] was called explicitly +/// +/// # Cancellation +/// +/// A `read` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready cannot lead to loss of stream data. This is true of methods which succeed immediately when +/// any progress is made, and is not true of methods which might need to perform multiple reads +/// internally before succeeding. Each `read` method documents whether it is cancel-safe. +/// +/// # Common issues +/// +/// ## Data never received on a locally-opened stream +/// +/// Peers are not notified of streams until they or a later-numbered stream are used to send +/// data. If a bidirectional stream is locally opened but never used to send, then the peer may +/// never see it. Application protocols should always arrange for the endpoint which will first +/// transmit on a stream to be the endpoint responsible for opening it. +/// +/// ## Data never received on a remotely-opened stream +/// +/// Verify that the stream you are receiving is the same one that the server is sending on, e.g. by +/// logging the [`id`] of each. Streams are always accepted in the same order as they are created, +/// i.e. ascending order by [`StreamId`]. For example, even if a sender first transmits on +/// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver +/// will be bidirectional stream 0. +/// +/// [`ReadError`]: crate::ReadError +/// [`stop()`]: RecvStream::stop +/// [`SendStream::finish`]: crate::SendStream::finish +/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`id`]: RecvStream::id +/// [`Connection::accept_bi`]: crate::Connection::accept_bi +#[derive(Debug)] +pub struct RecvStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, + all_data_read: bool, + reset: Option, +} + +impl RecvStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + all_data_read: false, + reset: None, + } + } + + /// Read data contiguously from the stream. + /// + /// Yields the number of bytes read into `buf` on success, or `None` if the stream was finished. + /// + /// This operation is cancel-safe. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + Read { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Read an exact number of bytes contiguously from the stream. + /// + /// See [`read()`] for details. This operation is *not* cancel-safe. + /// + /// [`read()`]: RecvStream::read + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + ReadExact { + stream: self, + buf: ReadBuf::new(buf), + } + .await + } + + /// Attempts to read from the stream into the provided buffer + /// + /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this + /// returns zero bytes read (and `buf` has a non-zero length), that indicates that the remote + /// side has [`finish`]ed the stream and the local side has already read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read( + &mut self, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(self.poll_read_buf(cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } + + /// Attempts to read from the stream into the provided buffer, which may be uninitialized + /// + /// On success, returns `Poll::Ready(Ok(()))` and places data into the unfilled portion of + /// `buf`. If this does not write any bytes to `buf` (and `buf.remaining()` is non-zero), that + /// indicates that the remote side has [`finish`]ed the stream and the local side has already + /// read all bytes. + /// + /// If no data is available for reading, this returns `Poll::Pending` and arranges for the + /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is + /// closed. + /// + /// [`finish`]: crate::SendStream::finish + pub fn poll_read_buf( + &mut self, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = false; + loop { + if buf.remaining() == 0 { + // We know `read` is `true` because `buf.remaining()` was not 0 before + return ReadStatus::Readable(()); + } + + match chunks.next(buf.remaining()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read = true; + } + res => return (if read { Some(()) } else { None }, res.err()).into(), + } + } + }) + .map(|res| res.map(|_| ())) + } + + /// Read the next segment of data + /// + /// Yields `None` if the stream was finished. Otherwise, yields a segment of data and its + /// offset in the stream. If `ordered` is `true`, the chunk's offset will be immediately after + /// the last data yielded by `read()` or `read_chunk()`. If `ordered` is `false`, segments may + /// be received in any order, and the `Chunk`'s `offset` field can be used to determine + /// ordering in the caller. Unordered reads are less prone to head-of-line blocking within a + /// stream, but require the application to manage reassembling the original data. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries do not correspond + /// to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunk( + &mut self, + max_length: usize, + ordered: bool, + ) -> Result, ReadError> { + ReadChunk { + stream: self, + max_length, + ordered, + } + .await + } + + /// Attempts to read a chunk from the stream. + /// + /// On success, returns `Poll::Ready(Ok(Some(chunk)))`. If `Poll::Ready(Ok(None))` + /// is returned, it implies that EOF has been reached. + /// + /// If no data is available for reading, the method returns `Poll::Pending` + /// and arranges for the current task (via cx.waker()) to receive a notification + /// when the stream becomes readable or is closed. + fn poll_read_chunk( + &mut self, + cx: &mut Context<'_>, + max_length: usize, + ordered: bool, + ) -> Poll, ReadError>> { + self.poll_read_generic(cx, ordered, |chunks| match chunks.next(max_length) { + Ok(Some(chunk)) => ReadStatus::Readable(chunk), + res => (None, res.err()).into(), + }) + } + + /// Read the next segments of data + /// + /// Fills `bufs` with the segments of data beginning immediately after the + /// last data yielded by `read` or `read_chunk`, or `None` if the stream was + /// finished. + /// + /// Slightly more efficient than `read` due to not copying. Chunk boundaries + /// do not correspond to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunks(&mut self, bufs: &mut [Bytes]) -> Result, ReadError> { + ReadChunks { stream: self, bufs }.await + } + + /// Foundation of [`Self::read_chunks`] + fn poll_read_chunks( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [Bytes], + ) -> Poll, ReadError>> { + if bufs.is_empty() { + return Poll::Ready(Ok(Some(0))); + } + + self.poll_read_generic(cx, true, |chunks| { + let mut read = 0; + loop { + if read >= bufs.len() { + // We know `read > 0` because `bufs` cannot be empty here + return ReadStatus::Readable(read); + } + + match chunks.next(usize::MAX) { + Ok(Some(chunk)) => { + bufs[read] = chunk.bytes; + read += 1; + } + res => return (if read == 0 { None } else { Some(read) }, res.err()).into(), + } + } + }) + } + + /// Convenience method to read all remaining data into a buffer + /// + /// Fails with [`ReadToEndError::TooLong`] on reading more than `size_limit` bytes, discarding + /// all data read. Uses unordered reads to be more efficient than using `AsyncRead` would + /// allow. `size_limit` should be set to limit worst-case memory use. + /// + /// If unordered reads have already been made, the resulting buffer may have gaps containing + /// arbitrary data. + /// + /// This operation is *not* cancel-safe. + /// + /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + ReadToEnd { + stream: self, + buffer: ReadToEndBuffer::new(size_limit), + } + .await + } + + /// Stop accepting data + /// + /// Discards unread data and notifies the peer to stop transmitting. Once stopped, further + /// attempts to operate on a stream will yield `ClosedStream` errors. + pub fn stop(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("RecvStream::stop"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.recv_stream(self.stream).stop(error_code)?; + conn.wake(); + self.all_data_read = true; + // Clean up shared state that might be left over from a cancalled read + // operation, so `drop` doesn't have to + conn.blocked_readers.remove(&self.stream); + Ok(()) + } + + /// Check if this stream predates completion of the handshake on an incoming connection. + /// + /// True only if the stream was accepted before the handshake completed, which is only possible + /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// client chose to send 0-RTT data. + /// + /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application + /// data may be a replay attack. To guard against this, applications should not execute + /// non-idempotent operations until + /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + pub fn is_0rtt(&self) -> bool { + self.is_0rtt + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Completes when the stream has been reset by the peer or otherwise closed + /// + /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` + /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was + /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// which it is no longer meaningful for the stream to be reset. + /// + /// This operation is cancel-safe. + pub async fn received_reset(&mut self) -> Result, ResetError> { + poll_fn(|cx| { + let mut conn = self.conn.state.lock("RecvStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Poll::Ready(Err(ResetError::ZeroRttRejected)); + } + + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + + match conn.inner.recv_stream(self.stream).received_reset() { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(error_code)) => { + // Stream state has just now been freed, so the connection may need to issue new + // stream ID flow control credit + conn.wake(); + Poll::Ready(Ok(Some(error_code))) + } + Ok(None) => { + if let Some(e) = &conn.error { + return Poll::Ready(Err(e.clone().into())); + } + // Resets always notify readers, since a reset is an immediate read error. We + // could introduce a dedicated channel to reduce the risk of spurious wakeups, + // but that increased complexity is probably not justified, as an application + // that is expecting a reset is not likely to receive large amounts of data. + conn.blocked_readers.insert(self.stream, cx.waker().clone()); + Poll::Pending + } + } + }) + .await + } + + /// Handle common logic related to reading out of a receive stream + /// + /// This takes an `FnMut` closure that takes care of the actual reading process, matching + /// the detailed read semantics for the calling function with a particular return type. + /// The closure can read from the passed `&mut Chunks` and has to return the status after + /// reading: the amount of data read, and the status after the final read call. + fn poll_read_generic( + &mut self, + cx: &mut Context<'_>, + ordered: bool, + mut read_fn: T, + ) -> Poll, ReadError>> + where + T: FnMut(&mut Chunks<'_>) -> ReadStatus, + { + use proto::ReadError::*; + use std::collections::hash_map::Entry; + + if self.all_data_read { + return Poll::Ready(Ok(None)); + } + + let mut conn = self.conn.state.lock("RecvStream::poll_read"); + if self.is_0rtt { + conn.check_0rtt().map_err(|()| ReadError::ZeroRttRejected)?; + } + + // If we stored an error during a previous call, return it now. This can happen if a + // `read_fn` both wants to return data and also returns an error in its final stream status. + let status = match self.reset { + Some(code) => ReadStatus::Failed(None, Reset(code)), + None => { + let mut recv = conn.inner.recv_stream(self.stream); + let mut chunks = recv.read(ordered)?; + let status = read_fn(&mut chunks); + if chunks.finalize().should_transmit() { + conn.wake(); + } + status + } + }; + + match status { + ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), + ReadStatus::Finished(read) => { + self.all_data_read = true; + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Ok(read)) + } + ReadStatus::Failed(read, Blocked) => match read { + Some(val) => Poll::Ready(Ok(Some(val))), + None => { + if let Some(x) = &conn.error { + return Poll::Ready(Err(ReadError::ConnectionLost(x.clone()))); + } + match conn.blocked_readers.entry(self.stream) { + Entry::Occupied(mut entry) => { + entry.get_mut().clone_from(cx.waker()); + } + Entry::Vacant(entry) => { + entry.insert(cx.waker().clone()); + } + } + Poll::Pending + } + }, + ReadStatus::Failed(read, Reset(error_code)) => match read { + None => { + self.all_data_read = true; + self.reset = Some(error_code); + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.stream); + Poll::Ready(Err(ReadError::Reset(error_code))) + } + done => { + self.reset = Some(error_code); + Poll::Ready(Ok(done)) + } + }, + } + } +} + +enum ReadStatus { + Readable(T), + Finished(Option), + Failed(Option, proto::ReadError), +} + +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { + match status { + (read, None) => Self::Finished(read), + (read, Some(e)) => Self::Failed(read, e), + } + } +} + +/// Future produced by [`RecvStream::read_to_end()`]. +/// +/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +struct ReadToEnd<'a> { + stream: &'a mut RecvStream, + buffer: ReadToEndBuffer, +} + +impl Future for ReadToEnd<'_> { + type Output = Result, ReadToEndError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + loop { + match ready!(self.stream.poll_read_chunk(cx, usize::MAX, false))? { + Some(chunk) => self.buffer.push(chunk)?, + None => return Poll::Ready(Ok(self.buffer.finish())), + } + } + } +} + +struct ReadToEndBuffer { + read: Vec<(Bytes, u64)>, + start: u64, + end: u64, + size_limit: usize, +} + +impl ReadToEndBuffer { + fn new(size_limit: usize) -> Self { + Self { + read: Vec::new(), + start: u64::MAX, + end: 0, + size_limit, + } + } + + fn push(&mut self, chunk: Chunk) -> Result<(), ReadToEndError> { + self.start = self.start.min(chunk.offset); + self.end = self.end.max(chunk.bytes.len() as u64 + chunk.offset); + if (self.end - self.start) > self.size_limit as u64 { + return Err(ReadToEndError::TooLong); + } + self.read.push((chunk.bytes, chunk.offset)); + Ok(()) + } + + fn finish(&mut self) -> Vec { + if self.end == 0 { + return Vec::new(); + } + let mut buffer = vec![0; (self.end - self.start) as usize]; + for (data, offset) in self.read.drain(..) { + let offset = (offset - self.start) as usize; + buffer[offset..offset + data.len()].copy_from_slice(&data); + } + buffer + } +} + +/// Errors from [`RecvStream::read_to_end`] +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadToEndError { + /// An error occurred during reading + #[error("read error: {0}")] + Read(#[from] ReadError), + /// The stream is larger than the user-supplied limit + #[error("stream too long")] + TooLong, +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let mut buf = ReadBuf::new(buf); + ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; + Poll::Ready(Ok(buf.filled().len())) + } +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + ready!(Self::poll_read_buf(self.get_mut(), cx, buf))?; + Poll::Ready(Ok(())) + } +} + +impl Drop for RecvStream { + fn drop(&mut self) { + if self.all_data_read { + debug_assert!( + !self + .conn + .state + .lock("RecvStream:drop") + .blocked_readers + .contains_key(&self.stream), + "Stream {} should not have a blocked reader when all data read is true", + self.stream + ); + return; + } + let mut conn = self.conn.state.lock("RecvStream::drop"); + + // clean up any previously registered wakers + conn.blocked_readers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + + // Ignore ClosedStream errors + let _ = conn.inner.recv_stream(self.stream).stop(0u32.into()); + conn.wake(); + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadError { + /// The peer abandoned transmitting data on this stream + /// + /// Carries an application-defined error code. + #[error("stream reset by peer: error {0}")] + Reset(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been stopped, finished, or reset + #[error("closed stream")] + ClosedStream, + /// Attempted an ordered read following an unordered read + /// + /// Performing an unordered read allows discontinuities to arise in the receive buffer of a + /// stream which cannot be recovered, making further ordered reads impossible. + #[error("ordered read after unordered read")] + IllegalOrderedRead, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for ReadError { + fn from(e: ReadableError) -> Self { + match e { + ReadableError::ClosedStream => Self::ClosedStream, + ReadableError::IllegalOrderedRead => Self::IllegalOrderedRead, + } + } +} + +impl From for ReadError { + fn from(e: ResetError) -> Self { + match e { + ResetError::ConnectionLost(e) => Self::ConnectionLost(e), + ResetError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: ReadError) -> Self { + use ReadError::*; + let kind = match x { + Reset { .. } | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + IllegalOrderedRead => io::ErrorKind::InvalidInput, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while waiting for a stream to be reset +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ResetError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: ResetError) -> Self { + use ResetError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Future produced by [`RecvStream::read()`]. +/// +/// [`RecvStream::read()`]: crate::RecvStream::read +struct Read<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for Read<'_> { + type Output = Result, ReadError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + match this.buf.filled().len() { + 0 if this.buf.capacity() != 0 => Poll::Ready(Ok(None)), + n => Poll::Ready(Ok(Some(n))), + } + } +} + +/// Future produced by [`RecvStream::read_exact()`]. +/// +/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +struct ReadExact<'a> { + stream: &'a mut RecvStream, + buf: ReadBuf<'a>, +} + +impl Future for ReadExact<'_> { + type Output = Result<(), ReadExactError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let mut remaining = this.buf.remaining(); + while remaining > 0 { + ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; + let new = this.buf.remaining(); + if new == remaining { + return Poll::Ready(Err(ReadExactError::FinishedEarly(this.buf.filled().len()))); + } + remaining = new; + } + Poll::Ready(Ok(())) + } +} + +/// Errors that arise from reading from a stream. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ReadExactError { + /// The stream finished before all bytes were read + #[error("stream finished early ({0} bytes read)")] + FinishedEarly(usize), + /// A read error occurred + #[error(transparent)] + ReadError(#[from] ReadError), +} + +/// Future produced by [`RecvStream::read_chunk()`]. +/// +/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +struct ReadChunk<'a> { + stream: &'a mut RecvStream, + max_length: usize, + ordered: bool, +} + +impl Future for ReadChunk<'_> { + type Output = Result, ReadError>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (max_length, ordered) = (self.max_length, self.ordered); + self.stream.poll_read_chunk(cx, max_length, ordered) + } +} + +/// Future produced by [`RecvStream::read_chunks()`]. +/// +/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +struct ReadChunks<'a> { + stream: &'a mut RecvStream, + bufs: &'a mut [Bytes], +} + +impl Future for ReadChunks<'_> { + type Output = Result, ReadError>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + this.stream.poll_read_chunks(cx, this.bufs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_to_end_reordered_limit() { + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 4, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 0, + bytes: Bytes::from_static(b"abcd") + }), + Err(ReadToEndError::TooLong) + ); + assert_eq!(buffer.read.len(), 1); + } + + #[test] + fn read_to_end_reordered_exact_limit_after_prefix() { + let mut buffer = ReadToEndBuffer::new(8); + // A previously consumed prefix must not count against the remaining-data limit. + buffer + .push(Chunk { + offset: 104, + bytes: Bytes::from_static(b"efgh"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 100, + bytes: Bytes::from_static(b"abcd"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"abcdefgh"); + } + + #[test] + fn read_to_end_gaps_and_empty() { + assert!(ReadToEndBuffer::new(0).finish().is_empty()); + let mut buffer = ReadToEndBuffer::new(5); + buffer + .push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e"), + }) + .unwrap(); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!(buffer.finish(), b"a\0\0\0e"); + let mut buffer = ReadToEndBuffer::new(4); + buffer + .push(Chunk { + offset: 10, + bytes: Bytes::from_static(b"a"), + }) + .unwrap(); + assert_eq!( + buffer.push(Chunk { + offset: 14, + bytes: Bytes::from_static(b"e") + }), + Err(ReadToEndError::TooLong) + ); + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs new file mode 100644 index 0000000000..9898a0e879 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -0,0 +1,330 @@ +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +use std::sync::Arc; +use std::{ + fmt::{self, Debug}, + future::Future, + io::{self, IoSliceMut}, + net::SocketAddr, + pin::Pin, + task::{self, Context, Poll}, + time::Duration, +}; + +use udp::{RecvMeta, Transmit}; + +use crate::Instant; + +/// Abstracts I/O and timer operations for runtime independence +pub trait Runtime: Send + Sync + Debug + 'static { + /// Construct a timer that will expire at `i` + fn new_timer(&self, i: Instant) -> Pin>; + /// Drive `future` to completion in the background + #[track_caller] + fn spawn(&self, future: Pin + Send>>); + /// Convert `t` into the socket type used by this runtime + #[cfg(not(wasm_browser))] + fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; + /// Look up the current time + /// + /// Allows simulating the flow of time for testing. + fn now(&self) -> Instant { + Instant::now() + } +} + +/// Abstract implementation of an async timer for runtime independence +pub trait AsyncTimer: Send + Debug + 'static { + /// Update the timer to expire at `i` + fn reset(self: Pin<&mut Self>, i: Instant); + /// Check whether the timer has expired, and register to be woken if not + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; +} + +/// Abstract implementation of a UDP socket for runtime independence +pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { + /// Create a [`UdpSender`] that can register a single task for write-readiness notifications + /// and send a transmit, if ready. + /// + /// A `poll_send` method on a single object can usually store only one [`Waker`] at a time, + /// i.e. allow at most one caller to wait for an event. This method allows any number of + /// interested tasks to construct their own [`UdpSender`] object. They can all then wait for the + /// same event and be notified concurrently, because each [`UdpSender`] can store a separate + /// [`Waker`]. + /// + /// [`Waker`]: std::task::Waker + fn create_sender(&self) -> Pin>; + + /// Receive UDP datagrams, or register to be woken if receiving may succeed in the future + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll>; + + /// Look up the local IP address and port used by this socket + fn local_addr(&self) -> io::Result; + + /// Maximum number of datagrams that might be described by a single [`RecvMeta`] + fn max_receive_segments(&self) -> usize { + 1 + } + + /// Whether datagrams might get fragmented into multiple parts + /// + /// Sockets should prevent this for best performance. See e.g. the `IPV6_DONTFRAG` socket + /// option. + fn may_fragment(&self) -> bool { + true + } +} + +/// An object for asynchronously writing to an associated [`AsyncUdpSocket`]. +/// +/// Any number of [`UdpSender`]s may exist for a single [`AsyncUdpSocket`]. Each [`UdpSender`] is +/// responsible for notifying at most one task for send readiness. +pub trait UdpSender: Send + Sync + Debug + 'static { + /// Send a UDP datagram, or register to be woken if sending may succeed in the future. + /// + /// Usually implementations of this will poll the socket for writability before trying to + /// write to them, and retry both if writing fails. + /// + /// Quinn will create multiple [`UdpSender`]s, one for each task it's using it from. Thus it's + /// important to poll the underlying socket in a way that doesn't overwrite wakers. + /// + /// A single [`UdpSender`] will be re-used, even if `poll_send` returns `Poll::Ready` once, + /// unlike [`Future::poll`], so calling it again after readiness should not panic. + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll>; + + /// Maximum number of datagrams that a [`Transmit`] may encode. + fn max_transmit_segments(&self) -> usize { + 1 + } +} + +pin_project_lite::pin_project! { + /// A helper for constructing [`UdpSender`]s from an underlying `Socket` type. + /// + /// This struct implements [`UdpSender`] if `MakeWritableFn` produces a `WritableFut`. + /// + /// Also serves as a trick, since `WritableFut` doesn't need to be a named future, + /// it can be an anonymous async block, as long as `MakeWritableFn` produces that + /// anonymous async block type. + /// + /// The `UdpSenderHelper` generic type parameters don't need to named, as it will be + /// used in its dyn-compatible form as a `Pin>`. + struct UdpSenderHelper { + socket: Socket, + last_send_error: Option, + make_writable_fut_fn: MakeWritableFutFn, + #[pin] + writable_fut: Option, + } +} + +impl Debug + for UdpSenderHelper +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("UdpSender") + } +} + +impl + UdpSenderHelper +{ + /// Create helper that implements [`UdpSender`] from a socket. + /// + /// Additionally you need to provide what is essentially an async function + /// that resolves once the socket is write-ready. + /// + /// See also the bounds on this struct's [`UdpSender`] implementation. + #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] + fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { + Self { + socket: inner, + last_send_error: None, + make_writable_fut_fn: make_fut, + writable_fut: None, + } + } +} + +impl UdpSender + for UdpSenderHelper +where + Socket: UdpSenderHelperSocket, + MakeWritableFutFn: Fn(&Socket) -> WritableFut + Send + Sync + 'static, + WritableFut: Future> + Send + Sync + 'static, +{ + fn poll_send( + self: Pin<&mut Self>, + transmit: &Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let mut this = self.project(); + loop { + if this.writable_fut.is_none() { + this.writable_fut + .set(Some((this.make_writable_fut_fn)(this.socket))); + } + // We're forced to `unwrap` here because `Fut` may be `!Unpin`, which means we can't safely + // obtain an `&mut WritableFut` after storing it in `self.writable_fut` when `self` is already behind `Pin`, + // and if we didn't store it then we wouldn't be able to keep it alive between + // `poll_send` calls. + let result = task::ready!(this.writable_fut.as_mut().as_pin_mut().unwrap().poll(cx)); + + // Polling an arbitrary `Future` after it becomes ready is a logic error, so arrange for + // a new `Future` to be created on the next call. + this.writable_fut.set(None); + + // If .writable() fails, propagate the error + result?; + + match this.socket.try_send(transmit) { + // We thought the socket was writable, but it wasn't, then retry so that either another + // `writable().await` call determines that the socket is indeed not writable and + // registers us for a wakeup, or the send succeeds if this really was just a + // transient failure. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue, + Err(e) => { + log_sendmsg_error(this.last_send_error, &e, transmit); + return Poll::Ready(Ok(())); + } + Ok(()) => return Poll::Ready(Ok(())), + } + } + } + + fn max_transmit_segments(&self) -> usize { + self.socket.max_transmit_segments() + } +} + +/// Limits I/O error logging to one message per minute. +const IO_ERROR_LOG_INTERVAL: Duration = Duration::from_secs(60); + +fn log_sendmsg_error( + last_send_error: &mut Option, + error: &io::Error, + transmit: &Transmit<'_>, +) { + #[cfg(unix)] + // Unix `EMSGSIZE` is expected for MTU probes. + if udp::is_msg_size_err(error) { + return; + } + + let now = Instant::now(); + if last_send_error + .is_some_and(|last| now.saturating_duration_since(last) <= IO_ERROR_LOG_INTERVAL) + { + return; + } + *last_send_error = Some(now); + + tracing::warn!( + "sendmsg error: {:?}, Transmit: {{ destination: {:?}, src_ip: {:?}, ecn: {:?}, len: {:?}, segment_size: {:?} }}", + error, + transmit.destination, + transmit.src_ip, + transmit.ecn, + transmit.contents.len(), + transmit.segment_size + ); +} + +/// Parts of the [`UdpSender`] trait that aren't asynchronous or require storing wakers. +/// +/// This trait is used by [`UdpSenderHelper`] to help construct [`UdpSender`]s. +trait UdpSenderHelperSocket: Send + Sync + 'static { + /// Try to send a transmit, if the socket happens to be write-ready. + /// + /// If not write-ready, this is allowed to return [`std::io::ErrorKind::WouldBlock`]. + /// + /// The [`UdpSenderHelper`] will use this to implement [`UdpSender::poll_send`]. + fn try_send(&self, transmit: &Transmit<'_>) -> io::Result<()>; + + /// See [`UdpSender::max_transmit_segments`]. + fn max_transmit_segments(&self) -> usize; +} + +/// Automatically select an appropriate runtime from those enabled at compile time +/// +/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, +/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is +/// returned. Otherwise, `None` is returned. +#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] +#[allow(clippy::needless_return)] // Be sure we return the right thing +pub fn default_runtime() -> Option> { + #[cfg(feature = "runtime-tokio")] + { + if ::tokio::runtime::Handle::try_current().is_ok() { + return Some(Arc::new(TokioRuntime)); + } + } + + #[cfg(feature = "runtime-smol")] + { + return Some(Arc::new(SmolRuntime)); + } + + #[cfg(not(feature = "runtime-smol"))] + None +} + +#[cfg(feature = "runtime-tokio")] +mod tokio; +#[cfg(feature = "runtime-tokio")] +pub use tokio::TokioRuntime; + +#[cfg(feature = "runtime-smol")] +mod smol; +#[cfg(feature = "runtime-smol")] +pub use smol::*; + +#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +mod tests { + use std::{future, task::Waker}; + + use super::*; + + #[test] + fn non_would_block_send_errors_are_logged_and_ignored() { + let mut sender = Box::pin(UdpSenderHelper::new(TestSocket, writable)); + let transmit = Transmit { + destination: SocketAddr::from(([127, 0, 0, 1], 4433)), + ecn: None, + contents: &[], + segment_size: None, + src_ip: None, + }; + let mut cx = Context::from_waker(Waker::noop()); + + let result = sender.as_mut().poll_send(&transmit, &mut cx); + + assert!(matches!(result, Poll::Ready(Ok(())))); + assert!(sender.last_send_error.is_some()); + } + + #[derive(Debug)] + struct TestSocket; + + impl UdpSenderHelperSocket for TestSocket { + fn try_send(&self, _transmit: &Transmit<'_>) -> io::Result<()> { + Err(io::ErrorKind::PermissionDenied.into()) + } + + fn max_transmit_segments(&self) -> usize { + 1 + } + } + + fn writable(_: &TestSocket) -> future::Ready> { + future::ready(Ok(())) + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs new file mode 100644 index 0000000000..b5ae3d0cf2 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/smol.rs @@ -0,0 +1,105 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, + time::Instant, +}; +use std::{io, sync::Arc, task::ready}; + +use async_io::Async; +use async_io::Timer; + +use super::AsyncTimer; +use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for smol +#[derive(Debug)] +pub struct SmolRuntime; + +impl Runtime for SmolRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(Timer::at(t)) + } + + fn spawn(&self, future: Pin + Send>>) { + ::smol::spawn(future).detach(); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket::new(sock)?)) + } +} + +impl AsyncTimer for Timer { + fn reset(mut self: Pin<&mut Self>, t: Instant) { + self.set_at(t) + } + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx).map(|_| ()) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc>, + inner: Arc, +} + +impl UdpSocket { + fn new(sock: std::net::UdpSocket) -> io::Result { + Ok(Self { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(Async::new_nonblocking(sock)?), + }) + } +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.inner.try_send((&self.io).into(), transmit)?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_readable(cx))?; + match self.inner.recv((&self.io).into(), bufs, meta) { + Ok(res) => return Poll::Ready(Ok(res)), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.as_ref().as_ref().local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs new file mode 100644 index 0000000000..ac5dfd6f20 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -0,0 +1,111 @@ +use std::{ + future::Future, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll, ready}, + time::Instant, +}; + +use tokio::{ + io::Interest, + time::{Sleep, sleep_until}, +}; + +use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; + +/// A Quinn runtime for Tokio +#[derive(Debug)] +pub struct TokioRuntime; + +impl Runtime for TokioRuntime { + fn new_timer(&self, t: Instant) -> Pin> { + Box::pin(sleep_until(t.into())) + } + + fn spawn(&self, future: Pin + Send>>) { + tokio::spawn(future); + } + + fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) + } + + fn now(&self) -> Instant { + tokio::time::Instant::now().into_std() + } +} + +impl AsyncTimer for Sleep { + fn reset(self: Pin<&mut Self>, t: Instant) { + Self::reset(self, t.into()) + } + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + Future::poll(self, cx) + } +} + +#[derive(Debug, Clone)] +struct UdpSocket { + io: Arc, + inner: Arc, +} + +impl UdpSenderHelperSocket for UdpSocket { + fn max_transmit_segments(&self) -> usize { + self.inner.max_gso_segments() + } + + fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + self.io.try_io(Interest::WRITABLE, || { + self.inner.try_send((&self.io).into(), transmit) + })?; + + Ok(()) + } +} + +impl AsyncUdpSocket for UdpSocket { + fn create_sender(&self) -> Pin> { + Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { + let socket = socket.clone(); + async move { socket.io.writable().await } + })) + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [io::IoSliceMut<'_>], + meta: &mut [udp::RecvMeta], + ) -> Poll> { + loop { + ready!(self.io.poll_recv_ready(cx))?; + match self.io.try_io(Interest::READABLE, || { + self.inner.recv((&self.io).into(), bufs, meta) + }) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`. Looping on any other + // error would spin: readiness stays asserted, so `poll_recv_ready` + // completes at once and the recv fails again, without ever yielding. + Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} + Err(e) => return Poll::Ready(Err(e)), + } + } + } + + fn local_addr(&self) -> io::Result { + self.io.local_addr() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } + + fn max_receive_segments(&self) -> usize { + self.inner.gro_segments() + } +} diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs new file mode 100644 index 0000000000..570db9ac81 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -0,0 +1,454 @@ +use std::{ + future::{Future, poll_fn}, + io, + pin::{Pin, pin}, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use thiserror::Error; + +use crate::{ + VarInt, + connection::{ConnectionRef, State}, +}; + +/// A stream that can only be used to send data +/// +/// If dropped, streams that haven't been explicitly [`reset()`] will be implicitly [`finish()`]ed, +/// continuing to (re)transmit previously written data until it has been fully acknowledged or the +/// connection is closed. +/// +/// # Cancellation +/// +/// A `write` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready will always result in no data being written to the stream. This is true of methods which +/// succeed immediately when any progress is made, and is not true of methods which might need to +/// perform multiple writes internally before succeeding. Each `write` method documents whether it is +/// cancel-safe. +/// +/// [`reset()`]: SendStream::reset +/// [`finish()`]: SendStream::finish +#[derive(Debug)] +pub struct SendStream { + conn: ConnectionRef, + stream: StreamId, + is_0rtt: bool, +} + +impl SendStream { + pub(crate) fn new(conn: ConnectionRef, stream: StreamId, is_0rtt: bool) -> Self { + Self { + conn, + stream, + is_0rtt, + } + } + + /// Write a buffer into this stream, returning how many bytes were written + /// + /// Unless this method errors, it waits until some amount of `buf` can be written into this + /// stream, and then writes as much as it can without waiting again. Due to congestion and flow + /// control, this may be shorter than `buf.len()`. On success this yields the length of the + /// prefix that was written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write(&mut self, buf: &[u8]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write(buf))).await + } + + /// Write a buffer into this stream in its entirety + /// + /// This method repeatedly calls [`write`](Self::write) until all bytes are written, or an + /// error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some prefix of `buf` + /// may have been written when previously polled. + pub async fn write_all(&mut self, mut buf: &[u8]) -> Result<(), WriteError> { + while !buf.is_empty() { + let written = self.write(buf).await?; + buf = &buf[written..]; + } + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream, returning how much was written + /// + /// Bytes to try to write are provided to this method as an array of cheaply cloneable chunks. + /// Unless this method errors, it waits until some amount of those bytes can be written into + /// this stream, and then writes as much as it can without waiting again. Due to congestion and + /// flow control, this may be less than the total number of bytes. + /// + /// On success, this method both mutates `bufs` and yields an informative [`Written`] struct + /// indicating how much was written: + /// + /// - [`Bytes`] chunks that were fully written are mutated to be [empty](Bytes::is_empty). + /// - If a [`Bytes`] chunk was partially written, it is [split to](Bytes::split_to) contain + /// only the suffix of bytes that were not written. + /// - The yielded [`Written`] struct indicates how many chunks were fully written as well as + /// how many bytes were written. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write_chunks(&mut self, bufs: &mut [Bytes]) -> Result { + poll_fn(|cx| self.execute_poll(cx, |s| s.write_chunks(bufs))).await + } + + /// Write a single [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an single cheaply cloneable chunk. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.write_all_chunks(&mut [buf]).await?; + Ok(()) + } + + /// Write a slice of [`Bytes`] into this stream in its entirety + /// + /// Bytes to write are provided to this method as an array of cheaply cloneable chunks. This + /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, + /// or an error occurs. This method mutates `bufs` by mutating all chunks to be + /// [empty](Bytes::is_empty). + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have + /// been written when previously polled. + pub async fn write_all_chunks(&mut self, mut bufs: &mut [Bytes]) -> Result<(), WriteError> { + while !bufs.is_empty() { + let written = self.write_chunks(bufs).await?; + bufs = &mut bufs[written.chunks..]; + } + Ok(()) + } + + fn execute_poll( + &mut self, + cx: &mut Context<'_>, + write_fn: F, + ) -> Poll> + where + F: FnOnce(&mut proto::SendStream<'_>) -> Result, + { + use proto::WriteError::*; + let mut conn = self.conn.state.lock("SendStream::poll_write"); + if self.is_0rtt { + conn.check_0rtt() + .map_err(|()| WriteError::ZeroRttRejected)?; + } + if let Some(x) = &conn.error { + return Poll::Ready(Err(WriteError::ConnectionLost(x.clone()))); + } + + let result = match write_fn(&mut conn.inner.send_stream(self.stream)) { + Ok(result) => result, + Err(Blocked) => { + conn.blocked_writers.insert(self.stream, cx.waker().clone()); + return Poll::Pending; + } + Err(Stopped(error_code)) => { + return Poll::Ready(Err(WriteError::Stopped(error_code))); + } + Err(ClosedStream) => { + return Poll::Ready(Err(WriteError::ClosedStream)); + } + }; + + conn.wake(); + Poll::Ready(Ok(result)) + } + + /// Notify the peer that no more data will ever be written to this stream + /// + /// It is an error to write to a [`SendStream`] after `finish()`ing it. [`reset()`](Self::reset) + /// may still be called after `finish` to abandon transmission of any stream data that might + /// still be buffered. + /// + /// To wait for the peer to receive all buffered stream data, see [`stopped()`](Self::stopped). + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("finish"); + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => { + conn.wake(); + Ok(()) + } + Err(FinishError::ClosedStream) => Err(ClosedStream::default()), + // Harmless. If the application needs to know about stopped streams at this point, it + // should call `stopped`. + Err(FinishError::Stopped(_)) => Ok(()), + } + } + + /// Close the send stream immediately. + /// + /// No new data can be written after calling this method. Locally buffered data is dropped, and + /// previously transmitted data will no longer be retransmitted if lost. If an attempt has + /// already been made to finish the stream, the peer may still receive all written data. + /// + /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously + /// called. This error is harmless and serves only to indicate that the caller may have + /// incorrect assumptions about the stream's state. + pub fn reset(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::reset"); + if self.is_0rtt && conn.check_0rtt().is_err() { + return Ok(()); + } + conn.inner.send_stream(self.stream).reset(error_code)?; + conn.wake(); + Ok(()) + } + + /// Set the priority of the send stream + /// + /// Every send stream has an initial priority of 0. Locally buffered data from streams with + /// higher priority will be transmitted before data from streams with lower priority. Changing + /// the priority of a stream with pending data may only take effect after that data has been + /// transmitted. Using many different priority levels per connection may have a negative + /// impact on performance. + pub fn set_priority(&self, priority: i32) -> Result<(), ClosedStream> { + let mut conn = self.conn.state.lock("SendStream::set_priority"); + conn.inner.send_stream(self.stream).set_priority(priority)?; + Ok(()) + } + + /// Get the priority of the send stream + pub fn priority(&self) -> Result { + let mut conn = self.conn.state.lock("SendStream::priority"); + conn.inner.send_stream(self.stream).priority() + } + + /// Completes when the peer stops the stream or reads the stream to completion + /// + /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the + /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt + /// of all stream data (although not necessarily the processing of it), after which the peer + /// closing the stream is no longer meaningful. + /// + /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving + /// data. As such, relying on `stopped` to know when the peer has read a stream to completion + /// may introduce more latency than using an application-level response of some sort. + /// + /// Clients may wish to await this after finishing a unidirectional 0-RTT stream to reliably + /// determine whether the stream was rejected. + pub fn stopped( + &self, + ) -> impl Future, StoppedError>> + Send + Sync + 'static { + let conn = self.conn.clone(); + let stream = self.stream; + let is_0rtt = self.is_0rtt; + async move { + loop { + // The `Notify::notified` future needs to be created while the lock is being held, + // otherwise a wakeup could be missed if triggered inbetween releasing the lock + // and creating the future. + // The lock may only be held in a block without `await`s, otherwise the future + // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore + // we need to declare `notify` outside of the block, and initialize it inside. + let notify; + { + let mut conn = conn.state.lock("SendStream::stopped"); + if let Some(output) = send_stream_stopped(&mut conn, stream, is_0rtt) { + return output; + } + + notify = conn.stopped.entry(stream).or_default().clone(); + notify.notified() + } + .await + } + } + } + + /// Get the identity of this stream + pub fn id(&self) -> StreamId { + self.stream + } + + /// Attempt to write bytes from buf into the stream. + /// + /// On success, returns Poll::Ready(Ok(num_bytes_written)). + /// + /// If the stream is not ready for writing, the method returns Poll::Pending and arranges + /// for the current task (via cx.waker().wake_by_ref()) to receive a notification when the + /// stream becomes writable or is closed. + pub fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + pin!(self.get_mut().write(buf)).as_mut().poll(cx) + } +} + +/// Check if a send stream is stopped. +/// +/// Returns `Some` if the stream is stopped or the connection is closed. +/// Returns `None` if the stream is not stopped. +fn send_stream_stopped( + conn: &mut State, + stream: StreamId, + is_0rtt: bool, +) -> Option, StoppedError>> { + if is_0rtt && conn.check_0rtt().is_err() { + return Some(Err(StoppedError::ZeroRttRejected)); + } + match conn.inner.send_stream(stream).stopped() { + Err(ClosedStream { .. }) => Some(Ok(None)), + Ok(Some(error_code)) => Some(Ok(Some(error_code))), + Ok(None) => conn.error.clone().map(|error| Err(error.into())), + } +} + +#[cfg(feature = "futures-io")] +impl futures_io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.poll_write(cx, buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.get_mut().finish().map_err(Into::into)) + } +} + +impl Drop for SendStream { + fn drop(&mut self) { + let mut conn = self.conn.state.lock("SendStream::drop"); + + // clean up any previously registered wakers + conn.blocked_writers.remove(&self.stream); + + if conn.error.is_some() || (self.is_0rtt && conn.check_0rtt().is_err()) { + return; + } + match conn.inner.send_stream(self.stream).finish() { + Ok(()) => conn.wake(), + Err(FinishError::Stopped(reason)) => { + if conn.inner.send_stream(self.stream).reset(reason).is_ok() { + conn.wake(); + } + } + // Already finished or reset, which is fine. + Err(FinishError::ClosedStream) => {} + } + } +} + +/// Errors that arise from writing to a stream +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum WriteError { + /// The peer is no longer accepting data on this stream + /// + /// Carries an application-defined error code. + #[error("sending stopped by peer: error {0}")] + Stopped(VarInt), + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// The stream has already been finished or reset + #[error("closed stream")] + ClosedStream, + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for WriteError { + #[inline] + fn from(_: ClosedStream) -> Self { + Self::ClosedStream + } +} + +impl From for WriteError { + fn from(x: StoppedError) -> Self { + match x { + StoppedError::ConnectionLost(e) => Self::ConnectionLost(e), + StoppedError::ZeroRttRejected => Self::ZeroRttRejected, + } + } +} + +impl From for io::Error { + fn from(x: WriteError) -> Self { + use WriteError::*; + let kind = match x { + Stopped(_) | ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// Errors that arise while monitoring for a send stream stop from the peer +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum StoppedError { + /// The connection was lost + #[error("connection lost")] + ConnectionLost(#[from] ConnectionError), + /// This was a 0-RTT stream and the server rejected it + /// + /// Can only occur on clients for 0-RTT streams, which can be opened using + /// [`Connecting::into_0rtt()`]. + /// + /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + #[error("0-RTT rejected")] + ZeroRttRejected, +} + +impl From for io::Error { + fn from(x: StoppedError) -> Self { + use StoppedError::*; + let kind = match x { + ZeroRttRejected => io::ErrorKind::ConnectionReset, + ConnectionLost(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs new file mode 100755 index 0000000000..e0c6f3ef9e --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -0,0 +1,1871 @@ +#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] + +#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +use rustls::crypto::aws_lc_rs::default_provider; +#[cfg(feature = "rustls-ring")] +use rustls::crypto::ring::default_provider; + +use std::{ + convert::TryInto, + future::Future, + io::{self, IoSliceMut}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}, + pin::{Pin, pin}, + str, + sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, +}; + +use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; +use crate::{Duration, Instant}; +use bytes::Bytes; +use proto::{ + ConnectionId, RandomConnectionIdGenerator, + crypto::rustls::{QuicClientConfig, QuicServerConfig}, + crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, + transport_parameters::TransportParameters, +}; +use rand::{Rng, SeedableRng, rngs::StdRng}; +use rustls::{ + RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}, +}; +use tokio::time::{sleep, timeout}; +use tokio::{ + join, + runtime::{Builder, Runtime}, +}; +use tracing::instrument::Instrument as _; +use tracing::{error_span, info}; +use tracing_subscriber::EnvFilter; +use udp::RecvMeta; + +use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; + +#[test] +fn handshake_timeout() { + let _guard = subscribe(); + let runtime = rt_threaded(); + let client = { + let _guard = runtime.enter(); + Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap() + }; + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + const IDLE_TIMEOUT: Duration = Duration::from_millis(500); + let mut transport_config = TransportConfig::default(); + transport_config + .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .initial_rtt(Duration::from_millis(10)); + client_config.transport_config(Arc::new(transport_config)); + + let start = Instant::now(); + runtime.block_on(async move { + match client + .connect_with( + client_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1), + "localhost", + ) + .unwrap() + .await + { + Err(crate::ConnectionError::TimedOut) => {} + Err(e) => panic!("unexpected error: {e:?}"), + Ok(_) => panic!("unexpected success"), + } + }); + let dt = start.elapsed(); + assert!(dt > IDLE_TIMEOUT && dt < 2 * IDLE_TIMEOUT); +} + +#[tokio::test] +async fn close_endpoint() { + let _guard = subscribe(); + + // Avoid NoRootAnchors error + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(cert.cert.into()).unwrap(); + + let endpoint = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + endpoint + .set_default_client_config(ClientConfig::with_root_certificates(Arc::new(roots)).unwrap()); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + + tokio::spawn(async move { + let _ = conn.await; + }); + + let conn = endpoint + .connect( + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234), + "localhost", + ) + .unwrap(); + endpoint.close(0u32.into(), &[]); + match conn.await { + Err(crate::ConnectionError::LocallyClosed) => (), + Err(e) => panic!("unexpected error: {e}"), + Ok(_) => { + panic!("unexpected success"); + } + } +} + +#[test] +fn local_addr() { + let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).unwrap(); + let addr = socket.local_addr().unwrap(); + let runtime = rt_basic(); + let ep = { + let _guard = runtime.enter(); + Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + }; + assert_eq!( + addr, + ep.local_addr() + .expect("Could not obtain our local endpoint") + ); +} + +#[test] +fn read_after_close() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + const MSG: &[u8] = b"goodbye!"; + let endpoint2 = endpoint.clone(); + runtime.spawn(async move { + let new_conn = endpoint2 + .accept() + .await + .expect("endpoint") + .await + .expect("connection"); + let mut s = new_conn.open_uni().await.unwrap(); + s.write_all(MSG).await.unwrap(); + s.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = s.stopped().await; + }); + runtime.block_on(async move { + let new_conn = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect"); + sleep(Duration::from_millis(100)).await; + let mut stream = new_conn.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG); + }); +} + +#[test] +fn export_keying_material() { + let _guard = subscribe(); + let runtime = rt_basic(); + let endpoint = { + let _guard = runtime.enter(); + endpoint() + }; + + runtime.block_on(async move { + let outgoing_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .expect("connect") + } + }); + let incoming_conn_fut = tokio::spawn({ + let endpoint = endpoint.clone(); + async move { + endpoint + .accept() + .await + .expect("endpoint") + .await + .expect("connection") + } + }); + let outgoing_conn = outgoing_conn_fut.await.unwrap(); + let incoming_conn = incoming_conn_fut.await.unwrap(); + let mut i_buf = [0u8; 64]; + incoming_conn + .export_keying_material(&mut i_buf, b"asdf", b"qwer") + .unwrap(); + let mut o_buf = [0u8; 64]; + outgoing_conn + .export_keying_material(&mut o_buf, b"asdf", b"qwer") + .unwrap(); + assert_eq!(&i_buf[..], &o_buf[..]); + }); +} + +#[tokio::test] +async fn ip_blocking() { + let _guard = subscribe(); + let endpoint_factory = EndpointFactory::new(); + let client_1 = endpoint_factory.endpoint(); + let client_1_addr = client_1.local_addr().unwrap(); + let client_2 = endpoint_factory.endpoint(); + let server = endpoint_factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let server_task = tokio::spawn(async move { + loop { + let accepting = server.accept().await.unwrap(); + if accepting.remote_address() == client_1_addr { + accepting.refuse(); + } else if accepting.remote_address_validated() { + accepting.await.expect("connection"); + } else { + accepting.retry().unwrap(); + } + } + }); + tokio::join!( + async move { + let e = client_1 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect_err("server should have blocked this"); + assert!( + matches!(e, crate::ConnectionError::ConnectionClosed(_)), + "wrong error" + ); + }, + async move { + client_2 + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + } + ); + server_task.abort(); +} + +/// Construct an endpoint suitable for connecting to itself +fn endpoint() -> Endpoint { + EndpointFactory::new().endpoint() +} + +fn endpoint_with_config(transport_config: TransportConfig) -> Endpoint { + EndpointFactory::new().endpoint_with_config(transport_config) +} + +/// Constructs endpoints suitable for connecting to themselves and each other +struct EndpointFactory { + cert: rcgen::CertifiedKey, + endpoint_config: EndpointConfig, +} + +impl EndpointFactory { + fn new() -> Self { + Self { + cert: rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(), + endpoint_config: EndpointConfig::default(), + } + } + + fn endpoint(&self) -> Endpoint { + self.endpoint_with_config(TransportConfig::default()) + } + + fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { + let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); + let transport_config = Arc::new(transport_config); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + server_config.transport_config(transport_config.clone()); + + let mut roots = RootCertStore::empty(); + roots.add(self.cert.cert.der().clone()).unwrap(); + let endpoint = Endpoint::new( + self.endpoint_config.clone(), + Some(server_config), + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + Arc::new(TokioRuntime), + ) + .unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(transport_config); + endpoint.set_default_client_config(client_config); + + endpoint + } +} + +#[derive(Default)] +struct HandshakeBlocker { + state: Mutex, + changed: Condvar, +} + +#[derive(Default)] +struct HandshakeBlockerState { + started: bool, + released: bool, +} + +impl HandshakeBlocker { + fn block(&self) { + let mut state = self.state.lock().unwrap(); + state.started = true; + self.changed.notify_all(); + while !state.released { + state = self.changed.wait(state).unwrap(); + } + } + + fn wait_until_started(&self) { + let state = self.state.lock().unwrap(); + let (state, _) = self + .changed + .wait_timeout_while(state, Duration::from_secs(5), |state| !state.started) + .unwrap(); + assert!(state.started, "timed out waiting for handshake to start"); + } + + fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.released = true; + self.changed.notify_all(); + } +} + +struct HandshakeReleaseGuard(Arc); + +impl Drop for HandshakeReleaseGuard { + fn drop(&mut self) { + self.0.release(); + } +} + +struct BlockingServerConfig { + inner: Arc, + blocker: Arc, +} + +impl ProtoServerConfig for BlockingServerConfig { + fn initial_keys( + &self, + version: u32, + dst_cid: ConnectionId, + ) -> Result { + self.inner.initial_keys(version, dst_cid) + } + + fn retry_tag(&self, version: u32, orig_dst_cid: ConnectionId, packet: &[u8]) -> [u8; 16] { + self.inner.retry_tag(version, orig_dst_cid, packet) + } + + fn start_session( + self: Arc, + version: u32, + params: &TransportParameters, + ) -> Box { + self.blocker.block(); + self.inner.clone().start_session(version, params) + } +} + +/// Wraps a socket so that a test can make its next receive fail, which ends the endpoint driver +#[derive(Debug)] +struct FailingSocket { + inner: Box, + failure: Arc, +} + +#[derive(Debug, Default)] +struct SocketFailure { + armed: AtomicBool, + recv_waker: Mutex>, +} + +impl SocketFailure { + /// Make the next receive fail, waking the endpoint driver so that it observes the failure + fn trigger(&self) { + self.armed.store(true, Ordering::SeqCst); + if let Some(waker) = self.recv_waker.lock().unwrap().as_ref() { + waker.wake_by_ref(); + } + } +} + +impl AsyncUdpSocket for FailingSocket { + fn create_sender(&self) -> Pin> { + self.inner.create_sender() + } + + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], + ) -> Poll> { + if self.failure.armed.load(Ordering::SeqCst) { + return Poll::Ready(Err(io::Error::other("injected socket failure"))); + } + *self.failure.recv_waker.lock().unwrap() = Some(cx.waker().clone()); + self.inner.poll_recv(cx, bufs, meta) + } + + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + + fn max_receive_segments(&self) -> usize { + self.inner.max_receive_segments() + } + + fn may_fragment(&self) -> bool { + self.inner.may_fragment() + } +} + +/// Poll a connection driver before its registration returns, once the endpoint has failed. +#[derive(Debug)] +struct DriverLossRuntime { + failure: Arc, + stopped_on_first_poll: AtomicBool, +} + +impl crate::runtime::Runtime for DriverLossRuntime { + fn new_timer(&self, deadline: Instant) -> Pin> { + TokioRuntime.new_timer(deadline) + } + + fn spawn(&self, mut future: Pin + Send>>) { + if self.failure.armed.load(Ordering::SeqCst) { + let mut cx = Context::from_waker(Waker::noop()); + if future.as_mut().poll(&mut cx).is_ready() { + self.stopped_on_first_poll.store(true, Ordering::SeqCst); + return; + } + } + TokioRuntime.spawn(future); + } + + fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { + TokioRuntime.wrap_udp_socket(socket) + } + + fn now(&self) -> Instant { + TokioRuntime.now() + } +} + +fn localhost_socket() -> Box { + TokioRuntime + .wrap_udp_socket( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), + ) + .unwrap() +} + +fn blocking_server_pair( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> (Endpoint, Endpoint, Arc) { + let factory = EndpointFactory::new(); + let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); + let mut server_crypto = rustls::ServerConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![factory.cert.cert.der().clone()], key) + .unwrap(); + server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; + let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + QuicServerConfig::try_from(server_crypto).unwrap(), + )); + let blocker = Arc::new(HandshakeBlocker::default()); + server_config.crypto = Arc::new(BlockingServerConfig { + inner: server_config.crypto.clone(), + blocker: blocker.clone(), + }); + + let server = Endpoint::new_with_abstract_socket( + factory.endpoint_config.clone(), + Some(server_config), + server_socket, + server_runtime, + ) + .unwrap(); + + let mut roots = RootCertStore::empty(); + roots.add(factory.cert.cert.der().clone()).unwrap(); + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.alpn_protocols = vec![client_alpn.to_vec()]; + client.set_default_client_config(ClientConfig::new(Arc::new( + QuicClientConfig::try_from(client_crypto).unwrap(), + ))); + + (client, server, blocker) +} + +async fn wait_for_blocked_handshake(blocker: Arc) { + tokio::task::spawn_blocking(move || blocker.wait_until_started()) + .await + .unwrap(); +} + +async fn accept_with_blocked_start_session( + server: Endpoint, +) -> Result { + let incoming = server.accept().await.unwrap(); + // The test crypto provider intentionally blocks in `start_session`, so run + // accept on the blocking pool rather than parking a Tokio worker thread. + let connecting = tokio::task::spawn_blocking(move || incoming.accept()) + .await + .unwrap()?; + connecting.await +} + +struct BlockedHandshake { + client: Endpoint, + server: Endpoint, + blocker: Arc, + release_guard: HandshakeReleaseGuard, + accept: tokio::task::JoinHandle>, + connect: tokio::task::JoinHandle>, +} + +fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { + blocked_handshake_with_socket(server_alpn, client_alpn, localhost_socket()) +} + +fn blocked_handshake_with_socket( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, +) -> BlockedHandshake { + blocked_handshake_with_runtime( + server_alpn, + client_alpn, + server_socket, + Arc::new(TokioRuntime), + ) +} + +fn blocked_handshake_with_runtime( + server_alpn: &[u8], + client_alpn: &[u8], + server_socket: Box, + server_runtime: Arc, +) -> BlockedHandshake { + let (client, server, blocker) = + blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let server_addr = server.local_addr().unwrap(); + + let accept = tokio::spawn({ + let server = server.clone(); + async move { accept_with_blocked_start_session(server).await } + }); + let connect = tokio::spawn({ + let client = client.clone(); + async move { client.connect(server_addr, "localhost").unwrap().await } + }); + + BlockedHandshake { + client, + server, + blocker: blocker.clone(), + release_guard: HandshakeReleaseGuard(blocker), + accept, + connect, + } +} + +async fn await_connection( + task: tokio::task::JoinHandle>, +) -> Result { + timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() +} + +async fn assert_wait_idle_pending(endpoint: &Endpoint) { + assert!( + timeout(Duration::from_millis(50), endpoint.wait_idle()) + .await + .is_err() + ); +} + +async fn wait_idle(endpoint: &Endpoint) { + timeout(Duration::from_secs(5), endpoint.wait_idle()) + .await + .unwrap(); +} + +/// A parked `wait_idle` must not hang when the endpoint driver is dropped. +/// +/// `EndpointDriver::drop` clears the connection senders, which is what makes +/// the endpoint idle, but it wakes only `incoming`. A caller already parked on +/// `idle` is therefore never woken, and with the driver gone nothing else can +/// wake it. `EndpointInner::accept` already guards this same hazard for accepts +/// that complete after driver loss. +#[test] +fn wait_idle_returns_when_the_driver_is_dropped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + + // The endpoint under test owns its runtime, so shutting that runtime down + // drops its driver. + let endpoint_runtime = rt_threaded(); + let endpoint_handle = endpoint_runtime.handle().clone(); + let endpoint = { + let _guard = endpoint_runtime.enter(); + factory.endpoint() + }; + let endpoint_addr = endpoint.local_addr().unwrap(); + + // A peer on a second runtime, so a live connection exists to wait for. + let peer_runtime = rt_threaded(); + let peer = { + let _guard = peer_runtime.enter(); + factory.endpoint() + }; + let peer_handle = peer_runtime.handle().clone(); + // Accept concurrently: the client's handshake cannot complete until the + // server drives its half, and the accepted handle must stay alive or the + // connection would close and the endpoint become idle before the assertion. + let accepted = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(async { + let incoming = endpoint.accept().await.expect("incoming"); + incoming.await.expect("accept") + }) + }) + }; + let connection = peer_handle.block_on(async { + peer.connect(endpoint_addr, "localhost") + .expect("connect") + .await + .expect("connection") + }); + let _server_connection = accepted.join().expect("accept thread"); + + // Park `wait_idle`; it blocks while that connection is live. + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let waiter = { + let handle = endpoint_handle.clone(); + let endpoint = endpoint.clone(); + std::thread::spawn(move || { + handle.block_on(endpoint.wait_idle()); + let _ = done_tx.send(()); + }) + }; + + // Otherwise a pass below would be vacuous. + assert!( + done_rx.recv_timeout(Duration::from_millis(200)).is_err(), + "wait_idle returned before the driver was dropped" + ); + + // Dropping the runtime drops the endpoint driver. + endpoint_runtime.shutdown_background(); + + let woke = done_rx.recv_timeout(Duration::from_secs(10)).is_ok(); + drop(connection); + drop(peer); + drop(peer_runtime); + assert!( + woke, + "wait_idle never returned after the endpoint driver was dropped" + ); + let _ = waiter.join(); +} + +async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { + timeout(Duration::from_secs(5), conn.closed()) + .await + .unwrap() +} + +#[tokio::test] +async fn zero_rtt() { + let _guard = subscribe(); + let endpoint = endpoint(); + + const MSG0: &[u8] = b"zero"; + const MSG1: &[u8] = b"one"; + let endpoint2 = endpoint.clone(); + let accept_connection = move |expect_0rtt| { + let endpoint2 = endpoint2.clone(); + async move { + let incoming = endpoint2.accept().await.unwrap().accept().unwrap(); + let connection = incoming.into_0rtt().unwrap_or_else(|_| unreachable!()); + let c = connection.clone(); + tokio::spawn(async move { + while let Ok(mut x) = c.accept_uni().await { + assert_eq!(x.is_0rtt(), expect_0rtt); + let msg = x.read_to_end(usize::MAX).await.unwrap(); + assert_eq!(msg, MSG0); + } + }); + info!("sending 0.5-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG0).await.expect("write"); + s.finish().unwrap(); + connection.authenticated().await.expect("connected"); + info!("sending 1-RTT"); + let mut s = connection.open_uni().await.expect("open_uni"); + s.write_all(MSG1).await.expect("write"); + // The peer might close the connection before ACKing + let _ = s.finish(); + } + }; + + tokio::spawn(accept_connection(false)); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .expect_err("0-RTT succeeded without keys") + .await + .expect("connect"); + + { + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + // Read a 1-RTT message to ensure the handshake completes fully, allowing the server's + // NewSessionTicket frame to be received. + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG1); + drop(connection); + } + + info!("initial connection complete"); + + let connection = endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .into_0rtt() + .unwrap_or_else(|_| panic!("missing 0-RTT keys")); + + // Send something before the handshake can make progress, thereby forcing 0-RTT + let mut stream_0rtt = connection.open_uni().await.expect("0-RTT open uni"); + info!("sending 0-RTT"); + stream_0rtt.write_all(MSG0).await.expect("0-RTT write"); + stream_0rtt.finish().unwrap(); + + tokio::spawn(accept_connection(true)); + + // Receive 0.5-RTT + let mut stream = connection.accept_uni().await.expect("incoming streams"); + let msg = stream.read_to_end(usize::MAX).await.expect("read_to_end"); + assert_eq!(msg, MSG0); + connection.authenticated().await.expect("connected"); + + // Ensure 0-RTT was accepted + stream_0rtt.stopped().await.expect("0-RTT stopped"); + + // Allow the connection to close + drop((stream_0rtt, stream, connection)); + + endpoint.wait_idle().await; +} + +#[test] +#[cfg_attr( + any(target_os = "solaris", target_os = "illumos"), + ignore = "Fails on Solaris and Illumos" +)] +fn echo_v6() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Sometimes hangs in poll() on Solaris")] +fn echo_v4() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn echo_dualstack() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 1, + stream_size: 10 * 1024, + receive_window: None, + stream_receive_window: None, + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_receive_window() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(100 * 1024 * 1024), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_stream_receive_window() { + // Note that there is no point in running this with too many streams, + // since the window is only active within a stream. + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 2, + stream_size: 250 * 1024 + 11, + receive_window: Some(100 * 1024 * 1024), + stream_receive_window: Some(37), + }); +} + +#[test] +#[ignore] +#[cfg_attr(target_os = "solaris", ignore = "Hangs in poll() on Solaris")] +fn stress_both_windows() { + run_echo(EchoArgs { + client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), + server_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + nr_streams: 50, + stream_size: 25 * 1024 + 11, + receive_window: Some(37), + stream_receive_window: Some(37), + }); +} + +fn run_echo(args: EchoArgs) { + let _guard = subscribe(); + let runtime = rt_basic(); + let handle = { + // Use small receive windows + let mut transport_config = TransportConfig::default(); + if let Some(receive_window) = args.receive_window { + transport_config.receive_window(receive_window.try_into().unwrap()); + } + if let Some(stream_receive_window) = args.stream_receive_window { + transport_config.stream_receive_window(stream_receive_window.try_into().unwrap()); + } + transport_config.max_concurrent_bidi_streams(1_u8.into()); + transport_config.max_concurrent_uni_streams(1_u8.into()); + let transport_config = Arc::new(transport_config); + + // We don't use the `endpoint` helper here because we want two different endpoints with + // different addresses. + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + let mut server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + + server_config.transport = transport_config.clone(); + let server_sock = UdpSocket::bind(args.server_addr).unwrap(); + let server_addr = server_sock.local_addr().unwrap(); + let server = { + let _guard = runtime.enter(); + let _guard = error_span!("server").entered(); + Endpoint::new( + Default::default(), + Some(server_config), + server_sock, + Arc::new(TokioRuntime), + ) + .unwrap() + }; + + let mut roots = RootCertStore::empty(); + roots.add(cert).unwrap(); + let mut client_crypto = + rustls::ClientConfig::builder_with_provider(default_provider().into()) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + client_crypto.key_log = Arc::new(rustls::KeyLogFile::new()); + + let client = { + let _guard = runtime.enter(); + let _guard = error_span!("client").entered(); + Endpoint::client(args.client_addr).unwrap() + }; + let mut client_config = + ClientConfig::new(Arc::new(QuicClientConfig::try_from(client_crypto).unwrap())); + client_config.transport_config(transport_config); + client.set_default_client_config(client_config); + + let handle = runtime.spawn(async move { + let incoming = server.accept().await.unwrap(); + + // Note for anyone modifying the platform support in this test: + // If `local_ip` gets available on additional platforms - which + // requires modifying this test - please update the list of supported + // platforms in the doc comment of `quinn_udp::RecvMeta::dst_ip`. + if cfg!(target_os = "linux") + || cfg!(target_os = "android") + || cfg!(target_os = "freebsd") + || cfg!(target_os = "openbsd") + || cfg!(target_os = "netbsd") + || cfg!(target_os = "macos") + || cfg!(target_os = "windows") + { + let local_ip = incoming.local_ip().expect("Local IP must be available"); + assert!(local_ip.is_loopback()); + } else { + assert_eq!(None, incoming.local_ip()); + } + + let new_conn = incoming.await.unwrap(); + tokio::spawn(async move { + while let Ok(stream) = new_conn.accept_bi().await { + tokio::spawn(echo(stream)); + } + }); + server.wait_idle().await; + }); + + info!("connecting from {} to {}", args.client_addr, server_addr); + runtime.block_on( + async move { + let new_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .expect("connect"); + + /// This is just an arbitrary number to generate deterministic test data + const SEED: u64 = 0x12345678; + + for i in 0..args.nr_streams { + println!("Opening stream {i}"); + let (mut send, mut recv) = new_conn.open_bi().await.expect("stream open"); + let msg = gen_data(args.stream_size, SEED); + + let send_task = async { + send.write_all(&msg).await.expect("write"); + send.finish().unwrap(); + }; + let recv_task = async { recv.read_to_end(usize::MAX).await.expect("read") }; + + let (_, data) = tokio::join!(send_task, recv_task); + + assert_eq!(data[..], msg[..], "Data mismatch"); + } + new_conn.close(0u32.into(), b"done"); + client.wait_idle().await; + } + .instrument(error_span!("client")), + ); + handle + }; + runtime.block_on(handle).unwrap(); +} + +struct EchoArgs { + client_addr: SocketAddr, + server_addr: SocketAddr, + nr_streams: usize, + stream_size: usize, + receive_window: Option, + stream_receive_window: Option, +} + +async fn echo((mut send, mut recv): (SendStream, RecvStream)) { + loop { + // These are 32 buffers, for reading approximately 32kB at once + #[rustfmt::skip] + let mut bufs = [ + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + Bytes::new(), Bytes::new(), Bytes::new(), Bytes::new(), + ]; + + match recv.read_chunks(&mut bufs).await.expect("read chunks") { + Some(n) => { + send.write_all_chunks(&mut bufs[..n]) + .await + .expect("write chunks"); + } + None => break, + } + } + + let _ = send.finish(); +} + +fn gen_data(size: usize, seed: u64) -> Vec { + let mut rng: StdRng = SeedableRng::seed_from_u64(seed); + let mut buf = vec![0; size]; + rng.fill_bytes(&mut buf); + buf +} + +fn subscribe() -> tracing::subscriber::DefaultGuard { + let sub = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(|| TestWriter) + .finish(); + tracing::subscriber::set_default(sub) +} + +struct TestWriter; + +impl io::Write for TestWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + print!( + "{}", + str::from_utf8(buf).expect("tried to log invalid UTF-8") + ); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + io::stdout().flush() + } +} + +fn rt_basic() -> Runtime { + Builder::new_current_thread().enable_all().build().unwrap() +} + +fn rt_threaded() -> Runtime { + Builder::new_multi_thread().enable_all().build().unwrap() +} + +#[tokio::test] +async fn retry_after_rebind() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + server + .rebind(UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap()) + .unwrap(); + + timeout(Duration::from_secs(5), async { + let connecting = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + let incoming = server.accept().await.unwrap(); + assert!(!incoming.remote_address_validated()); + incoming.retry().unwrap(); + + let incoming = server.accept().await.unwrap(); + assert!(incoming.remote_address_validated()); + let (accepted, connected) = join!(incoming, connecting); + let _server_connection = accepted.unwrap(); + let _client_connection = connected.unwrap(); + server.close(0u32.into(), b"done"); + client.close(0u32.into(), b"done"); + }) + .await + .expect("retried handshake must complete using the rebound socket"); +} + +#[tokio::test] +async fn rebind_recv() { + let _guard = subscribe(); + + let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); + let cert = CertificateDer::from(cert.cert); + + let mut roots = RootCertStore::empty(); + roots.add(cert.clone()).unwrap(); + + let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); + let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + client_config.transport_config(Arc::new({ + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + cfg + })); + client.set_default_client_config(client_config); + + let server_config = + crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + let server = { + let _guard = tracing::error_span!("server").entered(); + Endpoint::server( + server_config, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + ) + .unwrap() + }; + let server_addr = server.local_addr().unwrap(); + + const MSG: &[u8; 5] = b"hello"; + + let write_send = Arc::new(tokio::sync::Notify::new()); + let write_recv = write_send.clone(); + let connected_send = Arc::new(tokio::sync::Notify::new()); + let connected_recv = connected_send.clone(); + let server = tokio::spawn(async move { + let connection = server.accept().await.unwrap().await.unwrap(); + info!("got conn"); + connected_send.notify_one(); + write_recv.notified().await; + let mut stream = connection.open_uni().await.unwrap(); + stream.write_all(MSG).await.unwrap(); + stream.finish().unwrap(); + // Wait for the stream to be closed, one way or another. + _ = stream.stopped().await; + }); + + let connection = { + let _guard = tracing::error_span!("client").entered(); + client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap() + }; + info!("connected"); + connected_recv.notified().await; + client + .rebind(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) + .unwrap(); + info!("rebound"); + write_send.notify_one(); + let mut stream = connection.accept_uni().await.unwrap(); + assert_eq!(stream.read_to_end(MSG.len()).await.unwrap(), MSG); + server.await.unwrap(); +} + +/// Verify endpoint behavior while a connection is between `start_accept` and `finish_accept`: +/// `open_connections` must be 0, and `wait_idle` must not complete while `pending_accepts > 0`. +/// After the accept completes and the connection is closed, `wait_idle` must resolve. +#[tokio::test] +async fn split_accept_wait_idle_and_open_connections() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + let client_conn = await_connection(connect).await.unwrap(); + let server_conn = await_connection(accept).await.unwrap(); + assert_eq!(server.open_connections(), 1); + + client_conn.close(0u32.into(), b"done"); + let _ = wait_closed(&client_conn).await; + let _ = wait_closed(&server_conn).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// A failed accept never becomes a connection, so no Drained event will wake idle waiters. +#[tokio::test] +async fn split_accept_failure_wakes_idle_waiter() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"server", b"client"); + + wait_for_blocked_handshake(blocker.clone()).await; + assert_eq!(server.open_connections(), 0); + + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + let error = await_connection(accept).await.unwrap_err(); + assert!(matches!( + error, + crate::ConnectionError::TransportError(error) + if error.code == proto::TransportErrorCode::crypto(0x78) + )); + // Repolling wait_idle alone would hide a missing notification now that the endpoint is + // idle. Verify that the waiter registered before the failure was actually woken. + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + assert_eq!(server.open_connections(), 0); + + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +/// Verify that calling `Endpoint::close` while a connection is in the `Accepting` state +/// (between `start_accept` and `finish_accept`) produces `LocallyClosed` on the server side +/// and `ConnectionClosed` on the client side, and that `wait_idle` resolves afterward. +#[tokio::test] +async fn split_accept_close_during_pending_accept() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + assert_wait_idle_pending(&server).await; + + blocker.release(); + + match await_connection(accept).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + } + Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + } + + match await_connection(connect).await { + Ok(conn) => { + let err = wait_closed(&conn).await; + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + Err(err) => { + assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + } + } + + wait_idle(&server).await; + wait_idle(&client).await; + assert_eq!(server.open_connections(), 0); +} + +/// The endpoint driver may exit while an accept runs without the lock. The connection registered +/// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly +/// and `wait_idle` does not wait on a connection nobody drives. +#[tokio::test] +async fn split_accept_driver_lost_during_handshake() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_socket(b"test", b"test", socket); + + wait_for_blocked_handshake(blocker.clone()).await; + // Fail the server socket, which ends the endpoint driver while the handshake is blocked. + // `accept` yields `None` once the driver is gone. + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + // Register an idle waiter while the accept is still pending. Nothing but the accept's + // completion can wake it now that the driver is gone. + let mut idle = pin!(server.wait_idle()); + let (waker, wake_counter) = new_count_waker(); + assert!( + idle.as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + blocker.release(); + // Without the fix this waits for the handshake timeout, well past `await_connection`'s limit. + assert!(await_connection(accept).await.is_err()); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), idle).await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn split_accept_driver_lost_before_connection_spawn() { + let _guard = subscribe(); + let failure = Arc::new(SocketFailure::default()); + let runtime = Arc::new(DriverLossRuntime { + failure: failure.clone(), + stopped_on_first_poll: AtomicBool::new(false), + }); + let socket = Box::new(FailingSocket { + inner: localhost_socket(), + failure: failure.clone(), + }); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); + + wait_for_blocked_handshake(blocker.clone()).await; + failure.trigger(); + assert!( + timeout(Duration::from_secs(5), server.accept()) + .await + .unwrap() + .is_none() + ); + + blocker.release(); + // The runtime polls the connection driver before registration returns. It must observe the + // closed event channel immediately, before transmitting any handshake packets. + assert!(await_connection(accept).await.is_err()); + assert!( + runtime.stopped_on_first_poll.load(Ordering::SeqCst), + "connection driver ran before its event channel was closed" + ); + wait_idle(&server).await; + + client.close(0u32.into(), b"done"); + assert!(await_connection(connect).await.is_err()); + wait_idle(&client).await; +} + +#[tokio::test] +async fn remote_address_monitoring() { + let _guard = subscribe(); + + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let server_addr = server.local_addr().unwrap(); + let client = factory.endpoint(); + + let trigger_migration = Arc::new(tokio::sync::Notify::new()); + let trigger_migration_clone = trigger_migration.clone(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let old_remote = conn.remote_address(); + let mut addr_watch = conn.watch_remote_address(); + + trigger_migration_clone.notify_one(); + + let addr_a = addr_watch.wait().await.unwrap(); + assert_eq!(addr_a.port(), 23_001); + assert_ne!(old_remote, addr_a); + + trigger_migration_clone.notify_one(); + + let addr_b = addr_watch.wait().await.unwrap(); + assert_eq!(addr_b.port(), 23_003); + assert_ne!(addr_a, addr_b); + }); + + let client_conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + + let mut port = 23_000; + let mut migrate = async || { + port += 1; + client + .rebind( + UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port)).unwrap(), + ) + .unwrap(); + + // send data from the new socket to trigger migration on the server + let mut stream = client_conn.open_uni().await.unwrap(); + stream.write_all(b"hello").await.unwrap(); + stream.finish().unwrap(); + }; + + trigger_migration.notified().await; + migrate().await; + trigger_migration.notified().await; + migrate().await; + migrate().await; + + server_task.await.unwrap(); + client_conn.close(0u32.into(), b"done"); +} + +#[tokio::test] +async fn stream_id_flow_control() { + let _guard = subscribe(); + let mut cfg = TransportConfig::default(); + cfg.max_concurrent_uni_streams(1u32.into()); + let endpoint = endpoint_with_config(cfg); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + // If `open_uni` doesn't get unblocked when the previous stream is dropped, this will time out. + tokio::join!( + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + client.open_uni().await.unwrap(); + }, + async { + server.accept_uni().await.unwrap(); + server.accept_uni().await.unwrap(); + } + ); +} + +#[tokio::test] +async fn two_datagram_readers() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (client, server) = tokio::join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ); + let client = client.unwrap(); + let server = server.unwrap(); + + let done = tokio::sync::Notify::new(); + let (a, b, ()) = tokio::join!( + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + let x = client.read_datagram().await.unwrap(); + done.notify_waiters(); + x + }, + async { + server.send_datagram(b"one"[..].into()).unwrap(); + done.notified().await; + server.send_datagram_wait(b"two"[..].into()).await.unwrap(); + } + ); + assert!(*a == *b"one" || *b == *b"one"); + assert!(*a == *b"two" || *b == *b"two"); +} + +#[tokio::test] +async fn multiple_conns_with_zero_length_cids() { + let _guard = subscribe(); + let mut factory = EndpointFactory::new(); + factory + .endpoint_config + .cid_generator(Arc::new(|| Box::new(RandomConnectionIdGenerator::new(0)))); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client1 = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + let client2 = { + let _guard = error_span!("client2").entered(); + factory.endpoint() + }; + + let client1 = async move { + let conn = client1 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client1")); + let client2 = async move { + let conn = client2 + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + conn.closed().await; + } + .instrument(error_span!("client2")); + let server = async move { + let client1 = server.accept().await.unwrap().await.unwrap(); + let client2 = server.accept().await.unwrap().await.unwrap(); + // Both connections are now concurrently live. + client1.close(42u32.into(), &[]); + client2.close(42u32.into(), &[]); + } + .instrument(error_span!("server")); + tokio::join!(client1, client2, server); +} + +#[tokio::test] +async fn stream_stopped() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client1").entered(); + factory.endpoint() + }; + + let client = async move { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + let stopped1 = stream.stopped(); + let stopped2 = stream.stopped(); + let stopped3 = stream.stopped(); + + stream.write_all(b"hi").await.unwrap(); + // spawn one of the futures into a task + let stopped1 = tokio::task::spawn(stopped1); + // verify that both futures resolved + let (stopped1, stopped2) = tokio::join!(stopped1, stopped2); + assert!(matches!(stopped1, Ok(Ok(Some(val))) if val == 42u32.into())); + assert!(matches!(stopped2, Ok(Some(val)) if val == 42u32.into())); + // drop the stream + drop(stream); + // verify that a future also resolves after dropping the stream + let stopped3 = stopped3.await; + assert_eq!(stopped3, Ok(Some(42u32.into()))); + }; + let client = timeout(Duration::from_millis(100), client).instrument(error_span!("client")); + let server = async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + let mut buf = [0u8; 2]; + stream.read_exact(&mut buf).await.unwrap(); + stream.stop(42u32.into()).unwrap(); + conn + } + .instrument(error_span!("server")); + let (client, conn) = tokio::join!(client, server); + client.expect("timeout"); + drop(conn); +} + +#[tokio::test] +async fn stream_stopped_2() { + let _guard = subscribe(); + let endpoint = endpoint(); + + let (conn, _server_conn) = tokio::try_join!( + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap(), + async { endpoint.accept().await.unwrap().await } + ) + .unwrap(); + let send_stream = conn.open_uni().await.unwrap(); + let stopped = timeout(Duration::from_millis(100), send_stream.stopped()) + .instrument(error_span!("stopped")); + tokio::pin!(stopped); + // poll the future once so that the waker is registered. + tokio::select! { + biased; + _x = &mut stopped => {}, + _x = std::future::ready(()) => {} + } + // drop the send stream + drop(send_stream); + // make sure the stopped future still resolves + let res = stopped.await; + assert_eq!(res, Ok(Ok(None))); +} + +#[tokio::test] +async fn stream_drop_removes_blocked_reader() { + let _guard = subscribe(); + + for drop_stream in [false, true] { + let endpoint_factory = EndpointFactory::new(); + let server = endpoint_factory.endpoint(); + let server_address = server.local_addr().unwrap(); + let client = endpoint_factory.endpoint(); + + let server_task = tokio::spawn(async move { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut stream = conn.accept_uni().await.unwrap(); + + // read "hello" + let mut buf = [0u8; 5]; + stream.read_exact(&mut buf).await.unwrap(); + + let (waker, wake_counter) = new_count_waker(); + let mut cx = Context::from_waker(&waker); + // do a blocking read which will add the stream in conn.blocked_readers + { + let mut buf = [0u8; 64]; + let read_fut = stream.read(&mut buf); + tokio::pin!(read_fut); + assert!(matches!(read_fut.as_mut().poll(&mut cx), Poll::Pending)); + } + + if !drop_stream { + assert_eq!(wake_counter.wakes(), 0); + // We have a blocked reader, closing the connection should wake it. We use this as + // a proxy to assert that the stream is in conn.blocked_readers. + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 1); + } else { + // dropping the stream should remove it from conn.blocked_readers, so we don't + // expect any wakeups + drop(stream); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + conn.close(0u32.into(), b"done"); + assert_eq!(wake_counter.wakes(), 0, "no wakeups should have occurred"); + } + }); + + let conn = client + .connect(server_address, "localhost") + .unwrap() + .await + .unwrap(); + let mut stream = conn.open_uni().await.unwrap(); + // need to send some data to actually start the stream + stream.write_all(b"hello").await.unwrap(); + + server_task.await.unwrap(); + } +} + +/// Test that dropping a `RecvStream` after cancelling a read and then +/// explicitly `stop`ing it doesn't panic. +#[tokio::test] +async fn recv_stream_cancel_stop_drop() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = { + let _guard = error_span!("server").entered(); + factory.endpoint() + }; + let server_addr = server.local_addr().unwrap(); + + let client = { + let _guard = error_span!("client").entered(); + factory.endpoint() + }; + let recv_dropped = tokio::sync::SetOnce::new(); + join!( + async { + let conn = server.accept().await.unwrap().await.unwrap(); + let mut recv = conn.accept_uni().await.unwrap(); + // Create a future to read from the stream, poll it once, then immediately drop it + { + let fut = pin!(recv.read_to_end(usize::MAX)); + let mut cx = Context::from_waker(Waker::noop()); + assert!(fut.poll(&mut cx).is_pending()); + } + recv_dropped.set(()).unwrap(); + recv.stop(0u32.into()).unwrap(); + }, + async { + let conn = client + .connect(server_addr, "localhost") + .unwrap() + .await + .unwrap(); + let mut send = conn.open_uni().await.unwrap(); + _ = send.write_all(b"hello").await; + // Don't drop (finish) the send stream until the read has been + // cancelled by the server, ensuring that read_to_end can't complete + // immediately. + recv_dropped.wait().await; + }, + ); +} + +#[tokio::test(start_paused = true)] +async fn dropped_endpoint_cleans_up() { + let _guard = subscribe(); + + let mut endpoint_factory = EndpointFactory::new(); + let cid_generator = Arc::new(|| -> Box { + Box::::default() + }); + endpoint_factory + .endpoint_config + .cid_generator(cid_generator.clone()); + let endpoint = endpoint_factory.endpoint(); + drop(endpoint_factory); + assert_eq!(Arc::strong_count(&cid_generator), 2); + drop(endpoint); + // Let the driver task run; paused runtimes are guaranteed to drain pending work on sleep. + sleep(Duration::from_millis(1)).await; + assert_eq!(Arc::strong_count(&cid_generator), 1); +} + +#[tokio::test] +async fn dropped_connection_cleans_up() { + let _guard = subscribe(); + let endpoint = endpoint(); + join!( + async { + endpoint + .connect(endpoint.local_addr().unwrap(), "localhost") + .unwrap() + .await + .unwrap() + }, + async { endpoint.accept().await.unwrap().await.unwrap() } + ); + endpoint.wait_idle().await; +} + +#[derive(Default)] +struct WakeCounter { + wakes: AtomicUsize, +} + +impl WakeCounter { + fn wakes(&self) -> usize { + self.wakes.load(Ordering::SeqCst) + } +} + +fn new_count_waker() -> (Waker, Arc) { + // instance of WakeCounter + let counter = Arc::new(WakeCounter::default()); + + // convert + let waker = unsafe { Waker::from_raw(raw_waker(counter.clone())) }; + (waker, counter) +} + +fn raw_waker(counter: Arc) -> RawWaker { + // Store an Arc behind the raw pointer. + let ptr = Arc::into_raw(counter) as *const (); + RawWaker::new(ptr, &VTABLE) +} + +static VTABLE: RawWakerVTable = + RawWakerVTable::new(clone_waker, wake_waker, wake_by_ref_waker, drop_waker); + +unsafe fn clone_waker(data: *const ()) -> RawWaker { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + let cloned = arc.clone(); + std::mem::forget(arc); + raw_waker(cloned) +} + +unsafe fn wake_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + // arc drops here +} + +unsafe fn wake_by_ref_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + let arc = unsafe { Arc::::from_raw(data as *const WakeCounter) }; + arc.wakes.fetch_add(1, Ordering::SeqCst); + std::mem::forget(arc); +} + +unsafe fn drop_waker(data: *const ()) { + // SAFETY: pointer always comes from `Arc::into_raw()` (see `raw_waker()`) + drop(unsafe { Arc::::from_raw(data as *const WakeCounter) }); +} diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs new file mode 100644 index 0000000000..c3c3d3551d --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -0,0 +1,224 @@ +use crate::{Duration, Instant}; + +/// Limits the amount of time spent on a certain type of work in a cycle +/// +/// The limiter works dynamically: For a sampled subset of cycles it measures +/// the time that is approximately required for fulfilling 1 work item, and +/// calculates the amount of allowed work items per cycle. +/// The estimates are smoothed over all cycles where the exact duration is measured. +/// +/// In cycles where no measurement is performed the previously determined work limit +/// is used. +/// +/// For the limiter the exact definition of a work item does not matter. +/// It could for example track the amount of transmitted bytes per cycle, +/// or the amount of transmitted datagrams per cycle. +/// It will however work best if the required time to complete a work item is +/// constant. +#[derive(Debug)] +pub(crate) struct WorkLimiter { + /// Whether to measure the required work time, or to use the previous estimates + mode: Mode, + /// The current cycle number + cycle: u16, + /// The time the cycle started - only used in measurement mode + start_time: Option, + /// How many work items have been completed in the cycle + completed: usize, + /// The amount of work items which are allowed for a cycle + allowed: usize, + /// The desired cycle time + desired_cycle_time: Duration, + /// The estimated and smoothed time per work item in nanoseconds + smoothed_time_per_work_item_nanos: f64, +} + +impl WorkLimiter { + pub(crate) fn new(desired_cycle_time: Duration) -> Self { + Self { + mode: Mode::Measure, + cycle: 0, + start_time: None, + completed: 0, + allowed: 0, + desired_cycle_time, + smoothed_time_per_work_item_nanos: 0.0, + } + } + + /// Starts one work cycle + pub(crate) fn start_cycle(&mut self, now: impl Fn() -> Instant) { + self.completed = 0; + if let Mode::Measure = self.mode { + self.start_time = Some(now()); + } + } + + /// Returns whether more work can be performed inside the `desired_cycle_time` + /// + /// Requires that previous work was tracked using `record_work`. + pub(crate) fn allow_work(&mut self, now: impl Fn() -> Instant) -> bool { + match self.mode { + Mode::Measure => (now() - self.start_time.unwrap()) < self.desired_cycle_time, + Mode::HistoricData => self.completed < self.allowed, + } + } + + /// Records that `work` additional work items have been completed inside the cycle + /// + /// Must be called between `start_cycle` and `finish_cycle`. + pub(crate) fn record_work(&mut self, work: usize) { + self.completed += work; + } + + /// Finishes one work cycle + /// + /// For cycles where the exact duration is measured this will update the estimates + /// for the time per work item and the limit of allowed work items per cycle. + /// The estimate is updated using the same exponential averaging (smoothing) + /// mechanism which is used for determining QUIC path rtts: The last value is + /// weighted by 1/8, and the previous average by 7/8. + pub(crate) fn finish_cycle(&mut self, now: impl Fn() -> Instant) { + // If no work was done in the cycle drop the measurement, it won't be useful + if self.completed == 0 { + return; + } + + if let Mode::Measure = self.mode { + let elapsed = now() - self.start_time.unwrap(); + + let time_per_work_item_nanos = (elapsed.as_nanos()) as f64 / self.completed as f64; + + // Calculate the time per work item. We set this to at least 1ns to avoid + // dividing by 0 when calculating the allowed amount of work items. + self.smoothed_time_per_work_item_nanos = if self.allowed == 0 { + // Initial estimate + time_per_work_item_nanos + } else { + // Smoothed estimate + (7.0 * self.smoothed_time_per_work_item_nanos + time_per_work_item_nanos) / 8.0 + } + .max(1.0); + + // Allow at least 1 work item in order to make progress + self.allowed = (((self.desired_cycle_time.as_nanos()) as f64 + / self.smoothed_time_per_work_item_nanos) as usize) + .max(1); + self.start_time = None; + } + + self.cycle = self.cycle.wrapping_add(1); + self.mode = match self.cycle % SAMPLING_INTERVAL { + 0 => Mode::Measure, + _ => Mode::HistoricData, + }; + } +} + +/// We take a measurement sample once every `SAMPLING_INTERVAL` cycles +const SAMPLING_INTERVAL: u16 = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + Measure, + HistoricData, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[test] + fn limit_work() { + const CYCLE_TIME: Duration = Duration::from_millis(500); + const BATCH_WORK_ITEMS: usize = 12; + const BATCH_TIME: Duration = Duration::from_millis(100); + + const EXPECTED_INITIAL_BATCHES: usize = + (CYCLE_TIME.as_nanos() / BATCH_TIME.as_nanos()) as usize; + const EXPECTED_ALLOWED_WORK_ITEMS: usize = EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS; + + let mut limiter = WorkLimiter::new(CYCLE_TIME); + reset_time(); + + // The initial cycle is measuring + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, EXPECTED_ALLOWED_WORK_ITEMS); + let initial_time_per_work_item = limiter.smoothed_time_per_work_item_nanos; + + // The next cycles are using historic data + const BATCH_SIZES: [usize; 4] = [1, 2, 3, 5]; + for &batch_size in &BATCH_SIZES { + limiter.start_cycle(get_time); + let mut allowed_work = 0; + while limiter.allow_work(get_time) { + limiter.record_work(batch_size); + allowed_work += batch_size; + } + limiter.finish_cycle(get_time); + + assert_eq!(allowed_work, EXPECTED_ALLOWED_WORK_ITEMS); + } + + // After `SAMPLING_INTERVAL`, we get into measurement mode again + for _ in 0..(SAMPLING_INTERVAL as usize - BATCH_SIZES.len() - 1) { + limiter.start_cycle(get_time); + limiter.record_work(1); + limiter.finish_cycle(get_time); + } + + // We now do more work per cycle, and expect the estimate of allowed + // work items to go up + const BATCH_WORK_ITEMS_2: usize = 96; + const TIME_PER_WORK_ITEMS_2_NANOS: f64 = + CYCLE_TIME.as_nanos() as f64 / (EXPECTED_INITIAL_BATCHES * BATCH_WORK_ITEMS_2) as f64; + + let expected_updated_time_per_work_item = + (initial_time_per_work_item * 7.0 + TIME_PER_WORK_ITEMS_2_NANOS) / 8.0; + let expected_updated_allowed_work_items = + (CYCLE_TIME.as_nanos() as f64 / expected_updated_time_per_work_item) as usize; + + limiter.start_cycle(get_time); + let mut initial_batches = 0; + while limiter.allow_work(get_time) { + limiter.record_work(BATCH_WORK_ITEMS_2); + advance_time(BATCH_TIME); + initial_batches += 1; + } + limiter.finish_cycle(get_time); + + assert_eq!(initial_batches, EXPECTED_INITIAL_BATCHES); + assert_eq!(limiter.allowed, expected_updated_allowed_work_items); + } + + thread_local! { + /// Mocked time + pub static TIME: RefCell = RefCell::new(Instant::now()); + } + + fn reset_time() { + TIME.with(|t| { + *t.borrow_mut() = Instant::now(); + }) + } + + fn get_time() -> Instant { + TIME.with(|t| *t.borrow()) + } + + fn advance_time(duration: Duration) { + TIME.with(|t| { + *t.borrow_mut() += duration; + }) + } +} diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs new file mode 100644 index 0000000000..c3dcee477b --- /dev/null +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -0,0 +1,378 @@ +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use crate::proto::ConnectRequest; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use noq::crypto::rustls::QuicClientConfig; +use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; +use tokio::net::lookup_host; +use url::Host; + +use crate::crypto; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::ALPN; +use crate::{ClientError, Session}; + +/// Congestion control algorithm to use for the connection. +/// +/// Different algorithms make different tradeoffs between throughput and latency. +pub enum CongestionControl { + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Client] using sane defaults. +/// +/// This is optional; advanced users may use [Client::new] directly. +#[derive(Clone)] +pub struct ClientBuilder { + provider: crypto::Provider, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ClientBuilder { + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self + .builder() + .with_root_certificates(roots) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates( + self, + certs: Vec, + ) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes( + self, + hashes: Vec>, + ) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); + + let mut transport = noq::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ClientBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Builder for dangerous TLS configuration options. +/// +/// This builder provides access to potentially insecure TLS configurations. +/// These options should only be used when you understand the security implications, +/// such as in local development or over a secure VPN connection. +pub struct DangerousClientBuilder { + inner: ClientBuilder, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl DangerousClientBuilder { + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } +} + +/// A client for connecting to a WebTransport server. +#[derive(Clone, Debug)] +pub struct Client { + endpoint: noq::Endpoint, + config: noq::ClientConfig, +} + +impl Client { + /// Manually create a client via a Noq endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect( + &self, + request: impl Into, + ) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self + .endpoint + .connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for Client { + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } +} + +#[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] +#[derive(Debug)] +struct ServerFingerprints { + provider: crypto::Provider, + fingerprints: Vec>, +} + +impl ServerCertVerifier for ServerFingerprints { + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.provider.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider + .signature_verification_algorithms + .supported_schemes() + } +} + +#[derive(Debug)] +pub struct NoCertificateVerification(Arc); + +impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs new file mode 100644 index 0000000000..4307355d3f --- /dev/null +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -0,0 +1,158 @@ +use std::ops::Deref; + +use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum ConnectError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), + + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), + + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), +} + +/// An HTTP/3 CONNECT request/response for establishing a WebTransport session. +pub struct Connecting { + // The request that was sent by the client. + pub request: ConnectRequest, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connecting { + pub async fn accept(conn: &noq::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { + request, + send, + recv, + }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond( + mut self, + response: impl Into, + ) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol { + if !self.request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } +} + +impl Deref for Connecting { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.request + } +} + +pub struct Connected { + // The request that was sent by the client. + pub request: ConnectRequest, + + // The response sent by the server. + pub response: ConnectResponse, + + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: noq::SendStream, + pub(crate) recv: noq::RecvStream, +} + +impl Connected { + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol { + if !request.protocols.contains(protocol) { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. + // We don't use the noq::VarInt because that would mean a noq dependency in + // web-transport-proto + let stream_id = noq::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } +} diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs new file mode 100644 index 0000000000..1b8bb8ecf1 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -0,0 +1,67 @@ +//! Simple crypto provider utilities for rustls. +//! +//! This module provides helper functions for working with rustls crypto providers, +//! supporting both ring and aws-lc-rs backends. + +use std::sync::Arc; + +use rustls::crypto::hash::{self, HashAlgorithm}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::CertificateDer; + +/// A shared reference to a crypto provider. +pub type Provider = Arc; + +/// Returns the default crypto provider. +/// +/// This function checks for a process-wide default provider first, +/// then falls back to feature-enabled providers (aws-lc-rs or ring). +/// +/// # Panics +/// +/// Panics if no provider is available. Either call `CryptoProvider::set_default()` +/// or enable exactly one of the `ring` or `aws-lc-rs` features. +pub fn default_provider() -> Provider { + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } + + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!( + "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." + ); + } +} + +/// Computes the SHA-256 hash of a certificate using the provided crypto provider. +/// +/// # Panics +/// +/// Panics if the provider doesn't expose a SHA-256 hash algorithm. +pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } + + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); +} diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs new file mode 100644 index 0000000000..654260fc4c --- /dev/null +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -0,0 +1,405 @@ +use std::sync::{Arc, OnceLock}; + +use thiserror::Error; + +use crate::{ConnectError, SettingsError}; + +/// An error returned when connecting to a WebTransport endpoint. +#[derive(Error, Debug, Clone)] +pub enum ClientError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error: {0}")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write: {0}")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read: {0}")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), + + #[error("quic error: {0}")] + NoqError(#[from] noq::ConnectError), + + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// WebTransport errors. +#[derive(Clone, Error, Debug)] +pub enum SessionError { + #[error("connection error: {0}")] + ConnectionError(noq::ConnectionError), + + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), + + #[error("send datagram error: {0}")] + SendDatagramError(#[from] noq::SendDatagramError), +} + +impl From for SessionError { + fn from(e: noq::ConnectionError) -> Self { + match &e { + noq::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } +} + +/// Why a session closed, shared by the session and its streams. The first close wins. +/// +/// It also knows whether the session is raw QUIC, which decides how its close and stream +/// codes are encoded. +#[derive(Debug)] +pub(crate) struct CloseReason { + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, +} + +impl CloseReason { + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { + if self.raw { + return code.into(); + } + noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { + Some(conn) + } + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => WebTransportError::Closed( + code, + String::from_utf8_lossy(&close.reason).into_owned(), + ) + .into(), + Err(_) => err, + } + } + _ => err, + } + } +} + +/// An error that can occur when reading/writing the WebTransport stream header. +#[derive(Clone, Error, Debug)] +pub enum WebTransportError { + #[error("closed: code={0} reason={1}")] + Closed(u32, String), + + #[error("unknown session")] + UnknownSession, + + #[error("read error: {0}")] + ReadError(#[from] noq::ReadExactError), + + #[error("write error: {0}")] + WriteError(#[from] noq::WriteError), +} + +/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +#[derive(Clone, Error, Debug)] +pub enum WriteError { + #[error("STOP_SENDING: {0}")] + Stopped(u32), + + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(noq::VarInt), + + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("stream closed")] + ClosedStream, +} + +impl From for WriteError { + fn from(e: noq::WriteError) -> Self { + match e { + noq::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + noq::WriteError::ClosedStream => WriteError::ClosedStream, + noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadError { + #[error("session error: {0}")] + SessionError(#[from] SessionError), + + #[error("RESET_STREAM: {0}")] + Reset(u32), + + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(noq::VarInt), + + #[error("stream already closed")] + ClosedStream, +} + +impl From for ReadError { + fn from(value: noq::ReadError) -> Self { + match value { + noq::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + noq::ReadError::ClosedStream => ReadError::ClosedStream, + noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } +} + +/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadExactError { + #[error("finished early")] + FinishedEarly(usize), + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadExactError { + fn from(e: noq::ReadExactError) -> Self { + match e { + noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } +} + +/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +#[derive(Clone, Error, Debug)] +pub enum ReadToEndError { + #[error("too long")] + TooLong, + + #[error("read error: {0}")] + ReadError(#[from] ReadError), +} + +impl From for ReadToEndError { + fn from(e: noq::ReadToEndError) -> Self { + match e { + noq::ReadToEndError::TooLong => ReadToEndError::TooLong, + noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } +} + +/// An error indicating the stream was already closed. +#[derive(Clone, Error, Debug)] +#[error("stream closed")] +pub struct ClosedStream; + +impl From for ClosedStream { + fn from(_: noq::ClosedStream) -> Self { + ClosedStream + } +} + +/// An error returned when receiving a new WebTransport session. +#[derive(Error, Debug, Clone)] +pub enum ServerError { + #[error("unexpected end of stream")] + UnexpectedEnd, + + #[error("connection error")] + Connection(#[from] noq::ConnectionError), + + #[error("failed to write")] + WriteError(#[from] noq::WriteError), + + #[error("failed to read")] + ReadError(#[from] noq::ReadError), + + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), + + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), + + #[error("io error: {0}")] + IoError(Arc), + + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), +} + +// #[derive(Clone, Error, Debug)] +// pub enum SendDatagramError { +// #[error("Unsupported peer")] +// UnsupportedPeer, + +// #[error("Datagram support Disabled by peer")] +// DatagramSupportDisabled, + +// #[error("Datagram Too large")] +// TooLarge, + +// #[error("Session errorr: {0}")] +// SessionError(#[from] SessionError), +// } + +// impl From for SendDatagramError { +// fn from(value: noq::SendDatagramError) -> Self { +// match value { +// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// noq::SendDatagramError::ConnectionLost(e) => +// SendDatagramError::SessionError(e.into()), } +// } +// } + +impl web_transport_trait::Error for SessionError { + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } + + None + } +} + +impl web_transport_trait::Error for WriteError { + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } +} + +impl web_transport_trait::Error for ReadError { + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs new file mode 100644 index 0000000000..e5f3706492 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -0,0 +1,71 @@ +//! WebTransport is a protocol for client-server communication over QUIC. +//! It's [available in the browser](https://caniuse.com/webtransport) as an alternative to HTTP and WebSockets. +//! +//! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. +//! This library hides that detail and tries to expose only the QUIC API, delegating as much as +//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! +//! QUIC provides two primary APIs: +//! +//! # Streams +//! QUIC streams are ordered, reliable, flow-controlled, and optionally bidirectional. +//! Both endpoints can create and close streams (including an error code) with no overhead. +//! You can think of them as TCP connections, but shared over a single QUIC connection. +//! +//! # Datagrams +//! QUIC datagrams are unordered, unreliable, and not flow-controlled. +//! Both endpoints can send datagrams below the MTU size (~1.2kb minimum) and they might arrive out +//! of order or not at all. They are basically UDP packets, except they are encrypted and congestion +//! controlled. +//! +//! # Limitations +//! WebTransport is able to be pooled with HTTP/3 and multiple WebTransport sessions. +//! This crate avoids that complexity, doing the bare minimum to support a single WebTransport +//! session that owns the entire QUIC connection. If you want to support HTTP/3 on the same +//! host/port, you should use another crate (ex. `h3-webtransport`). If you want to support multiple +//! WebTransport sessions over the same QUIC connection... you should just dial a new QUIC +//! connection instead. + +// External +mod client; +mod error; +mod recv; +mod send; +mod server; +mod session; + +pub use client::*; +pub use error::*; +pub use recv::*; +pub use send::*; +pub use server::*; +pub use session::*; + +// Internal +mod connect; +mod settings; + +use connect::*; +use settings::*; + +// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +pub use connect::ConnectError; + +/// The HTTP/3 ALPN is required when negotiating a QUIC connection. +pub const ALPN: &str = "h3"; + +/// Export our simple crypto provider. +pub mod crypto; + +/// Re-export the underlying QUIC implementation. +pub use noq; + +/// Re-export the http crate because it's in the public API. +pub use http; + +/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of +/// [`SessionAccept`]'s poll methods holds its place in the accept queue. +pub use kio; + +/// Re-export the WebTransport protocol implementation. +pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs new file mode 100644 index 0000000000..4c3d188172 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -0,0 +1,159 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::Bytes; + +use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; + +/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +#[derive(Debug)] +pub struct RecvStream { + inner: noq::RecvStream, + close: Arc, +} + +impl RecvStream { + pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { + Self { + inner: stream, + close, + } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::ReadError) -> ReadError { + match e { + noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + noq::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. + pub async fn read_many_chunks( + &mut self, + bufs: &mut [Bytes], + ) -> Result, ReadError> { + self.inner + .read_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner + .read_to_end(size_limit) + .await + .map_err(|e| match e { + noq::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`noq::RecvStream::received_reset`]. + /// + /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.inner.id() + } + + /// Returns the number of bytes read from this stream. + /// + /// This is the offset of the next byte to be read, i.e. the length of the contiguous + /// prefix of the stream consumed by the application. + pub fn bytes_read(&self) -> Result { + self.inner.bytes_read().map_err(|_| ClosedStream) + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport +} + +impl tokio::io::AsyncRead for RecvStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl web_transport_trait::RecvStream for RecvStream { + type Error = ReadError; + + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } + + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } + + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } +} diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs new file mode 100644 index 0000000000..39af0d1fc7 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -0,0 +1,197 @@ +use std::{ + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::{Buf, Bytes}; + +use crate::{CloseReason, ClosedStream, SessionError, WriteError}; + +/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// +/// This wrapper is mainly needed for error codes, which is unfortunate. +/// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. +#[derive(Debug)] +pub struct SendStream { + stream: noq::SendStream, + close: Arc, +} + +impl SendStream { + pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: noq::WriteError) -> WriteError { + match e { + noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`noq::SendStream::stopped`]. + /// + /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. + /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(noq::StoppedError::ConnectionLost(conn_err)) => { + Err(self.close.map(conn_err.into())) + } + Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream + .write_all(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write chunks of data to the stream, returning the number of bytes written. + /// + /// See [`noq::SendStream::write_many_chunks`]. + pub async fn write_many_chunks( + &mut self, + bufs: &mut &mut [Bytes], + ) -> Result { + self.stream + .write_many_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream + .write_chunk(buf) + .await + .map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream + .write_all_chunks(bufs) + .await + .map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`noq::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> noq::StreamId { + self.stream.id() + } +} + +impl tokio::io::AsyncWrite for SendStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + // We have to use this syntax because noq added its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + +impl web_transport_trait::SendStream for SendStream { + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + // NOTE: This used to require &mut in an older version of Noq. + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } +} diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs new file mode 100644 index 0000000000..8af3e81c98 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -0,0 +1,200 @@ +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use std::sync::Arc; + +use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +use crate::{crypto, CongestionControl}; +use crate::{ + proto::{ConnectRequest, ConnectResponse}, + Connecting, ServerError, Session, Settings, +}; + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +/// Construct a WebTransport [Server] using sane defaults. +/// +/// This is optional; advanced users may use [Server::new] directly. +pub struct ServerBuilder { + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: + Option>, +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl Default for ServerBuilder { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +impl ServerBuilder { + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => { + Some(Arc::new(noq::congestion::CubicConfig::default())) + } + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard Noq setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important + + let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = noq::ServerConfig::with_crypto(Arc::new(config)); + + let server = + noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } +} + +/// A WebTransport server that accepts new sessions. +pub struct Server { + endpoint: noq::Endpoint, + accept: FuturesUnordered>>, +} + +impl core::ops::Deref for Server { + type Target = noq::Endpoint; + + fn deref(&self) -> &Self::Target { + &self.endpoint + } +} + +impl Server { + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. + pub fn new(endpoint: noq::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } +} + +/// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to +/// accept or reject the session based on the URL. +pub struct Request { + conn: noq::Connection, + settings: Settings, + connect: Connecting, +} + +impl Request { + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: noq::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond( + self, + response: impl Into, + ) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &noq::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } +} + +impl core::ops::Deref for Request { + type Target = ConnectRequest; + + fn deref(&self) -> &Self::Target { + &self.connect + } +} diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs new file mode 100644 index 0000000000..82a9149d84 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -0,0 +1,1331 @@ +use std::{ + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use bytes::{Bytes, BytesMut}; +use futures::stream::{FuturesUnordered, Stream, StreamExt}; +use kio::{Fan, Waiter}; + +use crate::{ + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, + WebTransportError, +}; + +/// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake +/// has not completed, or it is not valid UTF-8. +/// +/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// is owned rather than borrowed from the connection. +#[cfg(any(feature = "aws-lc-rs", feature = "ring"))] +fn negotiated_alpn(conn: &noq::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data.downcast::().ok()?; + + String::from_utf8(data.protocol?).ok() +} + +/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// handshake data type to name here. +/// +/// This crate's own builders are gated the same way, so normally there is no connection +/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// brings its own endpoint can still establish one, and [`Session::protocol`] reports +/// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to +/// get the ALPN. +#[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] +fn negotiated_alpn(_conn: &noq::Connection) -> Option { + None +} + +/// An established WebTransport session, acting like a full QUIC connection. See +/// [`noq::Connection`]. +/// +/// It is important to remember that WebTransport is layered on top of QUIC: +/// 1. Each stream starts with a few bytes identifying the stream type and session ID. +/// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. +/// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. +/// +/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// These should be safe to use with WebTransport, but file a PR if you find one that isn't. +#[derive(Clone)] +pub struct Session { + conn: noq::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, +} + +impl Session { + pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = noq::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { + code, + reason, + })) => return Some((code, reason)), + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: noq::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self + .conn + .accept_uni() + .await + .map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self + .conn + .read_datagram() + .await + .map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = + VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the Noq API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = + web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .unwrap_or(Duration::from_millis(100)); + let timeout = (rtt * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session + .close_with_capsule(send, capsule, code, timeout) + .await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: noq::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) + .try_into() + .unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!( + ?e, + "connection ended before the close capsule was acknowledged" + ); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()) + .await + .is_err() + { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`noq::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`noq::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: noq::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + let path = self.conn.path_stats(noq::PathId::ZERO); + let rtt = self + .conn + .rtt(noq::PathId::ZERO) + .or_else(|| path.map(|p| p.rtt)); + SessionStats { + stats: self.conn.stats(), + path, + rtt, + } + } +} + +impl Deref for Session { + type Target = noq::Connection; + + fn deref(&self) -> &Self::Target { + &self.conn + } +} + +impl fmt::Debug for Session { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } +} + +impl PartialEq for Session { + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } +} + +impl Eq for Session {} + +// Poll the shared accept state, then wake the *other* accepters once the lock is +// released. +// +// `SessionAccept` does not wake them itself: a waker is free to resume its task +// inline, and the first thing a resumed accepter does is take this same lock. An +// arrival or a failure is exactly what the others are parked waiting to retry +// after, so `Ready` is the signal. +fn poll_accept_uni_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +fn poll_accept_bi_shared( + accept: &Mutex, + waiter: &Waiter, +) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result +} + +// Type aliases just so clippy doesn't complain about the complexity. +type AcceptUni = dyn Stream> + Send; +type AcceptBi = + dyn Stream> + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = + dyn Future, SessionError>> + Send; + +// Logic just for accepting streams, which is annoying because of the stream header. +pub struct SessionAccept { + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, +} + +impl SessionAccept { + pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::new(); + let uni_waiters = Fan::new(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, noq::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi( + &mut self, + waiter: &Waiter, + ) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: noq::SendStream, + mut recv: noq::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } +} + +// Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any +// failure as a truncation, which would hide a stream the peer reset before its header +// arrived. +async fn read_varint(recv: &mut noq::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; + + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; + + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) +} + +// Without reliable reset, a stream the peer resets early loses its header with it, which +// moq does routinely. Anything else is a misbehaving peer. +fn log_header_error(err: SessionError, direction: &'static str) { + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError( + noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } +} + +pub struct SessionStats { + stats: noq::ConnectionStats, + path: Option, + rtt: Option, +} + +impl web_transport_trait::Stats for SessionStats { + fn bytes_sent(&self) -> Option { + Some(self.stats.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.stats.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + self.path.map(|path| path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.stats.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.stats.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + self.path.map(|path| path.lost_packets) + } + + fn rtt(&self) -> Option { + self.rtt + } + + fn estimated_send_rate(&self) -> Option { + let path = self.path?; + let rtt = self.rtt?; + let rtt_secs = rtt.as_secs_f64(); + if path.cwnd > 0 && rtt_secs > 0.0 { + Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) + } else { + None + } + } +} + +impl web_transport_trait::Session for Session { + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } +} + +#[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] +mod tests { + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl noq::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &noq::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl noq::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [noq::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + noq::Connection, + noq::Connection, + [noq::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( + &signing_key, + ))); + + let server_config = + noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let runtime: Arc = Arc::new(noq::TokioRuntime); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: runtime.wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = noq::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + runtime, + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { + crate::Request::accept(server_conn) + .await + .unwrap() + .ok() + .await + }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION) + .await + .err() + .unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION) + .await + .err() + .unwrap(); + assert!( + matches!( + err, + SessionError::WebTransportError(WebTransportError::UnknownSession) + ), + "{err:?}" + ); + } +} diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs new file mode 100644 index 0000000000..08ea9a53a1 --- /dev/null +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -0,0 +1,70 @@ +use futures::try_join; + +use thiserror::Error; + +#[derive(Error, Debug, Clone)] +pub enum SettingsError { + #[error("quic stream was closed early")] + UnexpectedEnd, + + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), + + #[error("WebTransport is not supported")] + WebTransportUnsupported, + + #[error("connection error")] + ConnectionError(#[from] noq::ConnectionError), + + #[error("read error")] + ReadError(#[from] noq::ReadError), + + #[error("write error")] + WriteError(#[from] noq::WriteError), +} + +pub struct Settings { + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: noq::SendStream, + + #[allow(dead_code)] + recv: noq::RecvStream, +} + +impl Settings { + // Establish the H3 connection. + pub async fn connect(conn: &noq::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); + + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } + + async fn accept(conn: &noq::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; + + tracing::debug!(?settings, "received SETTINGS frame"); + + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } + + Ok(recv) + } + + async fn open(conn: &noq::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); + + tracing::debug!(?settings, "sending SETTINGS frame"); + + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; + + Ok(send) + } +} From e8195bdb5952909497cf01bab7ea514c4eaf519f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:35:17 -0700 Subject: [PATCH 03/17] feat(quic)!: run moq-tokio and moq-uring on moq-quic Drop every moq-noq dependency. moq-tokio's QUIC backend now drives moq-quic through quinn's async layer (`quic::driver`) with web-transport-moq folded in as `quic::web`; moq-uring's sans-IO driver moves from moq-noq-proto to moq-quic. Upstream noq stays only behind moq-tokio's `iroh` feature. - The `noq` cargo feature is `quic` everywhere it was forwarded. - `moq_tokio::noq::Error` is `moq_tokio::quic::Error`; `Error::Noq` is `Error::Quic`. - moq-quic takes `moq_clock::Instant` and raises a once-per-connection Draining endpoint event, which backs the driver's `wait_all_draining`. - qlog writes one trace per endpoint (per dial on moq-uring's connect side), since moq-quic holds one sink per transport config. - Session stats report the congestion controller's bandwidth estimate. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 114 +- Cargo.toml | 12 +- doc/bin/cli.md | 2 +- doc/bin/relay/config.md | 2 +- doc/setup/upgrade.md | 7 + rs/hang/Cargo.toml | 2 +- rs/justfile | 10 +- rs/moq-bench/Cargo.toml | 4 +- rs/moq-boy/Cargo.toml | 4 +- rs/moq-c/Cargo.toml | 2 +- rs/moq-cli/Cargo.toml | 8 +- rs/moq-gst/src/lib.rs | 2 +- rs/moq-hls/Cargo.toml | 2 +- rs/moq-quic/Cargo.toml | 1 + rs/moq-quic/README.md | 15 +- rs/moq-quic/src/config/transport.rs | 15 +- rs/moq-quic/src/connection/mod.rs | 15 + rs/moq-quic/src/connection/qlog.rs | 16 +- rs/moq-quic/src/endpoint.rs | 1 + rs/moq-quic/src/lib.rs | 6 +- rs/moq-quic/src/shared.rs | 11 + rs/moq-quic/src/tests/mod.rs | 31 + rs/moq-quic/src/tests/util.rs | 15 + rs/moq-relay/Cargo.toml | 7 +- rs/moq-relay/src/config.rs | 2 +- rs/moq-relay/src/relay.rs | 6 +- rs/moq-relay/src/shutdown.rs | 2 +- rs/moq-relay/tests/cluster_idle.rs | 2 +- rs/moq-relay/tests/drills.rs | 2 +- rs/moq-relay/tests/goaway_cluster.rs | 2 +- rs/moq-relay/tests/runtime_uring.rs | 23 +- rs/moq-sock/src/bind.rs | 6 +- rs/moq-tokio/Cargo.toml | 39 +- rs/moq-tokio/README.md | 2 +- rs/moq-tokio/src/client.rs | 122 +- rs/moq-tokio/src/connect.rs | 4 +- rs/moq-tokio/src/connection.rs | 10 +- rs/moq-tokio/src/error.rs | 26 +- rs/moq-tokio/src/iroh.rs | 6 +- rs/moq-tokio/src/lib.rs | 12 +- rs/moq-tokio/src/listen.rs | 14 +- rs/moq-tokio/src/log.rs | 2 +- rs/moq-tokio/src/quic.rs | 33 +- rs/moq-tokio/src/{noq.rs => quic/backend.rs} | 289 +- rs/moq-tokio/src/quic/driver/.rustfmt.toml | 2 + rs/moq-tokio/src/quic/driver/connection.rs | 99 +- rs/moq-tokio/src/quic/driver/endpoint.rs | 178 +- rs/moq-tokio/src/quic/driver/incoming.rs | 8 +- rs/moq-tokio/src/quic/driver/mod.rs | 54 +- rs/moq-tokio/src/quic/driver/mutex.rs | 111 - rs/moq-tokio/src/quic/driver/recv_stream.rs | 59 +- rs/moq-tokio/src/quic/driver/runtime/mod.rs | 70 +- rs/moq-tokio/src/quic/driver/runtime/smol.rs | 105 - rs/moq-tokio/src/quic/driver/runtime/tokio.rs | 53 +- rs/moq-tokio/src/quic/driver/send_stream.rs | 31 +- rs/moq-tokio/src/quic/driver/tests.rs | 205 +- rs/moq-tokio/src/quic/driver/work_limiter.rs | 2 +- rs/moq-tokio/src/quic/web/client.rs | 577 ++-- rs/moq-tokio/src/quic/web/connect.rs | 238 +- rs/moq-tokio/src/quic/web/crypto.rs | 64 +- rs/moq-tokio/src/quic/web/error.rs | 547 ++-- rs/moq-tokio/src/quic/web/fan.rs | 140 + rs/moq-tokio/src/quic/web/mod.rs | 12 +- rs/moq-tokio/src/quic/web/recv.rs | 246 +- rs/moq-tokio/src/quic/web/send.rs | 323 +-- rs/moq-tokio/src/quic/web/server.rs | 302 +-- rs/moq-tokio/src/quic/web/session.rs | 2399 ++++++++--------- rs/moq-tokio/src/quic/web/settings.rs | 84 +- rs/moq-tokio/src/resolve.rs | 2 +- rs/moq-tokio/src/server.rs | 146 +- rs/moq-tokio/src/tcp.rs | 2 +- rs/moq-tokio/src/tls.rs | 34 +- rs/moq-tokio/src/transport.rs | 2 +- rs/moq-tokio/src/websocket.rs | 8 +- rs/moq-tokio/src/worker/group.rs | 8 +- rs/moq-tokio/src/worker/mod.rs | 4 +- rs/moq-tokio/tests/backend.rs | 100 +- rs/moq-tokio/tests/broadcast.rs | 2 +- rs/moq-tokio/tests/close_code.rs | 2 +- rs/moq-tokio/tests/stream_code.rs | 12 +- rs/moq-tokio/tests/worker.rs | 2 +- rs/moq-transcode/Cargo.toml | 2 +- rs/moq-uring/Cargo.toml | 23 +- rs/moq-uring/README.md | 24 +- .../benches/{echo_noq.rs => echo_quic.rs} | 10 +- rs/moq-uring/benches/session_lite.rs | 10 +- rs/moq-uring/src/lib.rs | 4 +- .../src/quic/{noq => driver}/connection.rs | 133 +- .../src/quic/{noq => driver}/endpoint.rs | 42 +- rs/moq-uring/src/quic/{noq => driver}/mod.rs | 134 +- .../src/quic/{noq => driver}/stream.rs | 38 +- rs/moq-uring/src/quic/endpoint.rs | 2 +- rs/moq-uring/src/quic/mod.rs | 17 +- rs/moq-uring/src/quic/qlog.rs | 37 +- rs/moq-uring/src/quic/web.rs | 2 +- rs/moq-uring/tests/echo.rs | 12 +- rs/moq-uring/tests/ecn.rs | 2 +- rs/moq-uring/tests/endpoint.rs | 10 +- rs/moq-uring/tests/identity.rs | 2 +- rs/moq-uring/tests/qlog.rs | 6 +- rs/moq-uring/tests/session.rs | 4 +- rs/moq-uring/tests/teardown.rs | 2 +- rs/moq-uring/tests/web.rs | 49 +- rs/moq-uring/tests/workers.rs | 2 +- sh/rs/select.test.py | 2 +- sh/rs/tokio-features.sh | 10 +- 106 files changed, 3656 insertions(+), 4064 deletions(-) rename rs/moq-tokio/src/{noq.rs => quic/backend.rs} (79%) create mode 100644 rs/moq-tokio/src/quic/driver/.rustfmt.toml delete mode 100644 rs/moq-tokio/src/quic/driver/runtime/smol.rs create mode 100644 rs/moq-tokio/src/quic/web/fan.rs rename rs/moq-uring/benches/{echo_noq.rs => echo_quic.rs} (93%) rename rs/moq-uring/src/quic/{noq => driver}/connection.rs (87%) rename rs/moq-uring/src/quic/{noq => driver}/endpoint.rs (92%) rename rs/moq-uring/src/quic/{noq => driver}/mod.rs (79%) rename rs/moq-uring/src/quic/{noq => driver}/stream.rs (90%) diff --git a/Cargo.lock b/Cargo.lock index 0447ee73e6..a2a305b404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4675,72 +4675,6 @@ dependencies = [ name = "moq-net-sim-macros" version = "0.0.0" -[[package]] -name = "moq-noq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20d9d1334f556a15e60f6d406020fc3a40409335ae8a2ad472f6bf074a01acfa" -dependencies = [ - "bytes", - "cfg_aliases", - "derive_more 2.1.1", - "moq-noq-proto", - "moq-noq-udp", - "pin-project-lite", - "rustc-hash 2.1.3", - "rustls", - "socket2 0.6.5", - "thiserror 2.0.21", - "tokio", - "tokio-stream", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-proto" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50ac444c6c1ed18d45536cd939be7f0fd664996e79c0519119b13d6ae9d3d5e2" -dependencies = [ - "aes-gcm", - "aws-lc-rs", - "bytes", - "derive_more 2.1.1", - "enum-assoc", - "fastbloom", - "getrandom 0.4.3", - "identity-hash", - "lru-slab", - "n0-qlog", - "rand 0.10.3", - "rand_pcg", - "ring", - "rustc-hash 2.1.3", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "slab", - "sorted-index-buffer", - "thiserror 2.0.21", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "moq-noq-udp" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e978cded333ca8a6d50b96a7a4388ccc8d98db51d89df7dbee598e9b849ee79" -dependencies = [ - "cfg_aliases", - "libc", - "socket2 0.6.5", - "tracing", - "windows-sys 0.61.2", -] - [[package]] name = "moq-nvenc" version = "0.1.2" @@ -4771,6 +4705,7 @@ dependencies = [ "getrandom 0.4.3", "hex-literal", "lru-slab", + "moq-clock", "qlog", "rand 0.10.3", "rand_pcg", @@ -4835,7 +4770,6 @@ dependencies = [ "tracing-subscriber", "url", "usage-rs", - "web-transport-moq", "web-transport-trait", ] @@ -4992,19 +4926,26 @@ dependencies = [ "futures", "hex", "hmac 0.13.0", + "http", "humantime", "jni 0.22.4", + "kio 0.6.2", "libc", "mdns-sd", + "moq-clock", "moq-net", + "moq-quic", "moq-sock", "noq-proto", "notify", + "pin-project-lite", "qmux", "rand 0.10.3", "rcgen", "reqwest", + "rustc-hash 2.1.3", "rustls", + "rustls-native-certs", "rustls-platform-verifier", "rustls-webpki", "serde", @@ -5027,7 +4968,6 @@ dependencies = [ "url", "usage-rs", "web-transport-iroh", - "web-transport-moq", "web-transport-proto", "web-transport-trait", "webpki-roots", @@ -5064,8 +5004,9 @@ dependencies = [ "io-uring", "kio 0.6.2", "libc", + "moq-clock", "moq-net", - "moq-noq-proto", + "moq-quic", "moq-sock", "moq-tokio", "nix", @@ -5081,7 +5022,6 @@ dependencies = [ "tokio", "tracing", "url", - "web-transport-moq", "web-transport-proto", "web-transport-trait", ] @@ -5292,19 +5232,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "n0-qlog" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1c2fc610d1facce51b0bf5d8245b309c91996a933defdf7742c92ac78a58cb" -dependencies = [ - "humantime", - "serde", - "serde_json", - "serde_with", - "strum", -] - [[package]] name = "n0-watcher" version = "1.0.0" @@ -10240,27 +10167,6 @@ dependencies = [ "web-transport-trait", ] -[[package]] -name = "web-transport-moq" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc946e718371f88a79d849c352b6bdd95e0a36139e9b76818c1a101b45ed97a" -dependencies = [ - "bytes", - "futures", - "http", - "kio 0.5.9", - "moq-noq", - "rustls", - "rustls-native-certs", - "thiserror 2.0.21", - "tokio", - "tracing", - "url", - "web-transport-proto", - "web-transport-trait", -] - [[package]] name = "web-transport-proto" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 19a3faf7ee..80f784458f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,7 @@ mdns-sd = { version = "0.21", features = ["async"] } moq-archive = { version = "0.0.11", path = "rs/moq-archive" } moq-audio = { version = "0.1.10", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.2.1", path = "rs/moq-auth" } +moq-clock = { version = "0.11.0", path = "rs/moq-clock" } moq-clock-macros = { version = "0.1.0", path = "rs/moq-clock/macros" } moq-flate = { version = "0.2.0", path = "rs/moq-flate" } moq-hls = { version = "0.5.11", path = "rs/moq-hls", default-features = false } @@ -157,14 +158,13 @@ moq-loc = { version = "0.2.19", path = "rs/moq-loc" } moq-msf = { version = "0.5.1", path = "rs/moq-msf" } moq-mux = { version = "0.10.11", path = "rs/moq-mux" } moq-net = { version = "0.3.10", path = "rs/moq-net" } -# The MoQ fork of noq (moq-dev/noq). iroh keeps upstream noq, so a build with the -# iroh feature carries both stacks. -moq-noq-proto = { version = "2.0.5", default-features = false } # NVENC bindings, forked from ViliamVadocz/nvidia-video-codec-sdk to dlopen the # driver at runtime. Compiles on any platform (macOS included) but only actually # used by moq-video on Linux. moq-nvenc = { version = "0.1.2", path = "rs/moq-nvenc" } moq-pattern = { version = "0.1.2", path = "rs/moq-pattern" } +# default-features off so each consumer picks its crypto provider. +moq-quic = { version = "0.1.0", path = "rs/moq-quic", default-features = false } moq-relay = { version = "0.17.2", path = "rs/moq-relay", default-features = false } moq-rtc = { version = "0.3.12", path = "rs/moq-rtc" } moq-rtmp = { version = "0.3.11", path = "rs/moq-rtmp" } @@ -178,7 +178,7 @@ moq-tokio = { version = "0.19.23", path = "rs/moq-tokio", default-features = fal # VAAPI is opt-in everywhere; its decoder is hardware-validated, while its # encoder is not yet. moq-transcode = { version = "0.1.10", path = "rs/moq-transcode", default-features = false } -# default-features off (the noq backend) so the consumer picks which QUIC +# default-features off (the `quic` backend) so the consumer opts into the QUIC # stack the io_uring path compiles; cargo features are additive, so a default-on # backend could not be opted out of. moq-uring = { version = "0.0.12", path = "rs/moq-uring", default-features = false } @@ -240,10 +240,6 @@ usage = { package = "usage-rs", version = "6.11.1" } uuid = { version = "1.26", features = ["v7"] } web-async = { version = "0.1.5", features = ["tracing"] } web-transport-iroh = "0.8.3" -# default-features off so the QUIC crypto provider is chosen by moq-tokio's -# aws-lc-rs / ring features. -# The WebTransport adapter released with moq-noq, from the same repository. -web-transport-moq = { version = "2.0.5", default-features = false } web-transport-proto = "0.6" web-transport-trait = "0.5" web-transport-wasm = "0.8" diff --git a/doc/bin/cli.md b/doc/bin/cli.md index d127621494..ac4f645cf2 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -168,7 +168,7 @@ Playback is behind the `play` feature, since it pulls in windowing and audio-device dependencies: ```bash -cargo install moq-cli --no-default-features --features "noq,websocket,play" +cargo install moq-cli --no-default-features --features "quic,websocket,play" ``` `moq play` skips disabled renditions. If none are enabled, it waits for a later catalog. Disabling the current rendition stops it and selects an enabled replacement. diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index f4a5099962..9bd91a4a5e 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -122,7 +122,7 @@ io_uring = false # Drive them with io_uring instead of tokio ``` Packets are steered by connection ID, so a client that migrates stays with its -worker. `workers` needs the `noq` feature. Every worker and the TLS stream +worker. `workers` needs the `quic` feature. Every worker and the TLS stream listener serve one shared identity, generated or file-backed, so a certificate rotation reaches them together and `/certificate.sha256` stays accurate. diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index dd5c605fdc..e0e6502d99 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -23,6 +23,13 @@ These land with the next breaking release, not the 2026-09-23 train. with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` orders audio by enabled state, bitrate, sample rate, channel count, and name. Both helpers put disabled renditions last. +- **The `noq` cargo feature is `quic`.** Rename it wherever you enable it on + moq-tokio, moq-uring, moq-relay, moq-cli, or the other crates that forward + it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` + crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and + `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one + file per endpoint instead of one per connection; on io_uring workers, one per + endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC `idle_timeout` and `keep_alive`; backoff `initial`, `max`, and `timeout`; diff --git a/rs/hang/Cargo.toml b/rs/hang/Cargo.toml index dd2f9d495b..24dfae2525 100644 --- a/rs/hang/Cargo.toml +++ b/rs/hang/Cargo.toml @@ -33,7 +33,7 @@ features = ["from", "display", "debug"] [dev-dependencies] anyhow = { workspace = true } moq-mux = { path = "../moq-mux" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } tokio = { workspace = true, features = ["macros", "fs"] } tracing = { workspace = true } url = { workspace = true } diff --git a/rs/justfile b/rs/justfile index 8f1bdf33cc..92fd223564 100644 --- a/rs/justfile +++ b/rs/justfile @@ -46,8 +46,8 @@ check-test *args: lint *args: python3 sh/rs/stats-docs.py cargo fmt --all --check - # quinn-udp keeps quinn's formatting; cargo fmt skips it (see moq-sock's lib.rs). - rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs + # quinn-udp and quinn keep quinn's formatting; cargo fmt skips them (see moq-sock's lib.rs, moq-tokio's quic.rs). + rustfmt --edition 2024 --check rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps "$@" cargo shear cargo sort --workspace --check --no-format @@ -57,7 +57,7 @@ lint *args: fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets "$@" cargo fmt --all - rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs + rustfmt --edition 2024 rs/moq-sock/src/udp/mod.rs rs/moq-tokio/src/quic/driver/mod.rs cargo shear --fix cargo sort --workspace --no-format @@ -312,11 +312,11 @@ bench-udp *args: cargo bench -p moq-uring --bench udp_tokio -- "$@" cargo bench -p moq-uring --bench udp_uring -- "$@" -# Measure a noq echo through the io_uring worker, ablating receive +# Measure a QUIC echo through the io_uring worker, ablating receive # batching, GRO, and GSO one at a time. Linux 6.12+ (skips loudly otherwise). [positional-arguments] bench-echo *args: - cargo bench -p moq-uring --bench echo_noq -- "$@" + cargo bench -p moq-uring --bench echo_quic -- "$@" # Measure a full moq-lite session (publisher to subscriber) through the # io_uring worker, same ablations and per-iteration unit as bench-echo. diff --git a/rs/moq-bench/Cargo.toml b/rs/moq-bench/Cargo.toml index 0de559a2e6..80e9ed8095 100644 --- a/rs/moq-bench/Cargo.toml +++ b/rs/moq-bench/Cargo.toml @@ -35,9 +35,9 @@ required-features = ["uring"] doc = false [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] iroh = ["moq-tokio/iroh"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] uring = ["dep:moq-uring"] websocket = ["moq-tokio/websocket"] diff --git a/rs/moq-boy/Cargo.toml b/rs/moq-boy/Cargo.toml index cffec17902..ca15ca12d1 100644 --- a/rs/moq-boy/Cargo.toml +++ b/rs/moq-boy/Cargo.toml @@ -12,9 +12,9 @@ edition = "2024" rust-version.workspace = true [features] -default = ["noq", "websocket"] +default = ["quic", "websocket"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] websocket = ["moq-tokio/websocket"] [dependencies] diff --git a/rs/moq-c/Cargo.toml b/rs/moq-c/Cargo.toml index c7564df387..a7cd65b3aa 100644 --- a/rs/moq-c/Cargo.toml +++ b/rs/moq-c/Cargo.toml @@ -26,7 +26,7 @@ moq-audio = { workspace = true, features = ["aac"] } moq-json = { workspace = true } moq-mux = { workspace = true } moq-net = { workspace = true } -moq-tokio = { workspace = true, features = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] } +moq-tokio = { workspace = true, features = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] } # Select the OpenH264 fallback plus NVIDIA and VAAPI hardware codecs explicitly # because workspace defaults are off. The hardware backends dlopen their driver; # `vaapi` also runs bindgen, so Linux builds need libclang. diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 9e8d106693..14401ce224 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,13 +26,13 @@ path = "src/main.rs" doc = false [features] -default = ["cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] +default = ["cluster-lan", "quic", "websocket", "nvidia", "s3", "gcs", "azure"] # Experimental transport with a heavy dependency tree, enabled only on request. iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["moq-tokio/noq"] +quic = ["moq-tokio/quic"] # qlog trace capture (`--client-quic-qlog `). Off by default: it compiles the # qlog machinery into the enabled backends and is only wanted for debugging or # congestion-control testing. @@ -63,7 +63,7 @@ play = ["dep:moq-video", "moq-video/render", "dep:moq-audio", "moq-audio/playbac # the matching moq-video feature, so it only bites when something else pulls in # moq-video: `capture`, `transcode`, or `play`. For a build with no CUDA # dependencies, drop it: -# cargo build -p moq-cli --no-default-features --features "noq,websocket,capture" +# cargo build -p moq-cli --no-default-features --features "quic,websocket,capture" nvidia = ["moq-video?/nvidia", "moq-transcode?/nvidia"] # Compatibility aliases for the pre-consolidation names, kept so a manifest that # already asks for the old split still resolves. Undocumented on purpose: the @@ -112,7 +112,7 @@ moq-hls = { workspace = true, features = ["server"] } moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the -# CLI's own feature set (iroh, noq, cluster-lan) is what selects them. +# CLI's own feature set (iroh, quic, cluster-lan) is what selects them. moq-relay = { workspace = true } moq-rtc = { workspace = true } moq-rtmp = { workspace = true } diff --git a/rs/moq-gst/src/lib.rs b/rs/moq-gst/src/lib.rs index 57c518542e..aaaa5f1f89 100644 --- a/rs/moq-gst/src/lib.rs +++ b/rs/moq-gst/src/lib.rs @@ -22,7 +22,7 @@ pub fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { .with_default_directive(LevelFilter::INFO.into()) .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse().unwrap()) - .add_directive("noq=info".parse().unwrap()) + .add_directive("moq_tokio::quic::driver=info".parse().unwrap()) .add_directive("tracing::span=off".parse().unwrap()) .add_directive("tracing::span::active=off".parse().unwrap()); diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 60ab13990e..4439f49b3c 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -40,6 +40,6 @@ url = { workspace = true } async-trait = "0.1" futures = { workspace = true } moq-archive = { path = "../moq-archive" } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-quic/Cargo.toml b/rs/moq-quic/Cargo.toml index a34ddfdadc..3580b37e95 100644 --- a/rs/moq-quic/Cargo.toml +++ b/rs/moq-quic/Cargo.toml @@ -56,6 +56,7 @@ aws-lc-rs = { version = "1.9", default-features = false, optional = true } bytes = { workspace = true } fastbloom = { version = "0.17", default-features = false, optional = true } lru-slab = "0.1.2" +moq-clock = { workspace = true } qlog = { version = "0.18", optional = true } rand = { workspace = true } rand_pcg = "0.10" diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index 789d1b1b44..d47b7629b0 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -34,6 +34,17 @@ Changes to its `Cargo.toml` and `build.rs` go into moq-sock's by hand, and its b The module logs through `tracing` only, so quinn-udp's `log` feature and no-op logger are dropped. A `.rustfmt.toml` in each imported directory keeps quinn's formatting; `cargo fmt` cannot apply it to a submodule, so moq-sock skips `udp` and `just rs fix` formats it separately. +quinn itself, the async layer from the same commit, is the `quic::driver` module of [moq-tokio](../moq-tokio), formatted the same way: + +```sh +git -C ../quinn format-patch -1 --stdout -- quinn/src \ + | sed -E 's#([ab])/quinn/src/lib\.rs#\1/rs/moq-tokio/src/quic/driver/mod.rs#g; s#([ab])/quinn/src/#\1/rs/moq-tokio/src/quic/driver/#g; s/\bcrate::/crate::quic::driver::/g; s/\bproto::/moq_quic::/g; s/\budp::/moq_sock::udp::/g' \ + | git am -3 +``` + +The driver is tokio-only: the `Runtime` trait, smol, the `futures-io` impls, and the lock-tracking mutex are dropped, so hunks touching them are skipped. +moq-tokio's `quic::web` module is `web-transport-moq` from moq-dev/noq at [`e27c0ce0`](https://github.com/moq-dev/noq/commit/e27c0ce00741d81c8620b98e345d4f2c88e0768c), which has no upstream left to track. + ### Carried changes Changes on top of the upstream commit, besides the renames: @@ -51,7 +62,9 @@ Changes on top of the upstream commit, besides the renames: - Apple fast-path fallback preserves the segmentation of batches prepared before the fast path was disabled, including when a private symbol is unavailable. - Discarding Initial or Handshake keys resets the PTO backoff, as RFC 9002 A.11 does, so a client's Initial backoff no longer delays the retransmission of a lost Finished. Also [moq-dev/noq#32](https://github.com/moq-dev/noq/pull/32); not offered upstream yet. - `TransportConfig::handshake_idle_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. - quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the switch's import has to set both timeouts there. + quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. +- **Instants on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. +- **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges. diff --git a/rs/moq-quic/src/config/transport.rs b/rs/moq-quic/src/config/transport.rs index eb72d81385..48e26c5b66 100644 --- a/rs/moq-quic/src/config/transport.rs +++ b/rs/moq-quic/src/config/transport.rs @@ -1,16 +1,16 @@ use std::{fmt, sync::Arc}; #[cfg(feature = "qlog")] -use std::{io, sync::Mutex, time::Instant}; +use std::{io, sync::Mutex}; #[cfg(feature = "qlog")] use qlog::streamer::QlogStreamer; -#[cfg(feature = "qlog")] -use crate::QlogStream; use crate::{ Duration, INITIAL_MTU, MAX_UDP_PAYLOAD, VarInt, VarIntBoundsExceeded, congestion, connection::qlog::QlogSink, }; +#[cfg(feature = "qlog")] +use crate::{Instant, QlogStream}; /// Parameters governing the core QUIC state machine /// @@ -634,10 +634,12 @@ impl QlogConfig { vec![], ); + // The streamer's own epoch only times the events it stamps itself, which this + // crate never asks it to; `QlogStream` stamps them on MoQ's clock instead. let mut streamer = QlogStreamer::new( self.title, self.description, - self.start_time, + std::time::Instant::now(), trace, qlog::events::EventImportance::Core, // `Instant` should have sub-microsecond precision on most platforms @@ -646,7 +648,10 @@ impl QlogConfig { ); match streamer.start_log() { - Ok(()) => Some(QlogStream(Arc::new(Mutex::new(streamer)))), + Ok(()) => Some(QlogStream { + streamer: Arc::new(Mutex::new(streamer)), + start: self.start_time, + }), Err(e) => { warn!("could not initialize endpoint qlog streamer: {e}"); None diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 0ad0ddbbb4..e828b46055 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -171,6 +171,8 @@ pub struct Connection { retry_src_cid: Option, events: VecDeque, endpoint_events: VecDeque, + /// Whether `EndpointEventInner::Draining` was emitted, which happens once. + draining_reported: bool, /// Whether the spin bit is in use for this connection spin_enabled: bool, /// Outgoing spin bit state @@ -317,6 +319,7 @@ impl Connection { retry_src_cid: None, events: VecDeque::new(), endpoint_events: VecDeque::new(), + draining_reported: false, spin_enabled: config.allow_spin && rng.random_ratio(7, 8), spin: false, spaces: [initial_space, PacketSpace::new(now), PacketSpace::new(now)], @@ -1256,6 +1259,7 @@ impl Connection { match timer { Timer::Close => { self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } Timer::Idle => { @@ -2625,6 +2629,9 @@ impl Connection { self.set_close_timer(now); } } + if matches!(self.state, State::Draining | State::Drained) { + self.report_draining(); + } if !was_drained && self.state.is_drained() { self.endpoint_events.push_back(EndpointEventInner::Drained); // Close timer may have been started previously, e.g. if we sent a close and got a @@ -3968,9 +3975,17 @@ impl Connection { self.close_common(); self.error = Some(reason); self.state = State::Drained; + self.report_draining(); self.endpoint_events.push_back(EndpointEventInner::Drained); } + /// Tell the endpoint the connection is no longer active, the first time it stops being so + fn report_draining(&mut self) { + if !mem::replace(&mut self.draining_reported, true) { + self.endpoint_events.push_back(EndpointEventInner::Draining); + } + } + /// Storage size required for the largest packet known to be supported by the current path /// /// Buffers passed to [`Connection::poll_transmit`] should be at least this large. diff --git a/rs/moq-quic/src/connection/qlog.rs b/rs/moq-quic/src/connection/qlog.rs index 6120e4e6f9..0294a69516 100644 --- a/rs/moq-quic/src/connection/qlog.rs +++ b/rs/moq-quic/src/connection/qlog.rs @@ -27,17 +27,23 @@ use crate::{ /// Shareable handle to a single qlog output stream #[cfg(feature = "qlog")] #[derive(Clone)] -pub struct QlogStream(pub(crate) Arc>); +pub struct QlogStream { + pub(crate) streamer: Arc>, + /// The epoch event times count from, on MoQ's clock rather than the std one the + /// streamer keeps. + pub(crate) start: Instant, +} #[cfg(feature = "qlog")] impl QlogStream { fn emit_event(&self, orig_rem_cid: ConnectionId, event: EventData, now: Instant) { - // Time will be overwritten by `add_event_with_instant` - let mut event = Event::with_time(0.0, event); + // Milliseconds since the epoch, at the nanosecond precision the streamer is built with. + let time = now.saturating_duration_since(self.start).as_nanos() as f64 / 1e6; + let mut event = Event::with_time(time, event); event.group_id = Some(Box::new(orig_rem_cid.to_string())); - let mut qlog_streamer = self.0.lock().unwrap(); - if let Err(e) = qlog_streamer.add_event_with_instant(event, now) { + let mut qlog_streamer = self.streamer.lock().unwrap(); + if let Err(e) = qlog_streamer.add_event(event) { warn!("could not emit qlog event: {e}"); } } diff --git a/rs/moq-quic/src/endpoint.rs b/rs/moq-quic/src/endpoint.rs index 616afffd82..19496b4478 100644 --- a/rs/moq-quic/src/endpoint.rs +++ b/rs/moq-quic/src/endpoint.rs @@ -126,6 +126,7 @@ impl Endpoint { } } } + Draining => {} Drained => { if let Some(conn) = self.connections.try_remove(ch.0) { self.index.remove(&conn); diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index 65ddf62da2..a5c39cb957 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -105,10 +105,12 @@ pub use token_memory_cache::TokenMemoryCache; use arbitrary::Arbitrary; // Deal with time +// MoQ's clock, so a caller-driven clock (a test's or the simulator's) reaches the state machine. +pub(crate) use moq_clock::Instant; #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] -pub(crate) use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(all(target_family = "wasm", target_os = "unknown"))] -pub(crate) use web_time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +pub(crate) use web_time::{Duration, SystemTime, UNIX_EPOCH}; #[cfg(fuzzing)] pub mod fuzzing { diff --git a/rs/moq-quic/src/shared.rs b/rs/moq-quic/src/shared.rs index f2d0ad5d2a..047c6d7ce0 100644 --- a/rs/moq-quic/src/shared.rs +++ b/rs/moq-quic/src/shared.rs @@ -45,10 +45,21 @@ impl EndpointEvent { pub fn is_drained(&self) -> bool { self.0 == EndpointEventInner::Drained } + + /// Determine whether the connection has stopped being active + /// + /// Emitted once per connection, before [`Self::is_drained`]'s event: when the peer's close + /// arrives, or at the drain if it never does. Past it the peer has our close or never will, + /// so a shutdown can stop waiting without sitting out the closing period. + pub fn is_draining(&self) -> bool { + self.0 == EndpointEventInner::Draining + } } #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum EndpointEventInner { + /// The connection has stopped being active; see [`EndpointEvent::is_draining`] + Draining, /// The connection has been drained Drained, /// The reset token and/or address eligible for generating resets has been updated diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index fedc9af91d..60a8e74199 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -2004,6 +2004,37 @@ fn initial_retransmit() { ); } +/// Each side tells its endpoint once that the connection stopped being active, before it drains, +/// so a shutdown waiting on every connection can stop at the first of these +#[test] +fn draining_is_reported_once_before_drained() { + let _guard = subscribe(); + let mut pair = Pair::default(); + let (client_ch, server_ch) = pair.connect(); + let now = pair.time; + pair.client_conn_mut(client_ch) + .close(now, VarInt(0), Bytes::new()); + pair.drive(); + + let expected = [Lifecycle::Draining, Lifecycle::Drained]; + let client: Vec<_> = pair + .client + .lifecycle + .iter() + .filter(|(ch, _)| *ch == client_ch) + .map(|(_, l)| *l) + .collect(); + let server: Vec<_> = pair + .server + .lifecycle + .iter() + .filter(|(ch, _)| *ch == server_ch) + .map(|(_, l)| *l) + .collect(); + assert_eq!(client, expected); + assert_eq!(server, expected); +} + #[test] fn instant_close_1() { let _guard = subscribe(); diff --git a/rs/moq-quic/src/tests/util.rs b/rs/moq-quic/src/tests/util.rs index e64c1ccf09..f98c7f5eca 100644 --- a/rs/moq-quic/src/tests/util.rs +++ b/rs/moq-quic/src/tests/util.rs @@ -397,6 +397,14 @@ pub(super) struct TestEndpoint { pub(super) capture_inbound_packets: bool, pub(super) handle_incoming: Box IncomingConnectionBehavior>, pub(super) waiting_incoming: Vec, + /// Every draining and drained endpoint event, in the order connections raised them + pub(super) lifecycle: Vec<(ConnectionHandle, Lifecycle)>, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum Lifecycle { + Draining, + Drained, } #[derive(Debug, Copy, Clone)] @@ -441,6 +449,7 @@ impl TestEndpoint { capture_inbound_packets: false, handle_incoming: Box::new(|_| IncomingConnectionBehavior::Accept), waiting_incoming: Vec::new(), + lifecycle: Vec::new(), } } @@ -536,6 +545,12 @@ impl TestEndpoint { } for (ch, event) in endpoint_events { + if event.is_draining() { + self.lifecycle.push((ch, Lifecycle::Draining)); + } + if event.is_drained() { + self.lifecycle.push((ch, Lifecycle::Drained)); + } if let Some(event) = self.handle_event(ch, event) && let Some(conn) = self.connections.get_mut(&ch) { diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 8b1a73eb74..7791f78594 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,12 +25,12 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] -default = ["cluster-lan", "noq", "websocket", "uds"] +default = ["cluster-lan", "quic", "websocket", "uds"] iroh = ["moq-tokio/iroh"] # Discover cluster peers on the LAN with mDNS (`--cluster-lan`). cluster-lan = ["moq-tokio/mdns"] jemalloc = ["moq-tokio/jemalloc"] -noq = ["_quic", "moq-tokio/noq"] +quic = ["_quic", "moq-tokio/quic"] # The tokio QUIC listener, and the thread-per-core worker group behind # `runtime.workers`. Not one to enable on its own: it names no backend, so # moq-tokio would have nothing to bind a socket with. @@ -46,7 +46,7 @@ websocket = ["moq-tokio/websocket", "dep:qmux", "axum/ws"] uds = ["moq-tokio/uds"] # Serve QUIC from io_uring workers (`--runtime-io-uring`), Linux-only. # This is off by default: the relay is useful without io_uring. -io-uring = ["_uring", "moq-uring/noq"] +io-uring = ["_uring", "moq-uring/quic"] # The listener machinery shared by the feature above. _uring = ["dep:moq-sock", "dep:moq-uring"] @@ -101,7 +101,6 @@ rcgen = "0.14" tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } -web-transport-moq = { workspace = true } # Which sockets the process still holds, so the embed tests can prove a stopped # relay closed its own without rebinding a port another test could take. diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 2750f1c024..707ffbc379 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -1570,7 +1570,7 @@ uid = [1001] "cannot be combined", ); - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] { let path = dir.join("typed.toml"); std::fs::write(&path, "[listen]\nload_balancer = { id = \"ab\", nonce = 8 }\n").unwrap(); diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index fdf4908f6d..8f463f4c3a 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -171,7 +171,7 @@ impl Relay { #[cfg(not(feature = "_quic"))] anyhow::ensure!( io_uring || config.runtime.workers().is_none(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); // Workers and the stream server split one relay across listeners, so they share @@ -552,7 +552,7 @@ impl Relay { // Before the readiness notify, for the same reason the cluster starts // before it: an unsupported kernel, a refused ring, or a certificate - // certificate identity noq will not load fails here, and reporting ready first would + // certificate identity rustls will not load fails here, and reporting ready first would // release the units depending on a relay that is about to exit. #[cfg(all(target_os = "linux", feature = "_uring"))] let mut uring = uring; @@ -929,7 +929,7 @@ mod tests { }; assert_eq!( error.to_string(), - "runtime.workers needs moq-relay built with the `noq` feature" + "runtime.workers needs moq-relay built with the `quic` feature" ); } } diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index 12476d1453..1cc1461dc8 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -285,7 +285,7 @@ mod tests { /// A WebSocket peer that upgrades and never sends SETUP ends at the drain /// deadline even with the SETUP timeout disabled, so its task does not /// outlive the relay. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test] async fn withheld_websocket_setup_ends_at_the_deadline() { let window = Duration::from_secs(1); diff --git a/rs/moq-relay/tests/cluster_idle.rs b/rs/moq-relay/tests/cluster_idle.rs index fc90ee1b7e..981fb408fe 100644 --- a/rs/moq-relay/tests/cluster_idle.rs +++ b/rs/moq-relay/tests/cluster_idle.rs @@ -2,7 +2,7 @@ //! //! A crashed or partitioned peer sends no close, so the idle timeout is the whole //! outage window for subscribes routed through it. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::time::{Duration, Instant}; diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index e699dc9c4e..bce5ce6144 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -23,7 +23,7 @@ //! See `test/drill/README.md` for the recipe, the sensitivity proof that each //! drill fails when its recovery behavior is removed, and the loom/fuzz cases //! covering the primitives underneath. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 726479103b..c7c0f47ac1 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -33,7 +33,7 @@ async fn within(step: &str, fut: impl std::future::Future) -> T { /// Run an integration-test future on a dedicated thread with a large stack. /// /// Under `--all-features`, `moq-tokio` compiles every transport -/// (noq, iroh, websocket, and qmux) into its `Session`/`Client` types. +/// (QUIC, iroh, websocket, and qmux) into its `Session`/`Client` types. /// These multi-relay tests hold several such values live across await points, /// so the single test future's state machine is large, and in an unoptimized /// build it overflows libtest's default 2 MiB per-test thread stack (a SIGABRT diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 5494fc4eea..5d66675801 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -763,10 +763,10 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { /// `listen.timeout` bounds every handshake phase from the first packet: TLS, /// HTTP/3 before CONNECT, and MoQ SETUP over raw QUIC and WebTransport. Each /// phase closes in its own error space. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] async fn uring_workers_bound_stalled_handshakes() { - use web_transport_moq::noq; + use moq_tokio::quic::{driver, web}; use web_transport_trait::Error as _; let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -788,10 +788,10 @@ async fn uring_workers_bound_stalled_handshakes() { let crypto = |alpn: &[u8]| { let mut tls = tls.clone(); tls.alpn_protocols = vec![alpn.to_vec()]; - let crypto = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); - noq::ClientConfig::new(std::sync::Arc::new(crypto)) + let crypto = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("QUIC crypto"); + driver::ClientConfig::new(std::sync::Arc::new(crypto)) }; - let client = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); + let client = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("client"); // TLS: the relay never hears more than the ClientHello, so only its own // deadline can end the handshake before the idle timeout does. @@ -806,7 +806,7 @@ async fn uring_workers_bound_stalled_handshakes() { .await .expect("listener did not bound the QUIC handshake"); assert!( - !matches!(error, noq::ConnectionError::TimedOut), + !matches!(error, driver::ConnectionError::TimedOut), "the relay must close a stalled QUIC handshake: {error}" ); @@ -823,11 +823,10 @@ async fn uring_workers_bound_stalled_handshakes() { .unwrap(); if scheme == "https" { let url: url::Url = format!("https://{addr}/stalled").parse().unwrap(); - let request = - web_transport_moq::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); - web_transport_moq::Session::connect(conn, request).await.unwrap() + let request = web::proto::ConnectRequest::new(url).with_protocol("moq-lite-06".to_string()); + web::Session::connect(conn, request).await.unwrap() } else { - web_transport_moq::Session::raw(conn) + web::Session::raw(conn) } }) .await @@ -842,7 +841,7 @@ async fn uring_workers_bound_stalled_handshakes() { "{scheme}: {error}" ); } - client.close(noq::VarInt::from_u32(0), b"test complete"); + client.close(driver::VarInt::from_u32(0), b"test complete"); client.wait_idle().await; running.abort(); let _ = running.await; @@ -851,7 +850,7 @@ async fn uring_workers_bound_stalled_handshakes() { /// A UDP hop to `server` that forwards one client's datagrams until the server /// first answers, and every answer back, so the server never sees the client /// finish its handshake. Returns the address to dial. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn stall_after_hello(server: SocketAddr) -> SocketAddr { let socket = tokio::net::UdpSocket::bind("127.0.0.1:0").await.expect("bind hop"); let addr = socket.local_addr().expect("hop address"); diff --git a/rs/moq-sock/src/bind.rs b/rs/moq-sock/src/bind.rs index ebd1e30851..13471fb997 100644 --- a/rs/moq-sock/src/bind.rs +++ b/rs/moq-sock/src/bind.rs @@ -27,7 +27,7 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); /// UDP socket buffer size requested in each direction, in bytes. /// /// A QUIC stack absorbs bursts in the kernel socket buffer: whatever doesn't fit -/// while the process is off the CPU is dropped before noq ever sees it, and +/// while the process is off the CPU is dropped before QUIC ever sees it, and /// congestion control reads those drops as congestion. The OS defaults are sized /// for a chatty TCP-era socket (208 KiB on Linux), which a single relay socket /// carrying every connection blows through in milliseconds. 8 MiB is roughly 64ms @@ -317,7 +317,7 @@ pub fn tcp(addr: SocketAddr) -> io::Result { /// A specific IPv6 address such as `[::1]` can never carry IPv4, so it is set /// v6-only instead, which Linux does on bind anyway. Windows keeps the cleared /// flag, so the socket claims an IPv4 side it lacks and then rejects the -/// IPv4-level options noq sets on a dual-stack socket with `WSAEINVAL`, failing +/// IPv4-level options the QUIC driver sets on a dual-stack socket with `WSAEINVAL`, failing /// endpoint setup. fn make_dual_stack(socket: &Socket, addr: SocketAddr) { let SocketAddr::V6(addr) = addr else { return }; @@ -364,7 +364,7 @@ mod tests { } /// A specific IPv6 address can't carry IPv4, so the socket must not claim to: - /// Windows rejects noq's IPv4 socket options on a dual-stack `[::1]` socket. + /// Windows rejects the QUIC driver's IPv4 socket options on a dual-stack `[::1]` socket. /// Linux forces v6-only on such a bind itself, so only Windows can regress. #[test] fn udp_ipv6_loopback_is_v6_only() { diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index 45ad002eab..9664cb7787 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -13,8 +13,24 @@ keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] [features] -default = ["noq", "aws-lc-rs", "websocket", "tcp", "uds"] -noq = ["_transport", "_certs", "dep:web-transport-moq", "dep:reqwest", "watch", "dep:dns-lookup"] +default = ["quic", "aws-lc-rs", "websocket", "tcp", "uds"] +# QUIC and WebTransport over moq-quic, for `https://`, `moqt://`, and `moql://`. +quic = [ + "_transport", + "_certs", + "dep:moq-quic", + "dep:moq-clock", + "dep:http", + "dep:kio", + "dep:pin-project-lite", + "dep:rustc-hash", + "dep:rustls-native-certs", + "dep:socket2", + "dep:web-transport-proto", + "dep:reqwest", + "watch", + "dep:dns-lookup", +] # Server-side certificate handling: chains loaded from disk, self-signed generation, # in-memory identities, and per-SNI selection. Not one to enable on its own, since # nothing turns it into a listener: every QUIC backend needs it for its own TLS, and @@ -28,12 +44,12 @@ _transport = [] watch = ["dep:notify"] # Capture qlog traces (`--*-quic-qlog `). Off by default so production # builds do not pull in the qlog crate and its serde_json encoder. -qlog = ["web-transport-moq?/qlog"] +qlog = ["moq-quic?/qlog"] # The QUIC backend pulls in its crypto provider from these features (default-features # are off on the dependency), so a backend without aws-lc-rs/ring has no initial cipher. # A provider is also all it takes to sign and serve certificates, so each turns on # `_certs` even where there is no QUIC backend to serve them for. -aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "web-transport-moq?/aws-lc-rs"] +aws-lc-rs = ["_certs", "rustls/aws-lc-rs", "rcgen/aws_lc_rs", "moq-quic?/rustls-aws-lc-rs"] iroh = ["_transport", "dep:web-transport-iroh", "dep:web-transport-proto", "dep:noq-proto"] # LAN peer discovery over mDNS (the `mdns` module). mdns = ["dep:mdns-sd", "dep:hmac", "dep:sha2", "dep:subtle"] @@ -44,7 +60,7 @@ tcp = ["_transport", "dep:qmux", "dep:dns-lookup"] # Unix-domain-socket qmux transport (`unix://`), unix-only. Adds peer-credential # inspection on accept. Pulls in `tcp` (shares qmux's stream transport). uds = ["tcp", "qmux/uds"] -ring = ["_certs", "rustls/ring", "rcgen/ring", "web-transport-moq?/ring"] +ring = ["_certs", "rustls/ring", "rcgen/ring", "moq-quic?/rustls-ring"] [dependencies] bytes = "1" @@ -54,27 +70,37 @@ dns-lookup = { version = "4", optional = true } futures = "0.3" hex = { workspace = true } hmac = { workspace = true, optional = true } +http = { version = "1", optional = true } humantime = "2.3" +kio = { workspace = true, optional = true } mdns-sd = { workspace = true, optional = true } +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } +# The QUIC state machine the `quic` module drives; the crypto-provider features pick its provider. +moq-quic = { workspace = true, features = ["bloom"], optional = true } moq-sock = { workspace = true } -# iroh runs on upstream noq, not the moq-noq fork, and re-exports only the +# iroh runs on upstream noq rather than moq-quic, and re-exports only the # ControllerFactory trait, not the concrete congestion configs. Version matched to # the copy iroh pulls in. noq-proto = { workspace = true, optional = true } notify = { version = "8", optional = true } +pin-project-lite = { version = "0.2", optional = true } qmux = { workspace = true, features = ["wss", "tls", "tcp"], optional = true } rand = "0.10.1" rcgen = { version = "0.14", default-features = false, optional = true } reqwest = { version = "0.13", default-features = false, optional = true } +rustc-hash = { workspace = true, optional = true } rustls = "0.23" +# The WebTransport client trusts the OS store alongside its custom roots. +rustls-native-certs = { version = "0.8", optional = true } # OS-native server-cert verifier for the rustls backend. rustls-platform-verifier = "0.7" rustls-webpki = { version = "0.103", features = ["aws-lc-rs"], optional = true } serde = { version = "1", features = ["derive"] } serde_with = { version = "3", features = ["hex"] } sha2 = { workspace = true, optional = true } +socket2 = { workspace = true, optional = true } subtle = { workspace = true, optional = true } thiserror = "2" tikv-jemalloc-ctl = { version = "0.6", optional = true } @@ -87,7 +113,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = { version = "2", features = ["serde"] } usage = { workspace = true, features = ["config"] } web-transport-iroh = { workspace = true, optional = true } -web-transport-moq = { workspace = true, optional = true } web-transport-proto = { workspace = true, optional = true } web-transport-trait = { workspace = true } x509-parser = "0.18" diff --git a/rs/moq-tokio/README.md b/rs/moq-tokio/README.md index 4802a6760b..3ae2db1988 100644 --- a/rs/moq-tokio/README.md +++ b/rs/moq-tokio/README.md @@ -12,7 +12,7 @@ Tokio-based connection helpers for native [Media over QUIC](https://moq.dev) app Establishes MoQ connections over a few different transports, selectable via cargo features: -- **WebTransport** (HTTP/3) via [noq](https://crates.io/crates/noq) +- **WebTransport** (HTTP/3) via [moq-quic](https://crates.io/crates/moq-quic) - **Raw QUIC** with ALPN negotiation - **WebSocket** as a fallback when QUIC isn't available - **Iroh** P2P (`iroh` feature) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 20c71f8b58..66166b129b 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -6,7 +6,7 @@ use crate::connection::Goaway; use crate::{Addrs, Backoff, Connection, Error}; -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] use std::future::Future; use url::Url; @@ -47,7 +47,7 @@ pub struct Client { /// raw TCP/UDS qmux and WebSocket). Resolved once in [`Client::new`] so the ALPN list /// can't diverge between transports. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -81,10 +81,10 @@ pub struct Client { #[cfg(feature = "websocket")] tls_host_name: Option, /// Only the TLS-based dials read this; the plaintext qmux transports have none. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls: rustls::ClientConfig, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "iroh")] iroh: Option, #[cfg(feature = "iroh")] @@ -98,7 +98,7 @@ impl Client { #[cfg(not(feature = "_transport"))] pub fn new(_config: Config) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -121,11 +121,11 @@ impl Client { // Only the rustls-backed transports use this. Iroh and the plaintext // qmux transports must not require a rustls crypto provider. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] let tls = config.tls.build()?; - #[cfg(feature = "noq")] - let noq = Some(crate::noq::NoqClient::new(&config, &quic)?); + #[cfg(feature = "quic")] + let quic = Some(crate::quic::Client::new(&config, &quic)?); let versions = config.versions(); // Read before the struct literal below moves fields out of `config`. @@ -145,7 +145,7 @@ impl Client { .with_versions(versions.clone()) .with_extensions(config.extensions), #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -168,10 +168,10 @@ impl Client { websocket: config.websocket, #[cfg(feature = "websocket")] tls_host_name, - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] tls, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "iroh")] iroh: None, #[cfg(feature = "iroh")] @@ -278,13 +278,13 @@ impl Client { /// This closes at once, discarding stream data the peer has not acknowledged yet. /// Call [`Connection::close`] on each connection first to deliver it. /// - /// Only the noq endpoint is closed. WebSocket, TCP, and UDS sessions end when their + /// Only the QUIC endpoint is closed. WebSocket, TCP, and UDS sessions end when their /// [`Connection`] is dropped (the kernel closes the socket on exit), and an iroh /// endpoint passed to `with_iroh` is closed by its owner. pub async fn close(self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq { - noq.close().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic { + quic.close().await; } } @@ -315,7 +315,7 @@ impl Client { /// /// Errors if no transport feature is compiled in. #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -323,7 +323,7 @@ impl Client { )))] pub(crate) async fn dial(&self, _addr: crate::connect::Addr) -> crate::Result { Err(Error::NoBackend( - "no backend compiled; enable noq, iroh, websocket, tcp, or uds feature", + "no backend compiled; enable quic, iroh, websocket, tcp, or uds feature", )) } @@ -334,7 +334,7 @@ impl Client { /// protocol driver is spawned on the current tokio runtime; the session /// closes once the last returned handle drops. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -362,7 +362,7 @@ impl Client { /// The moq client builder, advertising `path` in the SETUP when there is one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -376,7 +376,7 @@ impl Client { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -472,15 +472,15 @@ impl Client { return Ok(Dialed::new(session, crate::Transport::Iroh)); } - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { let quic_addr = addr.clone(); let binding = match quic_transport(&url) { crate::Transport::Quic => moq_net::transport::Binding::InBand, _ => moq_net::transport::Binding::Uri, }; let quic_handle = async { - noq.connect(&self.tls, quic_addr, &self.versions) + quic.connect(&self.tls, quic_addr, &self.versions) .await .map(|session| crate::transport::Session::new(session, binding)) .map_err(Error::from) @@ -530,7 +530,7 @@ impl Client { /// /// Only compiled when there is a QUIC dial to race: a WebSocket-only build connects /// over the fallback directly. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] async fn race_moq_connect( &self, moq: &moq_net::Client, @@ -574,7 +574,7 @@ impl Client { /// `None` when the result is empty, which means the same as omitting the parameter: the /// server's default path. A peer on published lite-05 rejects an empty value outright. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -600,7 +600,7 @@ fn request_target(url: &Url) -> Option { /// picked by ALPN negotiation rather than by the scheme; that dial reads the negotiated /// [`crate::iroh::Binding`] and calls [`request_target`] itself. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -623,9 +623,9 @@ fn setup_path(url: &Url) -> Option { } } -/// What a noq dial to `url` runs on: WebTransport for `https://` (and the `http://` +/// What a QUIC dial to `url` runs on: WebTransport for `https://` (and the `http://` /// bootstrap), raw QUIC for `moqt://` and `moql://`. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn quic_transport(url: &Url) -> crate::Transport { match url.scheme() { "moqt" | "moql" => crate::Transport::Quic, @@ -643,7 +643,7 @@ pub(crate) struct Dialed { impl Dialed { #[cfg_attr( not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -662,7 +662,7 @@ impl Dialed { /// from the host and port so URL userinfo never goes on the wire. `None` for every other /// scheme, and when the URL has no host, since an empty authority is meaningless. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -681,7 +681,7 @@ fn setup_authority(url: &Url) -> Option { } /// The arm that won a [`race_transport_connect`]; the other was dropped, cancelling it. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] enum TransportRace { Quic(Q), WebSocket(W), @@ -689,7 +689,7 @@ enum TransportRace { /// Race the QUIC dial to `url` against the WebSocket fallback, returning whichever /// transport comes up first. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] async fn race_transport_connect(url: &Url, quic: Q, websocket: W) -> crate::Result> where Q: Future>, @@ -751,7 +751,7 @@ where /// Auth is terminal only when both arms refused. A WebTransport-only endpoint /// answers the fallback with 403 while QUIC is still in flight, and reconnect /// treats is_auth() as terminal, so a mixed pair reports the retryable error. -#[cfg(all(feature = "websocket", feature = "noq"))] +#[cfg(all(feature = "websocket", feature = "quic"))] fn race_error(quic: Error, websocket: Error) -> Error { match (quic.is_auth(), websocket.is_auth()) { (false, false) => Error::TransportRace { @@ -764,7 +764,7 @@ fn race_error(quic: Error, websocket: Error) -> Error { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -791,25 +791,25 @@ mod tests { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn fixed_target_preserves_request_and_refuses_redirect() { check_fixed_redirect(false, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn one_shot_fixed_target_refuses_redirect() { check_fixed_redirect(true, true).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn unused_fixed_fallback_does_not_restrict_connected_target() { check_fixed_redirect(true, false).await; } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] async fn check_fixed_redirect(once: bool, pinned: bool) { let mut listen = crate::listen::Config { bind: Some("127.0.0.1:0".parse().unwrap()), @@ -886,7 +886,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -937,7 +937,7 @@ mod tests { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -976,7 +976,7 @@ mod tests { /// The iroh dial derives its target here rather than through [`setup_path`], since /// only the negotiated binding says whether to send one. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1351,7 +1351,7 @@ mod tests { assert_eq!(config.versions().alpns().len(), moq_net::ALPNS.len()); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_auth_error() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1367,7 +1367,7 @@ mod tests { assert!(matches!(value, super::TransportRace::WebSocket(1))); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_quic_after_websocket_forbidden() { let quic = async { @@ -1389,7 +1389,7 @@ mod tests { /// Inline rather than in `tests/` so each arm dials its own ephemeral port: the /// public connect sends the fallback to the QUIC port, which nothing reserves over /// TCP as well. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn websocket_forbidden_does_not_end_a_quic_connect() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -1431,14 +1431,14 @@ mod tests { // The same race `connect_inner` runs, except the fallback dials its own port, // as plain ws:// so the listener above can answer without TLS. - let noq = client.noq.clone().unwrap(); + let quic = client.quic.clone().unwrap(); let (tls, versions) = (client.tls.clone(), client.versions.clone()); let quic_addr: crate::connect::Addr = Url::parse(&format!("https://localhost:{quic_port}")).unwrap().into(); let ws_addr: crate::connect::Addr = Url::parse(&format!("http://localhost:{ws_port}")).unwrap().into(); // Hold QUIC until the fallback has been refused, so the 403 is always exercised. let quic = Box::pin(async move { forbid.await.unwrap().expect("fallback listener failed"); - noq.connect(&tls, quic_addr, &versions) + quic.connect(&tls, quic_addr, &versions) .await .map(|session| crate::transport::Session::new(session, moq_net::transport::Binding::Uri)) .map_err(Error::from) @@ -1460,7 +1460,7 @@ mod tests { /// QUIC wins while the fallback handshake is held until QUIC connects. /// Each transport owns its ephemeral port; this exercises the same race as /// connect_inner without depending on which real handshake runs faster. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tracing_test::traced_test] #[tokio::test] async fn broadcast_race_quic_wins() { @@ -1530,8 +1530,8 @@ mod tests { let client = client.with_subscriber(sub_origin); let dialer = client.clone(); let quic = Box::pin(async move { - let noq = dialer.noq.as_ref().expect("QUIC backend"); - let session = noq.connect(&dialer.tls, quic_addr, &dialer.versions).await?; + let quic = dialer.quic.as_ref().expect("QUIC backend"); + let session = quic.connect(&dialer.tls, quic_addr, &dialer.versions).await?; connected.send(()).expect("fallback listener alive"); Ok::<_, crate::Error>(crate::transport::Session::new( session, @@ -1577,7 +1577,7 @@ mod tests { .expect("server task failed"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_auth_when_both_refuse() { let quic = async { Err::(crate::ConnectError::Unauthorized.into()) }; @@ -1589,7 +1589,7 @@ mod tests { assert!(err.is_auth(), "unexpected error: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_reports_quic_error_when_websocket_forbidden() { let quic = async { @@ -1605,7 +1605,7 @@ mod tests { assert!(!err.is_auth(), "mixed auth/non-auth must stay retryable: {err}"); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_keeps_websocket_after_quic_non_auth_error() { let quic = async { Err::(Error::ConnectFailed) }; @@ -1619,14 +1619,14 @@ mod tests { /// The URL a race unit test dials. Each test runs in its own process under nextest, /// so the process-wide [`crate::websocket::won`] memo starts empty. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn race_url() -> Url { Url::parse("https://race.invalid/").unwrap() } /// QUIC winning a race clears WebSocket's earlier win, so the next dial gives QUIC /// its head start again. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_restores_its_head_start() { crate::websocket::remember(&race_url()); @@ -1644,7 +1644,7 @@ mod tests { } /// A race arm that never finishes, and reports through `dropped` once it is dropped. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] fn loser() -> ( impl Future, tokio::sync::oneshot::Receiver, @@ -1659,7 +1659,7 @@ mod tests { /// WebSocket winning cancels the QUIC dial before the race returns, so no QUIC /// session can come up behind the one WebSocket serves. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn websocket_winning_cancels_the_quic_dial() { let (quic, mut dropped) = loser::>(); @@ -1677,7 +1677,7 @@ mod tests { } /// QUIC winning cancels the WebSocket dial before the race returns. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test(start_paused = true)] async fn quic_winning_cancels_the_websocket_dial() { let quic = async { Ok("quic") }; @@ -1694,7 +1694,7 @@ mod tests { ); } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn race_transport_connect_returns_when_quic_transport_connects() { let quic = async { Ok("quic") }; @@ -1721,7 +1721,7 @@ mod tests { /// Iroh carries no rustls state, so constructing its client must not require an /// application-installed crypto provider. - #[cfg(all(feature = "iroh", not(any(feature = "noq", feature = "websocket"))))] + #[cfg(all(feature = "iroh", not(any(feature = "quic", feature = "websocket"))))] #[test] fn iroh_only_client_does_not_require_a_tls_provider() { crate::connect::Config::default().init().expect("iroh-only client"); diff --git a/rs/moq-tokio/src/connect.rs b/rs/moq-tokio/src/connect.rs index 789efcb0df..cd67fa6678 100644 --- a/rs/moq-tokio/src/connect.rs +++ b/rs/moq-tokio/src/connect.rs @@ -341,7 +341,7 @@ pub enum ConnectError { impl ConnectError { /// Only the transports that carry an HTTP status (WebTransport, WebSocket) can /// classify one; qmux over tcp/unix has no such response. - #[cfg(any(feature = "noq", feature = "websocket"))] + #[cfg(any(feature = "quic", feature = "websocket"))] pub(crate) fn from_status_u16(status: u16) -> Option { match status { 401 => Some(Self::Unauthorized), @@ -357,7 +357,7 @@ impl ConnectError { } } -#[cfg(all(test, any(feature = "noq", feature = "websocket")))] +#[cfg(all(test, any(feature = "quic", feature = "websocket")))] mod tests { use super::*; diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index f98e88f8e1..e44fbaa760 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -2070,7 +2070,7 @@ mod tests { } /// Invalid local configuration must end an unlimited reconnect loop without waiting. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_unsupported_scheme_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2088,7 +2088,7 @@ mod tests { } /// A URL host that parses but can never be a TLS server name ends an unlimited reconnect loop. - #[cfg(all(feature = "noq", feature = "websocket"))] + #[cfg(all(feature = "quic", feature = "websocket"))] #[tokio::test(start_paused = true)] async fn an_invalid_tls_server_name_ends_unlimited_reconnects() { let mut config = crate::connect::Config::default(); @@ -2200,12 +2200,12 @@ mod tests { /// /// Holding is what decides the transport race without depending on timing: the /// QUIC dial cannot finish until the test says so. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] struct Forwarder { gate: tokio::sync::watch::Sender, } - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] impl Forwarder { async fn start(front: tokio::net::UdpSocket, server: std::net::SocketAddr) -> Self { use std::sync::Arc; @@ -2264,7 +2264,7 @@ mod tests { /// When QUIC wins the race the WebSocket dial is cancelled at once: its socket /// closes without carrying a session, and the one session runs over QUIC. - #[cfg(all(feature = "websocket", feature = "noq"))] + #[cfg(all(feature = "websocket", feature = "quic"))] #[tokio::test] async fn quic_winning_cancels_the_websocket_dial() { use tokio::io::AsyncReadExt; diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index e1dbb14064..21aff0f4db 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -206,10 +206,10 @@ pub enum Error { #[error(transparent)] Tls(Arc), - /// The noq backend failed. - #[cfg(feature = "noq")] + /// The QUIC backend failed. + #[cfg(feature = "quic")] #[error(transparent)] - Noq(Arc), + Quic(Arc), /// The Iroh backend failed. #[cfg(feature = "iroh")] @@ -252,8 +252,8 @@ impl Error { | Self::WindowRange(_) | Self::QlogUnsupported | Self::Connect(_) => true, - #[cfg(feature = "noq")] - Self::Noq(err) => err.is_terminal(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.is_terminal(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.is_terminal(), #[cfg(feature = "tcp")] @@ -285,8 +285,8 @@ impl Error { Self::MoqNet( moq_net::Error::Unauthorized | moq_net::Error::Session(moq_net::SessionError::Unauthorized), ) => Some(crate::ConnectError::Unauthorized), - #[cfg(feature = "noq")] - Self::Noq(err) => err.connect_error(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.connect_error(), #[cfg(feature = "websocket")] Self::TransportRace { quic, websocket } => quic.connect_error().or_else(|| websocket.connect_error()), #[cfg(feature = "websocket")] @@ -318,8 +318,8 @@ impl Error { _ => None, }, - #[cfg(feature = "noq")] - Self::Noq(err) => err.status(), + #[cfg(feature = "quic")] + Self::Quic(err) => err.status(), #[cfg(feature = "websocket")] Self::WebSocket(err) => err.status(), _ => None, @@ -348,14 +348,14 @@ impl From for Error { } } -#[cfg(feature = "noq")] -impl From for Error { - fn from(err: crate::noq::Error) -> Self { +#[cfg(feature = "quic")] +impl From for Error { + fn from(err: crate::quic::Error) -> Self { if let Some(err) = err.connect_error() { return Self::Connect(err); } - Self::Noq(Arc::new(err)) + Self::Quic(Arc::new(err)) } } diff --git a/rs/moq-tokio/src/iroh.rs b/rs/moq-tokio/src/iroh.rs index 9e7350d1ad..058f33ca0c 100644 --- a/rs/moq-tokio/src/iroh.rs +++ b/rs/moq-tokio/src/iroh.rs @@ -9,7 +9,7 @@ use std::{net, path::PathBuf, str::FromStr, sync::Arc}; use crate::quic::CongestionControl; use url::Url; use web_transport_iroh::iroh::{self, SecretKey}; -// NOTE: web-transport-iroh should re-export proto like web-transport-moq does. +// NOTE: web-transport-iroh should re-export proto like `crate::quic::web` does. use web_transport_proto::{ConnectRequest, ConnectResponse}; pub use iroh::Endpoint; @@ -112,7 +112,7 @@ pub enum Error { RecvRequest(String), /// GSO is always on for iroh, so `--quic-gso=false` cannot be honored. - #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the noq backend")] + #[error("the iroh backend cannot disable GSO; drop --quic-gso=false or use the quic backend")] GsoUnsupported, } @@ -531,7 +531,7 @@ mod tests { /// Build a controller from each family's factory and downcast it to the /// concrete implementation it must map to. iroh runs on noq, so this is the - /// same pairing as the noq backend. + /// same pairing as the quic backend. #[test] fn congestion_factory_maps_each_family() { let now = std::time::Instant::now(); diff --git a/rs/moq-tokio/src/lib.rs b/rs/moq-tokio/src/lib.rs index 29a9b356e9..1dc43f9694 100644 --- a/rs/moq-tokio/src/lib.rs +++ b/rs/moq-tokio/src/lib.rs @@ -11,7 +11,7 @@ //! See [`Client`] for connecting to relays and [`Server`] for accepting //! connections. The `mdns` feature finds peers to connect to on the local network. //! -//! With `default-features = false`, the `noq` backend must be paired +//! With `default-features = false`, the `quic` backend must be paired //! with the `aws-lc-rs` or `ring` crypto-provider feature. Every other subset //! compiles, including no transport at all: such a build cannot connect to //! anything, though a crypto provider on its own is still enough to configure TLS @@ -22,7 +22,7 @@ // The protocol crates need a compiled provider for reset and retry-token keys. A rustls provider // installed at runtime cannot supply constructors removed by their compile-time feature gates. -#[cfg(all(feature = "noq", not(any(feature = "aws-lc-rs", feature = "ring"))))] +#[cfg(all(feature = "quic", not(any(feature = "aws-lc-rs", feature = "ring"))))] compile_error!("a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature"); mod abort; @@ -34,18 +34,16 @@ pub mod connect; pub mod connection; pub mod crypto; mod error; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] pub mod failover; #[cfg(feature = "jemalloc")] pub mod jemalloc; pub mod listen; mod log; -#[cfg(feature = "noq")] -pub mod noq; pub mod origin; pub mod panic; pub mod quic; -#[cfg(any(feature = "noq", feature = "tcp", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "tcp", feature = "websocket"))] mod resolve; #[cfg(feature = "_transport")] pub mod server; @@ -60,7 +58,7 @@ pub mod unix; pub mod worker; // Resolving a `host:port` bind string is a QUIC-listener concern; the stream // listeners take a `SocketAddr`/path straight from their config. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod util; #[cfg(feature = "watch")] pub mod watch; diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 20e6fa4f7a..16959fd75c 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -14,7 +14,7 @@ pub enum Bind { } impl Bind { - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn resolve(&self) -> std::io::Result { match self { Self::Addr(addr) => Ok(*addr), @@ -186,11 +186,11 @@ pub struct Config { /// IPv4 address advertised as the QUIC preferred_address. /// - /// Supporting clients (Chrome M131+, native noq) migrate to this address + /// Supporting clients (Chrome M131+, this crate) migrate to this address /// shortly after the handshake completes. Typical use: handshake on an /// anycast IP, steady-state on this host's unicast IP. /// - /// Honored by noq. Accept-only, which is why it lives + /// Honored by the QUIC backend. Accept-only, which is why it lives /// here rather than in the shared [`crate::quic::Config`]. #[usage( name = "listen-preferred-v4", @@ -292,7 +292,7 @@ impl Default for Config { } } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) use moq_sock::shard::Shard; /// One server's socket in a complete `SO_REUSEPORT` group. /// @@ -527,7 +527,7 @@ impl Config { (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { self.lb_id @@ -666,7 +666,7 @@ mod tests { assert!("relay.example.com:443:8443".parse::().is_err()); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn cli_load_balancer_survives_the_merge_round_trip() { let config = config_from(["test", "--listen-quic-lb-id", "ab", "--listen-quic-lb-nonce", "9"]); @@ -726,7 +726,7 @@ load_balancer = { id = "cd", nonce = 4 } } /// Programmatic configuration keeps the QUIC-LB id and nonce paired. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn load_balancer_is_a_single_typed_value() { assert_eq!(Config::default().load_balancer(), None); diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index ddf33bbcbe..5d8a5e4f19 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,7 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) - .add_directive("noq=info".parse()?) + .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) .add_directive("tracing::span=off".parse()?) diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 4935765246..4b28ac64ef 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -14,6 +14,29 @@ use std::time::Duration; use crate::cli::Duration as CliDuration; +#[cfg(feature = "quic")] +mod backend; +#[cfg(feature = "quic")] +pub use backend::Error; +#[cfg(feature = "quic")] +pub(crate) use backend::{Client, Options, Server, accept}; + +// The async QUIC layer driving `moq-quic` on tokio, forked from quinn (see its module docs). +// Public only so the workspace's tests can dial a bare endpoint; not a stable API. +// cargo fmt formats submodules with the crate root's config, so `just rs fix` +// formats this one separately with its own (quinn's). +#[cfg(feature = "quic")] +#[doc(hidden)] +#[rustfmt::skip] +pub mod driver; + +/// WebTransport over [`driver`], formerly the `web-transport-moq` crate. +/// +/// Public only so the workspace's tests can dial a bare session; not a stable API. +#[cfg(feature = "quic")] +#[doc(hidden)] +pub mod web; + /// The routable server ID a QUIC-LB load balancer encodes into connection IDs. /// /// Parsed from, and serialized as, a hex string. Its length must match the load @@ -55,7 +78,7 @@ pub struct LoadBalancer { /// The congestion control family for a QUIC connection. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. #[derive(Clone, Copy, Debug, PartialEq, Eq, usage::ValueEnum, serde::Serialize, serde::Deserialize)] #[usage(ignore_case)] #[serde(rename_all = "kebab-case")] @@ -87,7 +110,7 @@ pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Bounds how much unread data one peer can make us buffer across all of its streams, /// which the backends otherwise leave unlimited. Sized so a relay-to-relay session, which /// shares one connection, can still sustain several Gbps at intercontinental RTTs. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) const DEFAULT_RECEIVE_WINDOW: u64 = 64 << 20; /// Default idle timeout before an inactive connection is dropped. @@ -250,7 +273,7 @@ pub struct Config { /// Write qlog traces into this directory, which must already exist. /// - /// Noq writes one file per connection. + /// Each endpoint writes one file, shared by its connections. /// /// Requires the `qlog` feature; setting it errors at init otherwise. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -648,7 +671,7 @@ impl Resolved { /// Delay-based unless the operator says otherwise: BBR keeps queues short and the /// send rate steady enough for a live encoder to track, which is what this stack /// carries. - #[cfg_attr(not(any(feature = "noq", feature = "iroh")), allow(dead_code))] + #[cfg_attr(not(any(feature = "quic", feature = "iroh")), allow(dead_code))] pub(crate) fn congestion(&self) -> CongestionControl { self.congestion_control.unwrap_or(CongestionControl::Delay) } @@ -657,7 +680,7 @@ impl Resolved { /// /// Only meaningful once [`Config::validate`] has passed; a build without the /// `qlog` feature never gets here with a directory set. - #[cfg_attr(not(feature = "noq"), allow(dead_code))] + #[cfg_attr(not(feature = "quic"), allow(dead_code))] pub(crate) fn qlog_dir(&self) -> Option<&std::path::Path> { self.qlog.as_deref() } diff --git a/rs/moq-tokio/src/noq.rs b/rs/moq-tokio/src/quic/backend.rs similarity index 79% rename from rs/moq-tokio/src/noq.rs rename to rs/moq-tokio/src/quic/backend.rs index 27634e1648..b0b5b08414 100644 --- a/rs/moq-tokio/src/noq.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -1,5 +1,6 @@ -//! The noq QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +//! The QUIC backend, used for both WebTransport (`https://`) and raw QUIC (`moqt://`, `moql://`). +use super::{driver, web}; use crate::RedactedUrl; use crate::connect; use crate::listen; @@ -12,13 +13,12 @@ use std::pin::Pin; use std::sync::{Arc, Mutex, RwLock}; use std::task::{Context, Poll, Waker, ready}; use std::time::Duration; -use web_transport_moq::noq; -/// Attach a qlog factory writing into the configured directory, if any. +/// Attach a qlog stream writing into the configured directory, if any. /// -/// noq's factory opens one file per connection, named after its initial destination -/// connection ID, so nothing needs to be unique per endpoint here. -fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { +/// A [`driver::QlogStream`] is a shared handle: every connection on the endpoint writes into +/// it, tagged with its own qlog `group_id`, so this is one file per endpoint. +fn apply_qlog(transport: &mut driver::TransportConfig, quic: &Resolved, role: &str) -> Result<()> { // `Client::validate` already rejected a directory this build can't honor, so the // block below is only reached where capture actually works. let Some(dir) = quic.qlog_dir() else { @@ -27,8 +27,21 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) #[cfg(feature = "qlog")] { - transport.qlog_from_path(dir, &format!("moq-{role}")); - tracing::info!(dir = %dir.display(), "writing qlog"); + // The pid and sequence keep endpoints sharing a directory from clobbering each other. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = dir.join(format!("moq-{role}-{}-{seq}.sqlog", std::process::id())); + let file = std::fs::File::create(&path).map_err(Error::CreateQlog)?; + + // Deliberately unbuffered: qlog's streamer only flushes on `finish_log`, which is + // never called per event, so a BufWriter would hold every trace in memory until the + // endpoint drops and lose the lot if the process is killed. Killing a stuck process + // is exactly when these traces are worth having. + let mut config = driver::QlogConfig::default(); + config.writer(Box::new(file)).title(Some(format!("moq-tokio {role}"))); + + transport.qlog_stream(config.into_stream()); + tracing::info!(path = %path.display(), "writing qlog"); } #[cfg(not(feature = "qlog"))] @@ -37,17 +50,17 @@ fn apply_qlog(transport: &mut noq::TransportConfig, quic: &Resolved, role: &str) Ok(()) } -/// Apply the resolved quic knobs to a noq transport config. -fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { +/// Apply the resolved quic knobs to a transport config. +fn apply_transport(transport: &mut driver::TransportConfig, quic: &Resolved) { transport.max_idle_timeout(Some(quic.idle_timeout.try_into().expect("idle timeout out of range"))); transport.keep_alive_interval(quic.keep_alive); - // noq enables MTU discovery by default; disable it unless asked. + // MTU discovery is enabled by default; disable it unless asked. if !quic.mtu_discovery { transport.mtu_discovery_config(None); } - let max_streams = noq::VarInt::from_u64(quic.max_streams).unwrap_or(noq::VarInt::MAX); + let max_streams = driver::VarInt::from_u64(quic.max_streams).unwrap_or(driver::VarInt::MAX); transport.max_concurrent_bidi_streams(max_streams); transport.max_concurrent_uni_streams(max_streams); @@ -62,28 +75,28 @@ fn apply_transport(transport: &mut noq::TransportConfig, quic: &Resolved) { /// Apply the flow-control windows. The connection window defaults to ours, since the /// backend's is unlimited; the others stay at the backend default when unset. -fn apply_windows(transport: &mut noq::TransportConfig, quic: &Resolved) { +fn apply_windows(transport: &mut driver::TransportConfig, quic: &Resolved) { // Saturating rather than erroring: `Config::validate` already rejects a window // past the varint, so this only bites a `Resolved` built without it. let window = quic.receive_window.unwrap_or(crate::quic::DEFAULT_RECEIVE_WINDOW); - transport.receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); if let Some(window) = quic.stream_receive_window { - transport.stream_receive_window(noq::VarInt::from_u64(window).unwrap_or(noq::VarInt::MAX)); + transport.stream_receive_window(driver::VarInt::from_u64(window).unwrap_or(driver::VarInt::MAX)); } if let Some(window) = quic.send_window { transport.send_window(window); } } -/// The noq controller factory for a congestion control family. noq's BBR is v3. -fn congestion_factory(family: CongestionControl) -> Arc { +/// The controller factory for a congestion control family. moq-quic's BBR is v3. +fn congestion_factory(family: CongestionControl) -> Arc { match family { - CongestionControl::Loss => Arc::new(noq::congestion::CubicConfig::default()), - CongestionControl::Delay => Arc::new(noq::congestion::Bbr3Config::default()), + CongestionControl::Loss => Arc::new(driver::congestion::CubicConfig::default()), + CongestionControl::Delay => Arc::new(driver::congestion::Bbr3Config::default()), } } -/// Errors specific to the noq QUIC backend. +/// Errors specific to the QUIC backend. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -95,6 +108,11 @@ pub enum Error { #[error("failed to create QUIC endpoint")] CreateEndpoint(#[source] std::io::Error), + /// The qlog trace file couldn't be created, usually a missing directory. + #[cfg(feature = "qlog")] + #[error("failed to create qlog file")] + CreateQlog(#[source] std::io::Error), + /// No async runtime was found. Construct the client or server from within a tokio runtime. #[error("no async runtime")] NoRuntime, @@ -203,7 +221,7 @@ pub enum Error { message: String, /// The HTTP status the server answered the CONNECT with, when it answered with one. /// - /// Read at conversion time rather than kept as a `web-transport-moq` error, so the + /// Read at conversion time rather than kept as a [`web`] error, so the /// classification survives without that crate appearing in this crate's public API. status: Option, }, @@ -251,20 +269,20 @@ impl crate::failover::Aggregate for Error { } crate::error::from_message! { - noq::ConnectError => Connect, - noq::ConnectionError => Connection, - web_transport_moq::ServerError => Server, + driver::ConnectError => Connect, + driver::ConnectionError => Connection, + web::ServerError => Server, hex::FromHexError => InvalidFingerprint, } -impl From for Error { - fn from(_: noq::crypto::rustls::NoInitialCipherSuite) -> Self { +impl From for Error { + fn from(_: driver::crypto::rustls::NoInitialCipherSuite) -> Self { Self::NoInitialCipherSuite } } -impl From for Error { - fn from(err: web_transport_moq::ClientError) -> Self { +impl From for Error { + fn from(err: web::ClientError) -> Self { Self::Client { status: client_status(&err), message: crate::error::message(err), @@ -277,9 +295,9 @@ type Result = std::result::Result; // ── Client ────────────────────────────────────────────────────────── #[derive(Clone)] -pub(crate) struct NoqClient { - pub quic: noq::Endpoint, - pub transport: Arc, +pub(crate) struct Client { + pub quic: driver::Endpoint, + pub transport: Arc, /// Whether an `http://` URL may bootstrap a pin (see [crate::tls::Connect::allows_http_bootstrap]). pub http_bootstrap: bool, /// Optional TLS SNI / verification hostname override (from config). @@ -295,24 +313,25 @@ pub(crate) struct NoqClient { dual_stack: bool, } -impl NoqClient { +impl Client { pub fn new(config: &connect::Config, quic: &crate::quic::Config) -> Result { let resolved = config.resolve(); let socket = crate::bind::udp(crate::bind::Udp::new(resolved.bind)).map_err(Error::BindSocket)?; let dual_stack = crate::bind::udp_is_dual_stack(&socket); - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "client")?; let transport = Arc::new(transport); // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; - let endpoint_config = noq::EndpointConfig::default(); + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; + let endpoint_config = driver::EndpointConfig::default(); // Create the generic QUIC endpoint. - let quic = noq::Endpoint::new(endpoint_config, None, socket, runtime).map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new(endpoint_config, None, socket).map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -327,7 +346,7 @@ impl NoqClient { /// Close every connection, then wait until each has sent its close to the peer. pub async fn close(self) { - self.quic.close(noq::VarInt::from_u32(0), b"client shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"client shutdown"); // Not `wait_idle`, which also sits out each connection's 3 PTO closing // period: that only repeats the close to a peer that already has it. self.quic.wait_all_draining().await; @@ -338,7 +357,7 @@ impl NoqClient { tls: &rustls::ClientConfig, addr: crate::connect::Addr, versions: &moq_net::Versions, - ) -> Result { + ) -> Result { let mut url = addr.url().clone(); let mut config = tls.clone(); @@ -401,7 +420,7 @@ impl NoqClient { } let alpns: Vec> = match url.scheme() { - "https" => vec![web_transport_moq::ALPN.as_bytes().to_vec()], + "https" => vec![web::ALPN.as_bytes().to_vec()], "moqt" | "moql" => versions.alpns().iter().map(|alpn| alpn.as_bytes().to_vec()).collect(), _ => return Err(Error::InvalidScheme), }; @@ -409,8 +428,8 @@ impl NoqClient { config.alpn_protocols = alpns; config.key_log = Arc::new(rustls::KeyLogFile::new()); - let config: noq::crypto::rustls::QuicClientConfig = config.try_into()?; - let mut config = noq::ClientConfig::new(Arc::new(config)); + let config: driver::crypto::rustls::QuicClientConfig = config.try_into()?; + let mut config = driver::ClientConfig::new(Arc::new(config)); config.transport_config(self.transport.clone()); tracing::debug!(peer = %crate::connect::Endpoint(&url), "connecting"); @@ -429,15 +448,15 @@ impl NoqClient { let session = match url.scheme() { "https" => { - let mut request = web_transport_moq::proto::ConnectRequest::new(url.clone()); + let mut request = web::proto::ConnectRequest::new(url.clone()); for alpn in versions.alpns() { request = request.with_protocol(alpn.to_string()); } - web_transport_moq::Session::connect(connection, request) + web::Session::connect(connection, request) .await .map_err(map_client_error)? } - "moqt" | "moql" => web_transport_moq::Session::raw(connection), + "moqt" | "moql" => web::Session::raw(connection), _ => return Err(Error::UnsupportedScheme(url.scheme().to_string())), }; @@ -502,7 +521,7 @@ impl Error { } } -fn map_client_error(err: web_transport_moq::ClientError) -> Error { +fn map_client_error(err: web::ClientError) -> Error { match client_status(&err).and_then(crate::ConnectError::from_status_u16) { Some(rejected) => rejected.into(), None => err.into(), @@ -516,48 +535,49 @@ fn map_client_error(err: web_transport_moq::ClientError) -> Error { /// [`crate::ConnectError`], and [`Error::status`] hands it to the caller, whose backoff consults /// the status. A `404` or `405` is the server's settled answer, so retrying /// it just burns the reconnect budget on a URL that will never work. -fn client_status(err: &web_transport_moq::ClientError) -> Option { +fn client_status(err: &web::ClientError) -> Option { match err { - web_transport_moq::ClientError::HttpError(err) => connect_status(err), + web::ClientError::HttpError(err) => connect_status(err), _ => None, } } -fn connect_status(err: &web_transport_moq::ConnectError) -> Option { +fn connect_status(err: &web::ConnectError) -> Option { match err { - web_transport_moq::ConnectError::ErrorStatus(status) => Some(status.as_u16()), - web_transport_moq::ConnectError::ProtoError(err) => proto_status(err), + web::ConnectError::ErrorStatus(status) => Some(status.as_u16()), + web::ConnectError::ProtoError(err) => proto_status(err), _ => None, } } -fn proto_status(err: &web_transport_moq::proto::ConnectError) -> Option { +fn proto_status(err: &web::proto::ConnectError) -> Option { match err { - web_transport_moq::proto::ConnectError::ErrorStatus(status) - | web_transport_moq::proto::ConnectError::WrongStatus(Some(status)) => Some(status.as_u16()), + web::proto::ConnectError::ErrorStatus(status) | web::proto::ConnectError::WrongStatus(Some(status)) => { + Some(status.as_u16()) + } _ => None, } } // ── Server ────────────────────────────────────────────────────────── -pub(crate) struct NoqServer { - pub quic: noq::Endpoint, +pub(crate) struct Server { + pub quic: driver::Endpoint, tls: crate::tls::Serve, socket: Socket, } -/// The server's UDP socket, which [`NoqServer::shutdown`] closes even while noq +/// The server's UDP socket, which [`Server::shutdown`] closes even while the driver /// still holds references to it. /// -/// noq gives each connection a sender sharing the socket, and a connection lives +/// The driver gives each connection a sender sharing the socket, and a connection lives /// as long as anything holds its handle, so dropping the endpoint leaves the port /// bound until every accepted session is gone too. Taking the socket out of this /// slot closes it regardless. #[derive(Clone, Debug)] struct Socket { io: Arc>>>, - state: Arc, + state: Arc, local: net::SocketAddr, /// Wakes every sender blocked on a full send buffer from the socket's single /// write-readiness registration. @@ -566,7 +586,7 @@ struct Socket { impl Socket { fn new(socket: std::net::UdpSocket) -> std::io::Result { - let state = noq::udp::UdpSocketState::new((&socket).into())?; + let state = moq_sock::udp::UdpSocketState::new((&socket).into())?; let local = socket.local_addr()?; let io = tokio::net::UdpSocket::from_std(socket)?; Ok(Self { @@ -588,8 +608,8 @@ impl Socket { } } -impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { +impl driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { Box::pin(self.clone()) } @@ -597,7 +617,7 @@ impl noq::AsyncUdpSocket for Socket { &mut self, cx: &mut Context<'_>, bufs: &mut [std::io::IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { let io = self.io.read().unwrap(); // Released: nothing arrives again, and the endpoint stops through its close. @@ -617,7 +637,7 @@ impl noq::AsyncUdpSocket for Socket { Ok(self.local) } - fn max_receive_segments(&self) -> std::num::NonZeroUsize { + fn max_receive_segments(&self) -> usize { self.state.gro_segments() } @@ -626,10 +646,10 @@ impl noq::AsyncUdpSocket for Socket { } } -impl noq::UdpSender for Socket { +impl driver::UdpSender for Socket { fn poll_send( self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, + transmit: &moq_sock::udp::Transmit<'_>, cx: &mut Context<'_>, ) -> Poll> { let io = self.io.read().unwrap(); @@ -638,6 +658,9 @@ impl noq::UdpSender for Socket { return Poll::Ready(Ok(())); }; loop { + // `send` logs and drops a failed datagram rather than failing the connection, as + // UDP loss would; only `WouldBlock` comes back. + #[allow(deprecated)] match io.try_io(tokio::io::Interest::WRITABLE, || self.state.send(io.into(), transmit)) { Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} res => return Poll::Ready(res), @@ -650,7 +673,7 @@ impl noq::UdpSender for Socket { } } - fn max_transmit_segments(&self) -> std::num::NonZeroUsize { + fn max_transmit_segments(&self) -> usize { self.state.max_gso_segments() } } @@ -687,7 +710,7 @@ pub(crate) struct Options { pub tls: crate::tls::Serve, } -impl NoqServer { +impl Server { pub fn new(options: Options) -> Result { let Options { config, @@ -695,7 +718,7 @@ impl NoqServer { member, tls: served, } = options; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); let quic = quic.resolve(); apply_transport(&mut transport, &quic); apply_qlog(&mut transport, &quic, "server")?; @@ -708,17 +731,17 @@ impl NoqServer { .iter() .map(|alpn| alpn.as_bytes().to_vec()) .collect(); - alpns.push(web_transport_moq::ALPN.as_bytes().to_vec()); + alpns.push(web::ALPN.as_bytes().to_vec()); let mut tls = (*served.config(alpns)?).clone(); tls.key_log = Arc::new(rustls::KeyLogFile::new()); - let tls: noq::crypto::rustls::QuicServerConfig = tls.try_into()?; - let mut tls = noq::ServerConfig::with_crypto(Arc::new(tls)); + let tls: driver::crypto::rustls::QuicServerConfig = tls.try_into()?; + let mut tls = driver::ServerConfig::with_crypto(Arc::new(tls)); tls.transport_config(transport); // Advertise the preferred_address transport parameter (RFC 9000 §9.6). - // noq allocates a fresh CID + reset token for the address during the handshake. + // The driver allocates a fresh CID + reset token for the address during the handshake. if let Some(addr) = config.preferred_v4 { tls.preferred_address_v4(Some(addr)); } @@ -727,7 +750,8 @@ impl NoqServer { } // There's a bit more boilerplate to make a generic endpoint. - let runtime = noq::default_runtime().ok_or(Error::NoRuntime)?; + // The driver spawns its tasks on the current tokio runtime. + tokio::runtime::Handle::try_current().map_err(|_| Error::NoRuntime)?; let listen = config .bind @@ -739,7 +763,7 @@ impl NoqServer { let load_balancer = config.load_balancer(); // Configure connection ID generator with server ID if provided - let mut endpoint_config = noq::EndpointConfig::default(); + let mut endpoint_config = driver::EndpointConfig::default(); if let Some(shard) = member.as_ref().map(listen::Socket::shard) { if load_balancer.is_some() { return Err(Error::ShardWithQuicLb); @@ -781,9 +805,8 @@ impl NoqServer { // Create the generic QUIC endpoint. let socket = Socket::new(socket).map_err(Error::CreateEndpoint)?; - let quic = - noq::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone()), runtime) - .map_err(Error::CreateEndpoint)?; + let quic = driver::Endpoint::new_with_abstract_socket(endpoint_config, Some(tls), Box::new(socket.clone())) + .map_err(Error::CreateEndpoint)?; Ok(Self { quic, @@ -792,7 +815,7 @@ impl NoqServer { }) } - pub fn accept(&self) -> impl std::future::Future> + '_ { + pub fn accept(&self) -> impl std::future::Future> + '_ { self.quic.accept() } @@ -805,7 +828,7 @@ impl NoqServer { } pub fn close(&self) { - self.quic.close(noq::VarInt::from_u32(0), b"server shutdown"); + self.quic.close(driver::VarInt::from_u32(0), b"server shutdown"); } /// Close every connection, wait until each has sent its close to the peer, @@ -824,32 +847,34 @@ impl NoqServer { } } -// ── NoqRequest ────────────────────────────────────────────────────── +// ── QuicRequest ───────────────────────────────────────────────────── -/// A raw QUIC connection request without WebTransport framing (noq backend). +/// A raw QUIC connection request without WebTransport framing (QUIC backend). /// Accept a QUIC connection, negotiate WebTransport or raw moq, and complete the /// handshake (a `200 OK` for WebTransport). Returns the established session, the request /// URL and validated mTLS identity (both captured before the response consumes the /// request), and the dialed authority (the CONNECT authority on WebTransport, the TLS /// SNI on raw QUIC). Raw QUIC carries no request URL (the path rides the SETUP instead). pub(crate) async fn accept( - conn: noq::Incoming, + conn: driver::Incoming, alpns: Vec<&'static str>, -) -> Result> { +) -> Result> { let mut conn = conn.accept()?; - let handshake = conn - .handshake_data() - .await? - .downcast::() - .unwrap(); + // Scoped so the `!Send` boxed handshake data is gone before the next await. + let (alpn, host) = { + let handshake = conn + .handshake_data() + .await? + .downcast::() + .unwrap(); + (handshake.protocol, handshake.server_name) + }; - let alpn = handshake.protocol.ok_or(Error::MissingAlpn)?; + let alpn = alpn.ok_or(Error::MissingAlpn)?; let alpn = String::from_utf8(alpn)?; - let host = handshake.server_name.unwrap_or_default(); + let host = host.unwrap_or_default(); - // The established Connection no longer exposes a single peer address (noq 1.0 - // supports multipath), so capture it from the Connecting before awaiting. let remote = conn.remote_address(); tracing::debug!(%host, ip = %remote, %alpn, "accepting"); @@ -868,10 +893,10 @@ pub(crate) async fn accept( }; match alpn.as_str() { - web_transport_moq::ALPN => { + web::ALPN => { // Wait for the CONNECT request, then capture its URL and mTLS identity before // the response consumes it. - let request = web_transport_moq::Request::accept(conn) + let request = web::Request::accept(conn) .await .map_err(|err| Error::RecvRequest(crate::error::message(err)))?; let url = Some(request.url.clone()); @@ -879,7 +904,7 @@ pub(crate) async fn accept( // The authority the client put in its CONNECT URL. let authority = request.url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); - let mut response = web_transport_moq::proto::ConnectResponse::OK; + let mut response = web::proto::ConnectResponse::OK; let mut link = link; if let Some(protocol) = request.protocols.iter().find(|p| alpns.contains(&p.as_str())) { response = response.with_protocol(protocol); @@ -906,7 +931,7 @@ pub(crate) async fn accept( // Raw QUIC carries no request URL; the path rides the SETUP. The TLS SNI is the // only authority the client can offer here, and it is optional. let authority = (!host.is_empty()).then_some(host); - let session = web_transport_moq::Session::raw(conn); + let session = web::Session::raw(conn); Ok(crate::server::Accepted { session, url: None, @@ -939,13 +964,13 @@ impl ShardIdGenerator { } } -impl noq::ConnectionIdGenerator for ShardIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ShardIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let mut cid = Vec::with_capacity(Self::LEN); cid.push(moq_sock::shard::cid_prefix(self.shard)); cid.extend(rand::rng().random_iter::().take(Self::LEN - 1)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -970,8 +995,8 @@ impl ServerIdGenerator { } } -impl noq::ConnectionIdGenerator for ServerIdGenerator { - fn generate_cid(&mut self) -> noq::ConnectionId { +impl driver::ConnectionIdGenerator for ServerIdGenerator { + fn generate_cid(&mut self) -> driver::ConnectionId { use rand::RngExt; let cid_len = self.cid_len(); let mut cid = Vec::with_capacity(cid_len); @@ -979,7 +1004,7 @@ impl noq::ConnectionIdGenerator for ServerIdGenerator { cid.push((cid_len - 1) as u8); cid.extend(self.server_id.0.iter()); cid.extend(rand::rng().random_iter::().take(self.nonce_len)); - noq::ConnectionId::new(cid.as_slice()) + driver::ConnectionId::new(cid.as_slice()) } fn cid_len(&self) -> usize { @@ -1022,7 +1047,7 @@ mod tests { ..Default::default() }; let tls = config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config, quic: quic.clone(), member: None, @@ -1040,7 +1065,7 @@ mod tests { ..Default::default() }; let tls = config.tls.build().expect("tls config"); - let client = NoqClient::new(&config, &quic).expect("client init"); + let client = Client::new(&config, &quic).expect("client init"); let url: Url = format!("moqt://{addr}/.cluster/test").parse().unwrap(); let versions = moq_net::Versions::default(); let dial = client.connect(&tls, url.into(), &versions); @@ -1054,18 +1079,20 @@ mod tests { let (_client, _server) = result.expect("pinned connection failed"); } - /// noq exposes no getters for the flow-control windows, but its `Debug` prints + /// moq-quic exposes no getters for the flow-control windows, but its `Debug` prints /// them, which is enough to prove each one reached the transport config, that an - /// unset connection window gets our finite default instead of noq's unlimited one, - /// and that the other unset knobs leave noq's own defaults in place. + /// unset connection window gets our finite default instead of moq-quic's unlimited one, + /// and that the other unset knobs leave moq-quic's own defaults in place. #[test] fn apply_windows_writes_each_field() { - let mut quic = crate::quic::Config::default(); - quic.receive_window = Some(128 << 20); - quic.stream_receive_window = Some(8 << 20); - quic.send_window = Some(32 << 20); + let quic = crate::quic::Config { + receive_window: Some(128 << 20), + stream_receive_window: Some(8 << 20), + send_window: Some(32 << 20), + ..Default::default() + }; - let mut transport = noq::TransportConfig::default(); + let mut transport = driver::TransportConfig::default(); apply_windows(&mut transport, &quic.resolve()); let applied = format!("{:?}", transport); @@ -1077,33 +1104,33 @@ mod tests { assert!(applied.contains(&field), "{field} missing from {applied}"); } - let mut untouched = noq::TransportConfig::default(); + let mut untouched = driver::TransportConfig::default(); apply_windows(&mut untouched, &crate::quic::Config::default().resolve()); - let mut expected = noq::TransportConfig::default(); - expected.receive_window(noq::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); + let mut expected = driver::TransportConfig::default(); + expected.receive_window(driver::VarInt::from_u64(crate::quic::DEFAULT_RECEIVE_WINDOW).unwrap()); assert_eq!(format!("{:?}", untouched), format!("{:?}", expected)); assert_ne!( format!("{:?}", untouched), - format!("{:?}", noq::TransportConfig::default()), - "noq's own connection window is unlimited, so ours must replace it" + format!("{:?}", driver::TransportConfig::default()), + "moq-quic's own connection window is unlimited, so ours must replace it" ); } /// Build a controller from each family's factory and downcast it to the - /// concrete noq implementation it must map to. + /// concrete moq-quic implementation it must map to. #[test] fn congestion_factory_maps_each_family() { - let now = std::time::Instant::now(); + let now = moq_clock::Instant::now(); let mtu = 1200; let loss = congestion_factory(CongestionControl::Loss).build(now, mtu); - assert!(loss.into_any().downcast::().is_ok()); + assert!(loss.into_any().downcast::().is_ok()); let delay = congestion_factory(CongestionControl::Delay).build(now, mtu); - assert!(delay.into_any().downcast::().is_ok()); + assert!(delay.into_any().downcast::().is_ok()); } - /// Loopback regression test: with the knob unset, live noq connections must run + /// Loopback regression test: with the knob unset, live QUIC connections must run /// BBRv3 on both ends. #[tokio::test] async fn default_reaches_the_live_connection() { @@ -1120,7 +1147,7 @@ mod tests { let quic = crate::quic::Config::default(); let tls = server_config.tls.load().unwrap(); - let server = NoqServer::new(Options { + let server = Server::new(Options { config: server_config, quic: quic.clone(), member: None, @@ -1146,7 +1173,7 @@ mod tests { }; let tls = client_config.tls.build().expect("tls config"); - let client = NoqClient::new(&client_config, &quic).expect("client init"); + let client = Client::new(&client_config, &quic).expect("client init"); // Dial the loopback IP directly so the system resolver is never involved. let url: Url = format!("moqt://127.0.0.1:{}", addr.port()).parse().unwrap(); @@ -1158,7 +1185,7 @@ mod tests { .await .expect("connect failed"); - // web_transport_moq::Session derefs to the noq connection. + // web::Session derefs to the driver connection. assert!(is_bbr3(&session), "client connection is not running BBRv3"); assert!( accepted.await.expect("server task panicked"), @@ -1169,15 +1196,11 @@ mod tests { .expect("test timed out"); } - /// Whether a live connection's initial path is running BBRv3. - /// - /// noq is multipath, so the controller is per path rather than per connection; - /// `PathId::ZERO` is the path the handshake came up on. - fn is_bbr3(conn: &noq::Connection) -> bool { - conn.congestion_state(noq::PathId::ZERO) - .expect("no controller on the initial path") + /// Whether a live connection is running BBRv3. + fn is_bbr3(conn: &driver::Connection) -> bool { + conn.congestion_state() .into_any() - .downcast::() + .downcast::() .is_ok() } } diff --git a/rs/moq-tokio/src/quic/driver/.rustfmt.toml b/rs/moq-tokio/src/quic/driver/.rustfmt.toml new file mode 100644 index 0000000000..daf6479bc8 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/.rustfmt.toml @@ -0,0 +1,2 @@ +# quinn's formatting (rustfmt defaults) rather than rs/.rustfmt.toml, so +# upstream fixes cherry-pick without whitespace conflicts. diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 6ce968db47..605163a673 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -23,15 +23,11 @@ use tokio::sync::{ }; use tracing::{Instrument, Span, debug_span}; -use crate::{ - ConnectionEvent, Duration, Instant, VarInt, - mutex::Mutex, - recv_stream::RecvStream, - runtime::{AsyncTimer, Runtime, UdpSender}, - send_stream::SendStream, - udp_transmit, +use crate::quic::driver::{ + ConnectionEvent, Duration, Instant, VarInt, mutex::Mutex, recv_stream::RecvStream, + runtime::UdpSender, send_stream::SendStream, udp_transmit, }; -use proto::{ +use moq_quic::{ ConnectionError, ConnectionHandle, ConnectionStats, Dir, EndpointEvent, Side, StreamEvent, StreamId, TransportError, TransportErrorCode, congestion::Controller, }; @@ -53,11 +49,10 @@ pub struct Connecting { impl Connecting { pub(crate) fn new( handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, sender: Pin>, - runtime: Arc, ) -> Self { let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); @@ -69,21 +64,20 @@ impl Connecting { conn_events, on_handshake_data_send, sender, - runtime.clone(), )), shared: Shared::default(), })); let connected = Box::pin(conn.shared.connected.clone().notified_owned()); let driver = ConnectionDriver::new(conn.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {e}"); } } .instrument(Span::current()), - )); + ); Self { conn: Some(conn), @@ -104,7 +98,7 @@ impl Connecting { /// ## Outgoing /// /// For outgoing connections, the initial attempt to convert to a [`Connection`] which sends - /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::crypto::ClientConfig] + /// 0-RTT data will proceed if the [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig] /// attempts to resume a previous TLS session. However, **the remote endpoint may not actually /// _accept_ the 0-RTT data**--yet still accept the connection attempt in general. This /// possibility is conveyed through `ZeroRttRejected` errors on stream I/O methods, which may @@ -119,14 +113,14 @@ impl Connecting { /// relevant resumption state to be stored in the server, which servers may limit or lose for /// various reasons including not persisting resumption state across server restarts. /// - /// If manually providing a [`crypto::ClientConfig`][crate::crypto::ClientConfig], check your + /// If manually providing a [`crypto::ClientConfig`][crate::quic::driver::crypto::ClientConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Incoming /// /// For incoming connections, conversion to 0.5-RTT will always succeed. /// - /// If manually providing a [`crypto::ServerConfig`][crate::crypto::ServerConfig], check your + /// If manually providing a [`crypto::ServerConfig`][crate::quic::driver::crypto::ServerConfig], check your /// implementation's docs for 0-RTT pitfalls. /// /// ## Security @@ -156,9 +150,9 @@ impl Connecting { /// Parameters negotiated during the handshake /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a - /// [`crypto::rustls::HandshakeData`](crate::crypto::rustls::HandshakeData). + /// [`crypto::rustls::HandshakeData`](crate::quic::driver::crypto::rustls::HandshakeData). /// /// Peer certificates are not available at this stage; see [`Connection::peer_identity()`]. /// @@ -194,7 +188,7 @@ impl Connecting { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. /// /// Will panic if called after `poll` has returned `Ready`. @@ -341,9 +335,9 @@ impl Connection { /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing /// anything to [`SendStream`] will never succeed. /// - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream pub fn open_bi(&self) -> OpenBi<'_> { OpenBi { conn: &self.0, @@ -369,10 +363,10 @@ impl Connection { /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. /// - /// [`accept_bi()`]: crate::Connection::accept_bi - /// [`open_bi()`]: crate::Connection::open_bi - /// [`SendStream`]: crate::SendStream - /// [`RecvStream`]: crate::RecvStream + /// [`accept_bi()`]: crate::quic::driver::Connection::accept_bi + /// [`open_bi()`]: crate::quic::driver::Connection::open_bi + /// [`SendStream`]: crate::quic::driver::SendStream + /// [`RecvStream`]: crate::quic::driver::RecvStream /// /// # Cancel safety /// @@ -462,8 +456,8 @@ impl Connection { /// undelivered to the application, including data that was acknowledged as received to /// the local endpoint. /// - /// [`ConnectionError::LocallyClosed`]: crate::ConnectionError::LocallyClosed - /// [`Endpoint::wait_idle()`]: crate::Endpoint::wait_idle + /// [`ConnectionError::LocallyClosed`]: crate::quic::driver::ConnectionError::LocallyClosed + /// [`Endpoint::wait_idle()`]: crate::quic::driver::Endpoint::wait_idle /// [`close()`]: Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let conn = &mut *self.0.state.lock("close"); @@ -515,7 +509,7 @@ impl Connection { if let Some(x) = &conn.error { return Err(SendDatagramError::ConnectionLost(x.clone())); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match conn.inner.datagrams().send(data, true) { Ok(()) => { conn.wake(); @@ -614,7 +608,7 @@ impl Connection { /// the endpoint is bound to a wildcard address like `0.0.0.0` or `::`. /// /// This will return `None` for clients, or when the platform does not expose this - /// information. See [`quinn_udp::RecvMeta::dst_ip`](udp::RecvMeta::dst_ip) for a list of + /// information. See [`quinn_udp::RecvMeta::dst_ip`](moq_sock::udp::RecvMeta::dst_ip) for a list of /// supported platforms when using [`quinn_udp`](udp) for I/O, which is the default. pub fn local_ip(&self) -> Option { self.0.state.lock("local_ip").inner.local_ip() @@ -678,7 +672,7 @@ impl Connection { /// [`Connecting::handshake_data()`] succeeds. See that method's documentations for details on /// the returned value. /// - /// [`Connection::handshake_data()`]: crate::Connecting::handshake_data + /// [`Connection::handshake_data()`]: crate::quic::driver::Connecting::handshake_data pub fn handshake_data(&self) -> Option> { self.0 .state @@ -691,7 +685,7 @@ impl Connection { /// Cryptographic identity of the peer /// /// The dynamic type returned is determined by the configured - /// [`Session`](proto::crypto::Session). For the default `rustls` session, the return value can + /// [`Session`](moq_quic::crypto::Session). For the default `rustls` session, the return value can /// be [`downcast`](Box::downcast) to a Vec<[rustls::pki_types::CertificateDer]>. /// /// For custom asynchronous validation (e.g. checking a certificate against an external @@ -738,7 +732,7 @@ impl Connection { output: &mut [u8], label: &[u8], context: &[u8], - ) -> Result<(), proto::crypto::ExportKeyingMaterialError> { + ) -> Result<(), moq_quic::crypto::ExportKeyingMaterialError> { self.0 .state .lock("export_keying_material") @@ -758,14 +752,14 @@ impl Connection { conn.wake(); } - /// See [`proto::TransportConfig::send_window()`] + /// See [`moq_quic::TransportConfig::send_window()`] pub fn set_send_window(&self, send_window: u64) { let mut conn = self.0.state.lock("set_send_window"); conn.inner.set_send_window(send_window); conn.wake(); } - /// See [`proto::TransportConfig::receive_window()`] + /// See [`moq_quic::TransportConfig::receive_window()`] pub fn set_receive_window(&self, receive_window: VarInt) { let mut conn = self.0.state.lock("set_receive_window"); conn.inner.set_receive_window(receive_window); @@ -970,7 +964,7 @@ impl Future for SendDatagram<'_> { if let Some(e) = &state.error { return Poll::Ready(Err(SendDatagramError::ConnectionLost(e.clone()))); } - use proto::SendDatagramError::*; + use moq_quic::SendDatagramError::*; match state .inner .datagrams() @@ -1084,13 +1078,13 @@ pub(crate) struct Shared { } pub(crate) struct State { - pub(crate) inner: proto::Connection, + pub(crate) inner: moq_quic::Connection, driver: Option, handle: ConnectionHandle, on_handshake_data: Option>, connected: bool, handshake_confirmed: bool, - timer: Option>>, + timer: Option, timer_deadline: Option, conn_events: mpsc::UnboundedReceiver, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, @@ -1100,23 +1094,21 @@ pub(crate) struct State { /// Always set to Some before the connection becomes drained pub(crate) error: Option, sender: Pin>, - runtime: Arc, send_buffer: Vec, /// We buffer a transmit when the underlying I/O would block - buffered_transmit: Option, + buffered_transmit: Option, remote_address: watch::Sender, } impl State { #[allow(clippy::too_many_arguments)] fn new( - inner: proto::Connection, + inner: moq_quic::Connection, handle: ConnectionHandle, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::UnboundedReceiver, on_handshake_data: oneshot::Sender<()>, sender: Pin>, - runtime: Arc, ) -> Self { let remote_address = inner.remote_address(); @@ -1136,7 +1128,6 @@ impl State { stopped: FxHashMap::default(), error: None, sender, - runtime, send_buffer: Vec::new(), buffered_transmit: None, remote_address: watch::Sender::new(remote_address), @@ -1144,7 +1135,7 @@ impl State { } fn drive_transmit(&mut self, cx: &mut Context<'_>) -> io::Result { - let now = self.runtime.now(); + let now = Instant::now(); let mut transmits = 0; let max_datagrams = self @@ -1241,7 +1232,7 @@ impl State { fn forward_app_events(&mut self, shared: &Shared) { while let Some(event) = self.inner.poll() { - use proto::Event::*; + use moq_quic::Event::*; match event { HandshakeDataReady => { if let Some(x) = self.on_handshake_data.take() { @@ -1310,7 +1301,7 @@ impl State { // Use the clock rather than the async timer to detect expiry: Sleep::poll // respects Tokio's cooperative budget and can return Pending for elapsed // deadlines. - let now = self.runtime.now(); + let now = Instant::now(); if now >= deadline { self.inner.handle_timeout(now); self.timer_deadline = None; @@ -1320,26 +1311,22 @@ impl State { match &mut self.timer { // Avoid resetting the timer when the deadline is unchanged. Some(delay) if self.timer_deadline != Some(deadline) => { - delay.as_mut().reset(deadline); + delay.reset(deadline); } None => { - self.timer = Some(self.runtime.new_timer(deadline)); + self.timer = Some(moq_clock::sleep_until(deadline)); } _ => {} } self.timer_deadline = Some(deadline); - let delay = self - .timer - .as_mut() - .expect("timer must exist in this state") - .as_mut(); - if delay.poll(cx).is_pending() { + let delay = self.timer.as_mut().expect("timer must exist in this state"); + if Pin::new(delay).poll(cx).is_pending() { return false; } // The deadline elapsed in the window between the clock check and poll. - self.inner.handle_timeout(self.runtime.now()); + self.inner.handle_timeout(Instant::now()); self.timer_deadline = None; true } @@ -1372,7 +1359,7 @@ impl State { } fn close(&mut self, error_code: VarInt, reason: Bytes, shared: &Shared) { - self.inner.close(self.runtime.now(), error_code, reason); + self.inner.close(Instant::now(), error_code, reason); self.terminate(ConnectionError::LocallyClosed, shared); self.wake(); } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2012e415ff..2bae9031dc 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -14,35 +14,24 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] -use crate::runtime::default_runtime; -use crate::{ +use crate::quic::driver::{ Instant, - runtime::{AsyncUdpSocket, Runtime, UdpSender}, + runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}, udp_transmit, }; use bytes::{Bytes, BytesMut}; -use pin_project_lite::pin_project; -use proto::{ - self as proto, ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, - EndpointEvent, ServerConfig, +use moq_quic::{ + ClientConfig, ConnectError, ConnectionError, ConnectionHandle, DatagramEvent, EndpointEvent, + ServerConfig, }; -use rustc_hash::FxHashMap; -#[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), -))] +use moq_sock::udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; +use pin_project_lite::pin_project; +use rustc_hash::{FxHashMap, FxHashSet}; use socket2::{Domain, Protocol, Socket, Type}; use tokio::sync::{Notify, futures::Notified, mpsc}; use tracing::{Instrument, Span}; -use udp::{BATCH_SIZE, RecvMeta, is_msg_size_err}; -use crate::{ +use crate::quic::driver::{ ConnectionEvent, EndpointConfig, IO_LOOP_BOUND, RECV_TIME_BOUND, VarInt, connection::Connecting, incoming::Incoming, work_limiter::WorkLimiter, }; @@ -56,7 +45,6 @@ use crate::{ #[derive(Debug, Clone)] pub struct Endpoint { pub(crate) inner: EndpointRef, - runtime: Arc, } impl Endpoint { @@ -77,11 +65,6 @@ impl Endpoint { /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// client will only be able to connect to IPv6 servers. An IPv4 client is never dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn client(addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -90,13 +73,10 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), None, - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } @@ -119,11 +99,6 @@ impl Endpoint { /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 /// server will only be able to accept connections from IPv6 clients. An IPv4 server is never /// dual-stack. - #[cfg(all( - not(wasm_browser), - any(feature = "runtime-tokio", feature = "runtime-smol"), - any(feature = "aws-lc-rs", feature = "ring"), // `EndpointConfig::default()` is only available with these - ))] pub fn server(config: ServerConfig, addr: SocketAddr) -> io::Result { let socket = Socket::new(Domain::for_address(addr), Type::DGRAM, Some(Protocol::UDP))?; if addr.is_ipv6() @@ -132,26 +107,21 @@ impl Endpoint { tracing::debug!(%e, "unable to make socket dual-stack"); } socket.bind(&addr.into())?; - let runtime = - default_runtime().ok_or_else(|| io::Error::other("no async runtime found"))?; Self::new_with_abstract_socket( EndpointConfig::default(), Some(config), - runtime.wrap_udp_socket(socket.into())?, - runtime, + wrap_udp_socket(socket.into())?, ) } /// Construct an endpoint with arbitrary configuration and socket - #[cfg(not(wasm_browser))] pub fn new( config: EndpointConfig, server_config: Option, socket: std::net::UdpSocket, - runtime: Arc, ) -> io::Result { - let socket = runtime.wrap_udp_socket(socket)?; - Self::new_with_abstract_socket(config, server_config, socket, runtime) + let socket = wrap_udp_socket(socket)?; + Self::new_with_abstract_socket(config, server_config, socket) } /// Construct an endpoint with arbitrary configuration and pre-constructed abstract socket @@ -162,32 +132,30 @@ impl Endpoint { config: EndpointConfig, server_config: Option, socket: Box, - runtime: Arc, ) -> io::Result { let addr = socket.local_addr()?; let allow_mtud = !socket.may_fragment(); let rc = EndpointRef::new( socket, - proto::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), + moq_quic::Endpoint::new(Arc::new(config), server_config.map(Arc::new), allow_mtud), addr.is_ipv6(), - runtime.clone(), ); let driver = EndpointDriver(rc.clone()); - runtime.spawn(Box::pin( + tokio::spawn( async { if let Err(e) = driver.await { tracing::error!("I/O error: {}", e); } } .instrument(Span::current()), - )); - Ok(Self { inner: rc, runtime }) + ); + Ok(Self { inner: rc }) } /// Get the next incoming connection attempt from a client /// /// Yields [`Incoming`]s, or `None` if the endpoint is [`close`](Self::close)d. [`Incoming`] - /// can be `await`ed to obtain the final [`Connection`](crate::Connection), or used to e.g. + /// can be `await`ed to obtain the final [`Connection`](crate::quic::driver::Connection), or used to e.g. /// filter connection attempts or force address validation, or converted into an intermediate /// `Connecting` future which can be used to e.g. send 0.5-RTT data. pub fn accept(&self) -> Accept<'_> { @@ -252,22 +220,21 @@ impl Endpoint { let (ch, conn) = endpoint .inner - .connect(self.runtime.now(), config, addr, server_name)?; + .connect(Instant::now(), config, addr, server_name)?; let sender = endpoint.socket.create_sender(); endpoint.stats.outgoing_handshakes += 1; Ok(endpoint .recv_state .connections - .insert(ch, conn, sender, false, self.runtime.clone())) + .insert(ch, conn, sender, false)) } /// Switch to a new UDP socket /// /// See [`Endpoint::rebind_abstract()`] for details. - #[cfg(not(wasm_browser))] pub fn rebind(&self, socket: std::net::UdpSocket) -> io::Result<()> { - self.rebind_abstract(self.runtime.wrap_udp_socket(socket)?) + self.rebind_abstract(wrap_udp_socket(socket)?) } /// Switch to a new UDP socket @@ -322,7 +289,7 @@ impl Endpoint { /// /// See [`Connection::close()`] for details. /// - /// [`Connection::close()`]: crate::Connection::close + /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); let mut endpoint = self.inner.state.lock().unwrap(); @@ -360,6 +327,32 @@ impl Endpoint { .await; } } + + /// Wait until no connection on the endpoint is still active + /// + /// A connection stops being active once its peer's close arrives, or once its closing + /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out + /// the closing period for a peer that already has the close, nor wait for the + /// [`Connection`](crate::quic::driver::Connection) handles to drop. + /// + /// Does not proactively close existing connections or cause incoming connections to be + /// rejected. Consider calling [`close()`] if that is desired. + /// + /// [`close()`]: Endpoint::close + /// [`wait_idle()`]: Endpoint::wait_idle + pub async fn wait_all_draining(&self) { + loop { + { + let endpoint = &mut *self.inner.state.lock().unwrap(); + if endpoint.recv_state.connections.all_draining() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.all_draining.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -400,7 +393,7 @@ impl Future for EndpointDriver { None => endpoint.driver = Some(cx.waker().clone()), } - let now = endpoint.runtime.now(); + let now = Instant::now(); let mut keep_going = false; keep_going |= endpoint.drive_recv(cx, now)?; keep_going |= endpoint.handle_events(cx, &self.0.shared); @@ -432,6 +425,8 @@ impl Drop for EndpointDriver { // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); + endpoint.recv_state.connections.draining.clear(); + self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else // can wake it. Same reason as the equivalent check in `EndpointInner::accept`. @@ -450,7 +445,7 @@ pub(crate) struct EndpointInner { impl EndpointInner { pub(crate) fn accept( &self, - incoming: proto::Incoming, + incoming: moq_quic::Incoming, server_config: Option>, ) -> Result { let mut response_buffer = Vec::new(); @@ -458,7 +453,7 @@ impl EndpointInner { // Phase 1: reserve endpoint state for the connection under the lock. let accepting = { let mut state = self.state.lock().unwrap(); - let now = state.runtime.now(); + let now = Instant::now(); match state .inner .start_accept(incoming, now, &mut response_buffer, server_config) @@ -487,12 +482,11 @@ impl EndpointInner { Ok((handle, conn)) => { state.stats.accepted_handshakes += 1; let sender = state.socket.create_sender(); - let runtime = state.runtime.clone(); let driver_lost = state.driver_lost; Ok(state .recv_state .connections - .insert(handle, conn, sender, driver_lost, runtime)) + .insert(handle, conn, sender, driver_lost)) } Err(error) => { if let Some(transmit) = error.response { @@ -509,7 +503,7 @@ impl EndpointInner { result } - pub(crate) fn refuse(&self, incoming: proto::Incoming) { + pub(crate) fn refuse(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.refused_handshakes += 1; let mut response_buffer = Vec::new(); @@ -517,7 +511,7 @@ impl EndpointInner { respond(transmit, &response_buffer, &mut state.sender); } - pub(crate) fn retry(&self, incoming: proto::Incoming) -> Result<(), proto::RetryError> { + pub(crate) fn retry(&self, incoming: moq_quic::Incoming) -> Result<(), moq_quic::RetryError> { let mut state = self.state.lock().unwrap(); let mut response_buffer = Vec::new(); let transmit = state.inner.retry(incoming, &mut response_buffer)?; @@ -525,7 +519,7 @@ impl EndpointInner { Ok(()) } - pub(crate) fn ignore(&self, incoming: proto::Incoming) { + pub(crate) fn ignore(&self, incoming: moq_quic::Incoming) { let mut state = self.state.lock().unwrap(); state.stats.ignored_handshakes += 1; state.inner.ignore(incoming); @@ -539,13 +533,12 @@ pub(crate) struct State { /// During an active migration, abandoned_socket receives traffic /// until the first packet arrives on the new socket. prev_socket: Option>, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, recv_state: RecvState, driver: Option, ipv6: bool, events: mpsc::UnboundedReceiver<(ConnectionHandle, EndpointEvent)>, driver_lost: bool, - runtime: Arc, stats: EndpointStats, default_client_config: Option, /// Connections in the process of being accepted @@ -556,13 +549,14 @@ pub(crate) struct State { pub(crate) struct Shared { incoming: Notify, idle: Notify, + all_draining: Notify, /// Number of live handles that can be used to initiate or handle I/O; excludes the driver ref_count: AtomicUsize, } impl State { fn drive_recv(&mut self, cx: &mut Context<'_>, now: Instant) -> Result { - let get_time = || self.runtime.now(); + let get_time = Instant::now; self.recv_state.recv_limiter.start_cycle(get_time); if let Some(socket) = &mut self.prev_socket { // We don't care about the `PollProgress` from old sockets. @@ -571,7 +565,6 @@ impl State { &mut self.inner, &mut **socket, &mut self.sender, - &*self.runtime, now, ); if poll_res.is_err() { @@ -583,7 +576,6 @@ impl State { &mut self.inner, &mut *self.socket, &mut self.sender, - &*self.runtime, now, ); self.recv_state.recv_limiter.finish_cycle(get_time); @@ -606,12 +598,22 @@ impl State { } }; + let connections = &mut self.recv_state.connections; + if event.is_draining() && connections.senders.contains_key(&ch) { + connections.draining.insert(ch); + } if event.is_drained() { - self.recv_state.connections.senders.remove(&ch); + connections.senders.remove(&ch); + connections.draining.remove(&ch); if self.is_idle() { shared.idle.notify_waiters(); } } + if (event.is_draining() || event.is_drained()) + && self.recv_state.connections.all_draining() + { + shared.all_draining.notify_waiters(); + } let Some(event) = self.inner.handle_event(ch, event) else { continue; }; @@ -642,7 +644,7 @@ impl Drop for State { } fn respond( - transmit: proto::Transmit, + transmit: moq_quic::Transmit, response_buffer: &[u8], sender: &mut Pin>, ) { @@ -692,11 +694,11 @@ fn respond( } #[inline] -fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { +fn proto_ecn(ecn: moq_sock::udp::EcnCodepoint) -> moq_quic::EcnCodepoint { match ecn { - udp::EcnCodepoint::Ect0 => proto::EcnCodepoint::Ect0, - udp::EcnCodepoint::Ect1 => proto::EcnCodepoint::Ect1, - udp::EcnCodepoint::Ce => proto::EcnCodepoint::Ce, + moq_sock::udp::EcnCodepoint::Ect0 => moq_quic::EcnCodepoint::Ect0, + moq_sock::udp::EcnCodepoint::Ect1 => moq_quic::EcnCodepoint::Ect1, + moq_sock::udp::EcnCodepoint::Ce => moq_quic::EcnCodepoint::Ce, } } @@ -704,6 +706,9 @@ fn proto_ecn(ecn: udp::EcnCodepoint) -> proto::EcnCodepoint { struct ConnectionSet { /// Senders for communicating with the endpoint's connections senders: FxHashMap>, + /// The connections in `senders` that are no longer active: closed by the peer, or past + /// their closing period + draining: FxHashSet, /// Stored to give out clones to new ConnectionInners sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, /// Set if the endpoint has been manually closed @@ -714,10 +719,9 @@ impl ConnectionSet { fn insert( &mut self, handle: ConnectionHandle, - conn: proto::Connection, + conn: moq_quic::Connection, sender: Pin>, driver_lost: bool, - runtime: Arc, ) -> Connecting { let (send, recv) = mpsc::unbounded_channel(); if let Some((error_code, reason)) = &self.close { @@ -735,12 +739,17 @@ impl ConnectionSet { self.senders.insert(handle, send); } } - Connecting::new(handle, conn, self.sender.clone(), recv, sender, runtime) + Connecting::new(handle, conn, self.sender.clone(), recv, sender) } fn is_empty(&self) -> bool { self.senders.is_empty() } + + /// Whether no connection is still active + fn all_draining(&self) -> bool { + self.senders.len() == self.draining.len() + } } fn ensure_ipv6(x: SocketAddr) -> SocketAddrV6 { @@ -795,9 +804,8 @@ pub(crate) struct EndpointRef(Arc); impl EndpointRef { pub(crate) fn new( socket: Box, - inner: proto::Endpoint, + inner: moq_quic::Endpoint, ipv6: bool, - runtime: Arc, ) -> Self { let (sender, events) = mpsc::unbounded_channel(); let recv_state = RecvState::new(sender, socket.max_receive_segments(), &inner); @@ -806,6 +814,7 @@ impl EndpointRef { shared: Shared { incoming: Notify::new(), idle: Notify::new(), + all_draining: Notify::new(), ref_count: AtomicUsize::new(0), }, state: Mutex::new(State { @@ -818,7 +827,6 @@ impl EndpointRef { driver: None, driver_lost: false, recv_state, - runtime, stats: EndpointStats::default(), default_client_config: None, pending_accepts: 0, @@ -858,7 +866,7 @@ impl std::ops::Deref for EndpointRef { /// State directly involved in handling incoming packets struct RecvState { - incoming: VecDeque, + incoming: VecDeque, connections: ConnectionSet, recv_buf: Box<[u8]>, /// Reused buffer that received datagrams are copied into and split into owned slices. @@ -870,7 +878,7 @@ impl RecvState { fn new( sender: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, max_receive_segments: usize, - endpoint: &proto::Endpoint, + endpoint: &moq_quic::Endpoint, ) -> Self { let datagram_capacity = endpoint.config().get_max_udp_payload_size().min(64 * 1024) as usize @@ -879,6 +887,7 @@ impl RecvState { Self { connections: ConnectionSet { senders: FxHashMap::default(), + draining: FxHashSet::default(), sender, close: None, }, @@ -892,10 +901,9 @@ impl RecvState { fn poll_socket( &mut self, cx: &mut Context<'_>, - endpoint: &mut proto::Endpoint, + endpoint: &mut moq_quic::Endpoint, socket: &mut dyn AsyncUdpSocket, sender: &mut Pin>, - runtime: &dyn Runtime, now: Instant, ) -> Result { let mut received_connection_packet = false; @@ -990,7 +998,7 @@ impl RecvState { return Err(e); } } - if !self.recv_limiter.allow_work(|| runtime.now()) { + if !self.recv_limiter.allow_work(Instant::now) { return Ok(PollProgress { received_connection_packet, keep_going: true, diff --git a/rs/moq-tokio/src/quic/driver/incoming.rs b/rs/moq-tokio/src/quic/driver/incoming.rs index b5d60a08ec..64a08e493c 100644 --- a/rs/moq-tokio/src/quic/driver/incoming.rs +++ b/rs/moq-tokio/src/quic/driver/incoming.rs @@ -6,10 +6,10 @@ use std::{ task::{Context, Poll}, }; -use proto::{ConnectionError, ConnectionId, ServerConfig}; +use moq_quic::{ConnectionError, ConnectionId, ServerConfig}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ connection::{Connecting, Connection}, endpoint::EndpointRef, }; @@ -19,7 +19,7 @@ use crate::{ pub struct Incoming(Option); impl Incoming { - pub(crate) fn new(inner: proto::Incoming, endpoint: EndpointRef) -> Self { + pub(crate) fn new(inner: moq_quic::Incoming, endpoint: EndpointRef) -> Self { Self(Some(State { inner, endpoint })) } @@ -111,7 +111,7 @@ impl Drop for Incoming { #[derive(Debug)] struct State { - inner: proto::Incoming, + inner: moq_quic::Incoming, endpoint: EndpointRef, } diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index 649801fff8..d758cc9c7c 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -50,14 +50,11 @@ mod runtime; mod send_stream; mod work_limiter; -#[cfg(not(wasm_browser))] -pub(crate) use std::time::{Duration, Instant}; -#[cfg(wasm_browser)] -pub(crate) use web_time::{Duration, Instant}; +pub(crate) use moq_clock::Instant; +pub(crate) use std::time::Duration; -#[cfg(feature = "bloom")] -pub use proto::BloomTokenLog; -pub use proto::{ +pub use moq_quic::BloomTokenLog; +pub use moq_quic::{ AckFrequencyConfig, ApplicationClose, Chunk, ClientConfig, ClosedStream, ConfigError, ConnectError, ConnectionClose, ConnectionError, ConnectionId, ConnectionIdGenerator, ConnectionStats, Dir, EcnCodepoint, EndpointConfig, FrameStats, FrameType, IdleTimeout, @@ -67,42 +64,37 @@ pub use proto::{ VarIntBoundsExceeded, Written, congestion, crypto, }; #[cfg(feature = "qlog")] -pub use proto::{QlogConfig, QlogStream}; -#[cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] -pub use rustls; -pub use udp; +pub use moq_quic::{QlogConfig, QlogStream}; -pub use crate::connection::{ +pub use crate::quic::driver::connection::{ AcceptBi, AcceptUni, Connecting, Connection, OpenBi, OpenUni, ReadDatagram, RemoteAddressWatcher, SendDatagram, SendDatagramError, }; -pub use crate::endpoint::{Accept, Endpoint, EndpointStats}; -pub use crate::incoming::{Incoming, IncomingFuture, RetryError}; -pub use crate::recv_stream::{ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; -#[cfg(feature = "runtime-smol")] -pub use crate::runtime::SmolRuntime; -#[cfg(feature = "runtime-tokio")] -pub use crate::runtime::TokioRuntime; -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -pub use crate::runtime::default_runtime; -pub use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSender}; -pub use crate::send_stream::{SendStream, StoppedError, WriteError}; +pub use crate::quic::driver::endpoint::{Accept, Endpoint, EndpointStats}; +pub use crate::quic::driver::incoming::{Incoming, IncomingFuture, RetryError}; +pub use crate::quic::driver::recv_stream::{ + ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError, +}; +#[cfg(test)] +pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; +pub use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender}; +pub use crate::quic::driver::send_stream::{SendStream, StoppedError, WriteError}; #[cfg(test)] mod tests; #[derive(Debug)] -enum ConnectionEvent { +pub(crate) enum ConnectionEvent { Close { error_code: VarInt, reason: bytes::Bytes, }, - Proto(proto::ConnectionEvent), + Proto(moq_quic::ConnectionEvent), Rebind(Pin>), } -fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { - udp::Transmit { +fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> moq_sock::udp::Transmit<'a> { + moq_sock::udp::Transmit { destination: t.destination, ecn: t.ecn.map(udp_ecn), contents: buffer, @@ -111,11 +103,11 @@ fn udp_transmit<'a>(t: &Transmit, buffer: &'a [u8]) -> udp::Transmit<'a> { } } -fn udp_ecn(ecn: EcnCodepoint) -> udp::EcnCodepoint { +fn udp_ecn(ecn: EcnCodepoint) -> moq_sock::udp::EcnCodepoint { match ecn { - EcnCodepoint::Ect0 => udp::EcnCodepoint::Ect0, - EcnCodepoint::Ect1 => udp::EcnCodepoint::Ect1, - EcnCodepoint::Ce => udp::EcnCodepoint::Ce, + EcnCodepoint::Ect0 => moq_sock::udp::EcnCodepoint::Ect0, + EcnCodepoint::Ect1 => moq_sock::udp::EcnCodepoint::Ect1, + EcnCodepoint::Ce => moq_sock::udp::EcnCodepoint::Ce, } } diff --git a/rs/moq-tokio/src/quic/driver/mutex.rs b/rs/moq-tokio/src/quic/driver/mutex.rs index 91962cb072..d88d40e32a 100644 --- a/rs/moq-tokio/src/quic/driver/mutex.rs +++ b/rs/moq-tokio/src/quic/driver/mutex.rs @@ -3,116 +3,6 @@ use std::{ ops::{Deref, DerefMut}, }; -#[cfg(feature = "lock_tracking")] -mod tracking { - use super::*; - use crate::{Duration, Instant}; - use std::collections::VecDeque; - use tracing::warn; - - #[derive(Debug)] - struct Inner { - last_lock_owner: VecDeque<(&'static str, Duration)>, - value: T, - } - - /// A Mutex which optionally allows to track the time a lock was held and - /// emit warnings in case of excessive lock times - pub(crate) struct Mutex { - inner: std::sync::Mutex>, - } - - impl Debug for Mutex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Debug::fmt(&self.inner, f) - } - } - - impl Mutex { - pub(crate) fn new(value: T) -> Self { - Self { - inner: std::sync::Mutex::new(Inner { - last_lock_owner: VecDeque::new(), - value, - }), - } - } - - /// Acquires the lock for a certain purpose - /// - /// The purpose will be recorded in the list of last lock owners - pub(crate) fn lock(&self, purpose: &'static str) -> MutexGuard<'_, T> { - // We don't bother dispatching through Runtime::now because they're pure performance - // diagnostics. - let now = Instant::now(); - let guard = self.inner.lock().unwrap(); - - let lock_time = Instant::now(); - let elapsed = lock_time.duration_since(now); - - if elapsed > Duration::from_millis(1) { - warn!( - "Locking the connection for {} took {:?}. Last owners: {:?}", - purpose, elapsed, guard.last_lock_owner - ); - } - - MutexGuard { - guard, - start_time: lock_time, - purpose, - } - } - } - - pub(crate) struct MutexGuard<'a, T> { - guard: std::sync::MutexGuard<'a, Inner>, - start_time: Instant, - purpose: &'static str, - } - - impl Drop for MutexGuard<'_, T> { - fn drop(&mut self) { - if self.guard.last_lock_owner.len() == MAX_LOCK_OWNERS { - self.guard.last_lock_owner.pop_back(); - } - - let duration = self.start_time.elapsed(); - - if duration > Duration::from_millis(1) { - warn!( - "Utilizing the connection for {} took {:?}", - self.purpose, duration - ); - } - - self.guard - .last_lock_owner - .push_front((self.purpose, duration)); - } - } - - impl Deref for MutexGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.guard.value - } - } - - impl DerefMut for MutexGuard<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.guard.value - } - } - - const MAX_LOCK_OWNERS: usize = 20; -} - -#[cfg(feature = "lock_tracking")] -pub(crate) use tracking::Mutex; - -#[cfg(not(feature = "lock_tracking"))] mod non_tracking { use super::*; @@ -159,5 +49,4 @@ mod non_tracking { } } -#[cfg(not(feature = "lock_tracking"))] pub(crate) use non_tracking::Mutex; diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index d3548a1a33..027349ae12 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -6,11 +6,11 @@ use std::{ }; use bytes::Bytes; -use proto::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; +use moq_quic::{Chunk, Chunks, ClosedStream, ConnectionError, ReadableError, StreamId}; use thiserror::Error; use tokio::io::ReadBuf; -use crate::{VarInt, connection::ConnectionRef}; +use crate::quic::driver::{VarInt, connection::ConnectionRef}; /// A stream that can only be used to receive data /// @@ -42,12 +42,12 @@ use crate::{VarInt, connection::ConnectionRef}; /// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver /// will be bidirectional stream 0. /// -/// [`ReadError`]: crate::ReadError +/// [`ReadError`]: crate::quic::driver::ReadError /// [`stop()`]: RecvStream::stop -/// [`SendStream::finish`]: crate::SendStream::finish -/// [`WriteError::Stopped`]: crate::WriteError::Stopped +/// [`SendStream::finish`]: crate::quic::driver::SendStream::finish +/// [`WriteError::Stopped`]: crate::quic::driver::WriteError::Stopped /// [`id`]: RecvStream::id -/// [`Connection::accept_bi`]: crate::Connection::accept_bi +/// [`Connection::accept_bi`]: crate::quic::driver::Connection::accept_bi #[derive(Debug)] pub struct RecvStream { conn: ConnectionRef, @@ -104,7 +104,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read( &mut self, cx: &mut Context<'_>, @@ -126,7 +126,7 @@ impl RecvStream { /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is /// closed. /// - /// [`finish`]: crate::SendStream::finish + /// [`finish`]: crate::quic::driver::SendStream::finish pub fn poll_read_buf( &mut self, cx: &mut Context<'_>, @@ -256,7 +256,7 @@ impl RecvStream { /// /// This operation is *not* cancel-safe. /// - /// [`ReadToEndError::TooLong`]: crate::ReadToEndError::TooLong + /// [`ReadToEndError::TooLong`]: crate::quic::driver::ReadToEndError::TooLong pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { ReadToEnd { stream: self, @@ -286,13 +286,13 @@ impl RecvStream { /// Check if this stream predates completion of the handshake on an incoming connection. /// /// True only if the stream was accepted before the handshake completed, which is only possible - /// if you successfully called [`Connecting::into_0rtt`](crate::Connecting::into_0rtt) and the + /// if you successfully called [`Connecting::into_0rtt`](crate::quic::driver::Connecting::into_0rtt) and the /// client chose to send 0-RTT data. /// /// Under those conditions, depending on cryptographic layer configuration, 0-RTT application /// data may be a replay attack. To guard against this, applications should not execute /// non-idempotent operations until - /// [`Connection::authenticated`](crate::Connection::authenticated) succeeds. + /// [`Connection::authenticated`](crate::quic::driver::Connection::authenticated) succeeds. pub fn is_0rtt(&self) -> bool { self.is_0rtt } @@ -306,7 +306,7 @@ impl RecvStream { /// /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was - /// [`finish()`](crate::SendStream::finish)ed by the peer and all data has been received, after + /// [`finish()`](crate::quic::driver::SendStream::finish)ed by the peer and all data has been received, after /// which it is no longer meaningful for the stream to be reset. /// /// This operation is cancel-safe. @@ -360,7 +360,7 @@ impl RecvStream { where T: FnMut(&mut Chunks<'_>) -> ReadStatus, { - use proto::ReadError::*; + use moq_quic::ReadError::*; use std::collections::hash_map::Entry; if self.all_data_read { @@ -432,11 +432,11 @@ impl RecvStream { enum ReadStatus { Readable(T), Finished(Option), - Failed(Option, proto::ReadError), + Failed(Option, moq_quic::ReadError), } -impl From<(Option, Option)> for ReadStatus { - fn from(status: (Option, Option)) -> Self { +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { match status { (read, None) => Self::Finished(read), (read, Some(e)) => Self::Failed(read, e), @@ -446,7 +446,7 @@ impl From<(Option, Option)> for ReadStatus { /// Future produced by [`RecvStream::read_to_end()`]. /// -/// [`RecvStream::read_to_end()`]: crate::RecvStream::read_to_end +/// [`RecvStream::read_to_end()`]: crate::quic::driver::RecvStream::read_to_end struct ReadToEnd<'a> { stream: &'a mut RecvStream, buffer: ReadToEndBuffer, @@ -515,19 +515,6 @@ pub enum ReadToEndError { TooLong, } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncRead for RecvStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut [u8], - ) -> Poll> { - let mut buf = ReadBuf::new(buf); - ready!(Self::poll_read_buf(self.get_mut(), cx, &mut buf))?; - Poll::Ready(Ok(buf.filled().len())) - } -} - impl tokio::io::AsyncRead for RecvStream { fn poll_read( self: Pin<&mut Self>, @@ -594,7 +581,7 @@ pub enum ReadError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -640,7 +627,7 @@ pub enum ResetError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -658,7 +645,7 @@ impl From for io::Error { /// Future produced by [`RecvStream::read()`]. /// -/// [`RecvStream::read()`]: crate::RecvStream::read +/// [`RecvStream::read()`]: crate::quic::driver::RecvStream::read struct Read<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -679,7 +666,7 @@ impl Future for Read<'_> { /// Future produced by [`RecvStream::read_exact()`]. /// -/// [`RecvStream::read_exact()`]: crate::RecvStream::read_exact +/// [`RecvStream::read_exact()`]: crate::quic::driver::RecvStream::read_exact struct ReadExact<'a> { stream: &'a mut RecvStream, buf: ReadBuf<'a>, @@ -715,7 +702,7 @@ pub enum ReadExactError { /// Future produced by [`RecvStream::read_chunk()`]. /// -/// [`RecvStream::read_chunk()`]: crate::RecvStream::read_chunk +/// [`RecvStream::read_chunk()`]: crate::quic::driver::RecvStream::read_chunk struct ReadChunk<'a> { stream: &'a mut RecvStream, max_length: usize, @@ -732,7 +719,7 @@ impl Future for ReadChunk<'_> { /// Future produced by [`RecvStream::read_chunks()`]. /// -/// [`RecvStream::read_chunks()`]: crate::RecvStream::read_chunks +/// [`RecvStream::read_chunks()`]: crate::quic::driver::RecvStream::read_chunks struct ReadChunks<'a> { stream: &'a mut RecvStream, bufs: &'a mut [Bytes], diff --git a/rs/moq-tokio/src/quic/driver/runtime/mod.rs b/rs/moq-tokio/src/quic/driver/runtime/mod.rs index 9898a0e879..cc89f2f90e 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/mod.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/mod.rs @@ -1,5 +1,3 @@ -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -use std::sync::Arc; use std::{ fmt::{self, Debug}, future::Future, @@ -10,35 +8,9 @@ use std::{ time::Duration, }; -use udp::{RecvMeta, Transmit}; - -use crate::Instant; - -/// Abstracts I/O and timer operations for runtime independence -pub trait Runtime: Send + Sync + Debug + 'static { - /// Construct a timer that will expire at `i` - fn new_timer(&self, i: Instant) -> Pin>; - /// Drive `future` to completion in the background - #[track_caller] - fn spawn(&self, future: Pin + Send>>); - /// Convert `t` into the socket type used by this runtime - #[cfg(not(wasm_browser))] - fn wrap_udp_socket(&self, t: std::net::UdpSocket) -> io::Result>; - /// Look up the current time - /// - /// Allows simulating the flow of time for testing. - fn now(&self) -> Instant { - Instant::now() - } -} +use moq_sock::udp::{RecvMeta, Transmit}; -/// Abstract implementation of an async timer for runtime independence -pub trait AsyncTimer: Send + Debug + 'static { - /// Update the timer to expire at `i` - fn reset(self: Pin<&mut Self>, i: Instant); - /// Check whether the timer has expired, and register to be woken if not - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; -} +use crate::quic::driver::Instant; /// Abstract implementation of a UDP socket for runtime independence pub trait AsyncUdpSocket: Send + Sync + Debug + 'static { @@ -143,7 +115,6 @@ impl /// that resolves once the socket is write-ready. /// /// See also the bounds on this struct's [`UdpSender`] implementation. - #[cfg(any(feature = "runtime-smol", feature = "runtime-tokio",))] fn new(inner: Socket, make_fut: MakeWritableFutFn) -> Self { Self { socket: inner, @@ -215,7 +186,7 @@ fn log_sendmsg_error( ) { #[cfg(unix)] // Unix `EMSGSIZE` is expected for MTU probes. - if udp::is_msg_size_err(error) { + if moq_sock::udp::is_msg_size_err(error) { return; } @@ -253,41 +224,10 @@ trait UdpSenderHelperSocket: Send + Sync + 'static { fn max_transmit_segments(&self) -> usize; } -/// Automatically select an appropriate runtime from those enabled at compile time -/// -/// If `runtime-tokio` is enabled and this function is called from within a Tokio runtime context, -/// then `TokioRuntime` is returned. Otherwise, if `runtime-smol` is enabled, `SmolRuntime` is -/// returned. Otherwise, `None` is returned. -#[cfg(any(feature = "runtime-tokio", feature = "runtime-smol"))] -#[allow(clippy::needless_return)] // Be sure we return the right thing -pub fn default_runtime() -> Option> { - #[cfg(feature = "runtime-tokio")] - { - if ::tokio::runtime::Handle::try_current().is_ok() { - return Some(Arc::new(TokioRuntime)); - } - } - - #[cfg(feature = "runtime-smol")] - { - return Some(Arc::new(SmolRuntime)); - } - - #[cfg(not(feature = "runtime-smol"))] - None -} - -#[cfg(feature = "runtime-tokio")] mod tokio; -#[cfg(feature = "runtime-tokio")] -pub use tokio::TokioRuntime; - -#[cfg(feature = "runtime-smol")] -mod smol; -#[cfg(feature = "runtime-smol")] -pub use smol::*; +pub(crate) use tokio::wrap_udp_socket; -#[cfg(all(test, any(feature = "runtime-tokio", feature = "runtime-smol")))] +#[cfg(test)] mod tests { use std::{future, task::Waker}; diff --git a/rs/moq-tokio/src/quic/driver/runtime/smol.rs b/rs/moq-tokio/src/quic/driver/runtime/smol.rs deleted file mode 100644 index b5ae3d0cf2..0000000000 --- a/rs/moq-tokio/src/quic/driver/runtime/smol.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::{ - future::Future, - pin::Pin, - task::{Context, Poll}, - time::Instant, -}; -use std::{io, sync::Arc, task::ready}; - -use async_io::Async; -use async_io::Timer; - -use super::AsyncTimer; -use super::{AsyncUdpSocket, Runtime, UdpSender, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for smol -#[derive(Debug)] -pub struct SmolRuntime; - -impl Runtime for SmolRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(Timer::at(t)) - } - - fn spawn(&self, future: Pin + Send>>) { - ::smol::spawn(future).detach(); - } - - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket::new(sock)?)) - } -} - -impl AsyncTimer for Timer { - fn reset(mut self: Pin<&mut Self>, t: Instant) { - self.set_at(t) - } - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx).map(|_| ()) - } -} - -#[derive(Debug, Clone)] -struct UdpSocket { - io: Arc>, - inner: Arc, -} - -impl UdpSocket { - fn new(sock: std::net::UdpSocket) -> io::Result { - Ok(Self { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(Async::new_nonblocking(sock)?), - }) - } -} - -impl UdpSenderHelperSocket for UdpSocket { - fn max_transmit_segments(&self) -> usize { - self.inner.max_gso_segments() - } - - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { - self.inner.try_send((&self.io).into(), transmit)?; - - Ok(()) - } -} - -impl AsyncUdpSocket for UdpSocket { - fn create_sender(&self) -> Pin> { - Box::pin(UdpSenderHelper::new(self.clone(), |socket: &Self| { - let socket = socket.clone(); - async move { socket.io.writable().await } - })) - } - - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], - ) -> Poll> { - loop { - ready!(self.io.poll_readable(cx))?; - match self.inner.recv((&self.io).into(), bufs, meta) { - Ok(res) => return Poll::Ready(Ok(res)), - Err(e) if e.kind() == io::ErrorKind::WouldBlock => {} - Err(e) => return Poll::Ready(Err(e)), - } - } - } - - fn local_addr(&self) -> io::Result { - self.io.as_ref().as_ref().local_addr() - } - - fn may_fragment(&self) -> bool { - self.inner.may_fragment() - } - - fn max_receive_segments(&self) -> usize { - self.inner.gro_segments() - } -} diff --git a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs index ac5dfd6f20..750bff8ac6 100644 --- a/rs/moq-tokio/src/quic/driver/runtime/tokio.rs +++ b/rs/moq-tokio/src/quic/driver/runtime/tokio.rs @@ -1,57 +1,26 @@ use std::{ - future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll, ready}, - time::Instant, }; -use tokio::{ - io::Interest, - time::{Sleep, sleep_until}, -}; - -use super::{AsyncTimer, AsyncUdpSocket, Runtime, UdpSenderHelper, UdpSenderHelperSocket}; - -/// A Quinn runtime for Tokio -#[derive(Debug)] -pub struct TokioRuntime; - -impl Runtime for TokioRuntime { - fn new_timer(&self, t: Instant) -> Pin> { - Box::pin(sleep_until(t.into())) - } - - fn spawn(&self, future: Pin + Send>>) { - tokio::spawn(future); - } +use tokio::io::Interest; - fn wrap_udp_socket(&self, sock: std::net::UdpSocket) -> io::Result> { - Ok(Box::new(UdpSocket { - inner: Arc::new(udp::UdpSocketState::new((&sock).into())?), - io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), - })) - } +use super::{AsyncUdpSocket, UdpSenderHelper, UdpSenderHelperSocket}; - fn now(&self) -> Instant { - tokio::time::Instant::now().into_std() - } -} - -impl AsyncTimer for Sleep { - fn reset(self: Pin<&mut Self>, t: Instant) { - Self::reset(self, t.into()) - } - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { - Future::poll(self, cx) - } +/// Register `sock` with the current tokio runtime. +pub(crate) fn wrap_udp_socket(sock: std::net::UdpSocket) -> io::Result> { + Ok(Box::new(UdpSocket { + inner: Arc::new(moq_sock::udp::UdpSocketState::new((&sock).into())?), + io: Arc::new(tokio::net::UdpSocket::from_std(sock)?), + })) } #[derive(Debug, Clone)] struct UdpSocket { io: Arc, - inner: Arc, + inner: Arc, } impl UdpSenderHelperSocket for UdpSocket { @@ -59,7 +28,7 @@ impl UdpSenderHelperSocket for UdpSocket { self.inner.max_gso_segments() } - fn try_send(&self, transmit: &udp::Transmit<'_>) -> io::Result<()> { + fn try_send(&self, transmit: &moq_sock::udp::Transmit<'_>) -> io::Result<()> { self.io.try_io(Interest::WRITABLE, || { self.inner.try_send((&self.io).into(), transmit) })?; @@ -80,7 +49,7 @@ impl AsyncUdpSocket for UdpSocket { &mut self, cx: &mut Context<'_>, bufs: &mut [io::IoSliceMut<'_>], - meta: &mut [udp::RecvMeta], + meta: &mut [moq_sock::udp::RecvMeta], ) -> Poll> { loop { ready!(self.io.poll_recv_ready(cx))?; diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index 570db9ac81..e51af7cf25 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -6,10 +6,10 @@ use std::{ }; use bytes::Bytes; -use proto::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; +use moq_quic::{ClosedStream, ConnectionError, FinishError, StreamId, Written}; use thiserror::Error; -use crate::{ +use crate::quic::driver::{ VarInt, connection::{ConnectionRef, State}, }; @@ -140,9 +140,9 @@ impl SendStream { write_fn: F, ) -> Poll> where - F: FnOnce(&mut proto::SendStream<'_>) -> Result, + F: FnOnce(&mut moq_quic::SendStream<'_>) -> Result, { - use proto::WriteError::*; + use moq_quic::WriteError::*; let mut conn = self.conn.state.lock("SendStream::poll_write"); if self.is_0rtt { conn.check_0rtt() @@ -315,25 +315,6 @@ fn send_stream_stopped( } } -#[cfg(feature = "futures-io")] -impl futures_io::AsyncWrite for SendStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.poll_write(cx, buf).map_err(Into::into) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().finish().map_err(Into::into)) - } -} - impl tokio::io::AsyncWrite for SendStream { fn poll_write( self: Pin<&mut Self>, @@ -394,7 +375,7 @@ pub enum WriteError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } @@ -437,7 +418,7 @@ pub enum StoppedError { /// Can only occur on clients for 0-RTT streams, which can be opened using /// [`Connecting::into_0rtt()`]. /// - /// [`Connecting::into_0rtt()`]: crate::Connecting::into_0rtt() + /// [`Connecting::into_0rtt()`]: crate::quic::driver::Connecting::into_0rtt() #[error("0-RTT rejected")] ZeroRttRejected, } diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index e0c6f3ef9e..0b284f1770 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1,8 +1,8 @@ -#![cfg(any(feature = "rustls-aws-lc-rs", feature = "rustls-ring"))] +#![cfg(any(feature = "aws-lc-rs", feature = "ring"))] -#[cfg(all(feature = "rustls-aws-lc-rs", not(feature = "rustls-ring")))] +#[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] use rustls::crypto::aws_lc_rs::default_provider; -#[cfg(feature = "rustls-ring")] +#[cfg(feature = "ring")] use rustls::crypto::ring::default_provider; use std::{ @@ -19,15 +19,16 @@ use std::{ task::{Context, Poll, RawWaker, RawWakerVTable, Waker}, }; -use crate::runtime::{AsyncTimer, AsyncUdpSocket, Runtime as _, TokioRuntime, UdpSender}; -use crate::{Duration, Instant}; +use crate::quic::driver::runtime::{AsyncUdpSocket, UdpSender, wrap_udp_socket}; +use crate::quic::driver::{Duration, Instant}; use bytes::Bytes; -use proto::{ +use moq_quic::{ ConnectionId, RandomConnectionIdGenerator, crypto::rustls::{QuicClientConfig, QuicServerConfig}, crypto::{Keys, ServerConfig as ProtoServerConfig, Session, UnsupportedVersion}, transport_parameters::TransportParameters, }; +use moq_sock::udp::RecvMeta; use rand::{Rng, SeedableRng, rngs::StdRng}; use rustls::{ RootCertStore, @@ -41,7 +42,6 @@ use tokio::{ use tracing::instrument::Instrument as _; use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; -use udp::RecvMeta; use super::{ClientConfig, Endpoint, EndpointConfig, RecvStream, SendStream, TransportConfig}; @@ -64,6 +64,7 @@ fn handshake_timeout() { let mut transport_config = TransportConfig::default(); transport_config .max_idle_timeout(Some(IDLE_TIMEOUT.try_into().unwrap())) + .handshake_idle_timeout(IDLE_TIMEOUT) .initial_rtt(Duration::from_millis(10)); client_config.transport_config(Arc::new(transport_config)); @@ -78,7 +79,7 @@ fn handshake_timeout() { .unwrap() .await { - Err(crate::ConnectionError::TimedOut) => {} + Err(crate::quic::driver::ConnectionError::TimedOut) => {} Err(e) => panic!("unexpected error: {e:?}"), Ok(_) => panic!("unexpected success"), } @@ -119,7 +120,7 @@ async fn close_endpoint() { .unwrap(); endpoint.close(0u32.into(), &[]); match conn.await { - Err(crate::ConnectionError::LocallyClosed) => (), + Err(crate::quic::driver::ConnectionError::LocallyClosed) => (), Err(e) => panic!("unexpected error: {e}"), Ok(_) => { panic!("unexpected success"); @@ -134,7 +135,7 @@ fn local_addr() { let runtime = rt_basic(); let ep = { let _guard = runtime.enter(); - Endpoint::new(Default::default(), None, socket, Arc::new(TokioRuntime)).unwrap() + Endpoint::new(Default::default(), None, socket).unwrap() }; assert_eq!( addr, @@ -254,7 +255,7 @@ async fn ip_blocking() { .await .expect_err("server should have blocked this"); assert!( - matches!(e, crate::ConnectionError::ConnectionClosed(_)), + matches!(e, crate::quic::driver::ConnectionError::ConnectionClosed(_)), "wrong error" ); }, @@ -299,8 +300,11 @@ impl EndpointFactory { fn endpoint_with_config(&self, transport_config: TransportConfig) -> Endpoint { let key = PrivateKeyDer::Pkcs8(self.cert.signing_key.serialize_der().into()); let transport_config = Arc::new(transport_config); - let mut server_config = - crate::ServerConfig::with_single_cert(vec![self.cert.cert.der().clone()], key).unwrap(); + let mut server_config = crate::quic::driver::ServerConfig::with_single_cert( + vec![self.cert.cert.der().clone()], + key, + ) + .unwrap(); server_config.transport_config(transport_config.clone()); let mut roots = RootCertStore::empty(); @@ -309,7 +313,6 @@ impl EndpointFactory { self.endpoint_config.clone(), Some(server_config), UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - Arc::new(TokioRuntime), ) .unwrap(); let mut client_config = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); @@ -448,43 +451,8 @@ impl AsyncUdpSocket for FailingSocket { } } -/// Poll a connection driver before its registration returns, once the endpoint has failed. -#[derive(Debug)] -struct DriverLossRuntime { - failure: Arc, - stopped_on_first_poll: AtomicBool, -} - -impl crate::runtime::Runtime for DriverLossRuntime { - fn new_timer(&self, deadline: Instant) -> Pin> { - TokioRuntime.new_timer(deadline) - } - - fn spawn(&self, mut future: Pin + Send>>) { - if self.failure.armed.load(Ordering::SeqCst) { - let mut cx = Context::from_waker(Waker::noop()); - if future.as_mut().poll(&mut cx).is_ready() { - self.stopped_on_first_poll.store(true, Ordering::SeqCst); - return; - } - } - TokioRuntime.spawn(future); - } - - fn wrap_udp_socket(&self, socket: UdpSocket) -> io::Result> { - TokioRuntime.wrap_udp_socket(socket) - } - - fn now(&self) -> Instant { - TokioRuntime.now() - } -} - fn localhost_socket() -> Box { - TokioRuntime - .wrap_udp_socket( - UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(), - ) + wrap_udp_socket(UdpSocket::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap()) .unwrap() } @@ -492,7 +460,6 @@ fn blocking_server_pair( server_alpn: &[u8], client_alpn: &[u8], server_socket: Box, - server_runtime: Arc, ) -> (Endpoint, Endpoint, Arc) { let factory = EndpointFactory::new(); let key = PrivateKeyDer::Pkcs8(factory.cert.signing_key.serialize_der().into()); @@ -503,7 +470,7 @@ fn blocking_server_pair( .with_single_cert(vec![factory.cert.cert.der().clone()], key) .unwrap(); server_crypto.alpn_protocols = vec![server_alpn.to_vec()]; - let mut server_config = crate::ServerConfig::with_crypto(Arc::new( + let mut server_config = crate::quic::driver::ServerConfig::with_crypto(Arc::new( QuicServerConfig::try_from(server_crypto).unwrap(), )); let blocker = Arc::new(HandshakeBlocker::default()); @@ -516,14 +483,20 @@ fn blocking_server_pair( factory.endpoint_config.clone(), Some(server_config), server_socket, - server_runtime, ) .unwrap(); let mut roots = RootCertStore::empty(); roots.add(factory.cert.cert.der().clone()).unwrap(); let client = Endpoint::client(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)).unwrap(); - let mut client_crypto = rustls::ClientConfig::builder_with_provider(default_provider().into()) + // One classical key share keeps the ClientHello in the first Initial, so a handshake failure + // surfaces from `accept` itself. aws-lc-rs's default post-quantum share spills it into a + // second packet, which only reaches the connection after it is registered. + let mut provider = default_provider(); + provider + .kx_groups + .retain(|group| group.name() == rustls::NamedGroup::X25519); + let mut client_crypto = rustls::ClientConfig::builder_with_provider(provider.into()) .with_protocol_versions(&[&rustls::version::TLS13]) .unwrap() .with_root_certificates(roots) @@ -544,7 +517,7 @@ async fn wait_for_blocked_handshake(blocker: Arc) { async fn accept_with_blocked_start_session( server: Endpoint, -) -> Result { +) -> Result { let incoming = server.accept().await.unwrap(); // The test crypto provider intentionally blocks in `start_session`, so run // accept on the blocking pool rather than parking a Tokio worker thread. @@ -559,8 +532,12 @@ struct BlockedHandshake { server: Endpoint, blocker: Arc, release_guard: HandshakeReleaseGuard, - accept: tokio::task::JoinHandle>, - connect: tokio::task::JoinHandle>, + accept: tokio::task::JoinHandle< + Result, + >, + connect: tokio::task::JoinHandle< + Result, + >, } fn blocked_handshake(server_alpn: &[u8], client_alpn: &[u8]) -> BlockedHandshake { @@ -572,22 +549,7 @@ fn blocked_handshake_with_socket( client_alpn: &[u8], server_socket: Box, ) -> BlockedHandshake { - blocked_handshake_with_runtime( - server_alpn, - client_alpn, - server_socket, - Arc::new(TokioRuntime), - ) -} - -fn blocked_handshake_with_runtime( - server_alpn: &[u8], - client_alpn: &[u8], - server_socket: Box, - server_runtime: Arc, -) -> BlockedHandshake { - let (client, server, blocker) = - blocking_server_pair(server_alpn, client_alpn, server_socket, server_runtime); + let (client, server, blocker) = blocking_server_pair(server_alpn, client_alpn, server_socket); let server_addr = server.local_addr().unwrap(); let accept = tokio::spawn({ @@ -610,8 +572,10 @@ fn blocked_handshake_with_runtime( } async fn await_connection( - task: tokio::task::JoinHandle>, -) -> Result { + task: tokio::task::JoinHandle< + Result, + >, +) -> Result { timeout(Duration::from_secs(5), task) .await .unwrap() @@ -713,7 +677,9 @@ fn wait_idle_returns_when_the_driver_is_dropped() { let _ = waiter.join(); } -async fn wait_closed(conn: &crate::Connection) -> crate::ConnectionError { +async fn wait_closed( + conn: &crate::quic::driver::Connection, +) -> crate::quic::driver::ConnectionError { timeout(Duration::from_secs(5), conn.closed()) .await .unwrap() @@ -914,7 +880,8 @@ fn run_echo(args: EchoArgs) { let key = PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()); let cert = CertificateDer::from(cert.cert); let mut server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); server_config.transport = transport_config.clone(); let server_sock = UdpSocket::bind(args.server_addr).unwrap(); @@ -922,13 +889,7 @@ fn run_echo(args: EchoArgs) { let server = { let _guard = runtime.enter(); let _guard = error_span!("server").entered(); - Endpoint::new( - Default::default(), - Some(server_config), - server_sock, - Arc::new(TokioRuntime), - ) - .unwrap() + Endpoint::new(Default::default(), Some(server_config), server_sock).unwrap() }; let mut roots = RootCertStore::empty(); @@ -1144,7 +1105,8 @@ async fn rebind_recv() { client.set_default_client_config(client_config); let server_config = - crate::ServerConfig::with_single_cert(vec![cert.clone()], key.into()).unwrap(); + crate::quic::driver::ServerConfig::with_single_cert(vec![cert.clone()], key.into()) + .unwrap(); let server = { let _guard = tracing::error_span!("server").entered(); Endpoint::server( @@ -1254,8 +1216,8 @@ async fn split_accept_failure_wakes_idle_waiter() { let error = await_connection(accept).await.unwrap_err(); assert!(matches!( error, - crate::ConnectionError::TransportError(error) - if error.code == proto::TransportErrorCode::crypto(0x78) + crate::quic::driver::ConnectionError::TransportError(error) + if error.code == moq_quic::TransportErrorCode::crypto(0x78) )); // Repolling wait_idle alone would hide a missing notification now that the endpoint is // idle. Verify that the waiter registered before the failure was actually woken. @@ -1291,18 +1253,30 @@ async fn split_accept_close_during_pending_accept() { match await_connection(accept).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::LocallyClosed)); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )); } - Err(err) => assert!(matches!(err, crate::ConnectionError::LocallyClosed)), + Err(err) => assert!(matches!( + err, + crate::quic::driver::ConnectionError::LocallyClosed + )), } match await_connection(connect).await { Ok(conn) => { let err = wait_closed(&conn).await; - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } Err(err) => { - assert!(matches!(err, crate::ConnectionError::ConnectionClosed(_))); + assert!(matches!( + err, + crate::quic::driver::ConnectionError::ConnectionClosed(_) + )); } } @@ -1364,51 +1338,6 @@ async fn split_accept_driver_lost_during_handshake() { wait_idle(&client).await; } -#[tokio::test] -async fn split_accept_driver_lost_before_connection_spawn() { - let _guard = subscribe(); - let failure = Arc::new(SocketFailure::default()); - let runtime = Arc::new(DriverLossRuntime { - failure: failure.clone(), - stopped_on_first_poll: AtomicBool::new(false), - }); - let socket = Box::new(FailingSocket { - inner: localhost_socket(), - failure: failure.clone(), - }); - let BlockedHandshake { - client, - server, - blocker, - release_guard: _release_guard, - accept, - connect, - } = blocked_handshake_with_runtime(b"test", b"test", socket, runtime.clone()); - - wait_for_blocked_handshake(blocker.clone()).await; - failure.trigger(); - assert!( - timeout(Duration::from_secs(5), server.accept()) - .await - .unwrap() - .is_none() - ); - - blocker.release(); - // The runtime polls the connection driver before registration returns. It must observe the - // closed event channel immediately, before transmitting any handshake packets. - assert!(await_connection(accept).await.is_err()); - assert!( - runtime.stopped_on_first_poll.load(Ordering::SeqCst), - "connection driver ran before its event channel was closed" - ); - wait_idle(&server).await; - - client.close(0u32.into(), b"done"); - assert!(await_connection(connect).await.is_err()); - wait_idle(&client).await; -} - #[tokio::test] async fn remote_address_monitoring() { let _guard = subscribe(); @@ -1782,8 +1711,8 @@ async fn dropped_endpoint_cleans_up() { let _guard = subscribe(); let mut endpoint_factory = EndpointFactory::new(); - let cid_generator = Arc::new(|| -> Box { - Box::::default() + let cid_generator = Arc::new(|| -> Box { + Box::::default() }); endpoint_factory .endpoint_config diff --git a/rs/moq-tokio/src/quic/driver/work_limiter.rs b/rs/moq-tokio/src/quic/driver/work_limiter.rs index c3c3d3551d..eda6e92d6c 100644 --- a/rs/moq-tokio/src/quic/driver/work_limiter.rs +++ b/rs/moq-tokio/src/quic/driver/work_limiter.rs @@ -1,4 +1,4 @@ -use crate::{Duration, Instant}; +use crate::quic::driver::{Duration, Instant}; /// Limits the amount of time spent on a certain type of work in a cycle /// diff --git a/rs/moq-tokio/src/quic/web/client.rs b/rs/moq-tokio/src/quic/web/client.rs index c3dcee477b..e38cbd62e9 100644 --- a/rs/moq-tokio/src/quic/web/client.rs +++ b/rs/moq-tokio/src/quic/web/client.rs @@ -1,28 +1,28 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; -use crate::proto::ConnectRequest; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use noq::crypto::rustls::QuicClientConfig; +use crate::quic::driver::crypto::rustls::QuicClientConfig; +use crate::quic::web::proto::ConnectRequest; use rustls::{client::danger::ServerCertVerifier, pki_types::CertificateDer}; use tokio::net::lookup_host; use url::Host; -use crate::crypto; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::ALPN; -use crate::{ClientError, Session}; +use crate::quic::web::ALPN; +use crate::quic::web::crypto; +use crate::quic::web::{ClientError, Session}; /// Congestion control algorithm to use for the connection. /// /// Different algorithms make different tradeoffs between throughput and latency. pub enum CongestionControl { - /// Use the default congestion control algorithm (typically CUBIC). - Default, - /// Optimize for throughput (typically CUBIC). - Throughput, - /// Optimize for low latency (typically BBR). - LowLatency, + /// Use the default congestion control algorithm (typically CUBIC). + Default, + /// Optimize for throughput (typically CUBIC). + Throughput, + /// Optimize for low latency (typically BBR). + LowLatency, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -31,136 +31,124 @@ pub enum CongestionControl { /// This is optional; advanced users may use [Client::new] directly. #[derive(Clone)] pub struct ClientBuilder { - provider: crypto::Provider, - congestion_controller: - Option>, + provider: crypto::Provider, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ClientBuilder { - /// Create a Client builder, which can be used to establish multiple [Session]s. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - congestion_controller: None, - } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Accept any certificate from the server if it uses a known root CA. - pub fn with_system_roots(self) -> Result { - let mut roots = rustls::RootCertStore::empty(); - - let native = rustls_native_certs::load_native_certs(); - - // Log any errors that occurred while loading the native root certificates. - for err in native.errors { - tracing::warn!(?err, "failed to load root cert"); - } - - // Add the platform's native root certificates. - for cert in native.certs { - if let Err(err) = roots.add(cert) { - tracing::warn!(?err, "failed to add root cert"); - } - } - - let crypto = self - .builder() - .with_root_certificates(roots) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Supply certificates for accepted servers instead of using root CAs. - pub fn with_server_certificates( - self, - certs: Vec, - ) -> Result { - let hashes = certs.iter().map({ - let provider = self.provider.clone(); - move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() - }); - - self.with_server_certificate_hashes(hashes.collect()) - } - - /// Supply sha256 hashes for accepted certificates instead of using root CAs. - pub fn with_server_certificate_hashes( - self, - hashes: Vec>, - ) -> Result { - // Use a custom fingerprint verifier. - let fingerprints = Arc::new(ServerFingerprints { - provider: self.provider.clone(), - fingerprints: hashes, - }); - - // Configure the crypto client. - let crypto = self - .builder() - .dangerous() - .with_custom_certificate_verifier(fingerprints.clone()) - .with_no_client_auth(); - - self.build(crypto) - } - - /// Access dangerous configuration options. - /// - /// This method returns a builder that provides access to potentially insecure - /// TLS configurations. These options are opt-in and require explicit acknowledgment - /// through the builder pattern, making the security implications clear at the call site. - pub fn dangerous(self) -> DangerousClientBuilder { - DangerousClientBuilder { inner: self } - } - - fn builder(&self) -> rustls::ConfigBuilder { - rustls::ClientConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13]) - .unwrap() - } - - fn build(self, mut crypto: rustls::ClientConfig) -> Result { - crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; - - let client_config = QuicClientConfig::try_from(crypto).unwrap(); - let mut client_config = noq::ClientConfig::new(Arc::new(client_config)); - - let mut transport = noq::TransportConfig::default(); - if let Some(cc) = &self.congestion_controller { - transport.congestion_controller_factory(cc.clone()); - } - - client_config.transport_config(transport.into()); - - let client = noq::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); - Ok(Client { - endpoint: client, - config: client_config, - }) - } + /// Create a Client builder, which can be used to establish multiple [Session]s. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + congestion_controller: None, + } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Accept any certificate from the server if it uses a known root CA. + pub fn with_system_roots(self) -> Result { + let mut roots = rustls::RootCertStore::empty(); + + let native = rustls_native_certs::load_native_certs(); + + // Log any errors that occurred while loading the native root certificates. + for err in native.errors { + tracing::warn!(?err, "failed to load root cert"); + } + + // Add the platform's native root certificates. + for cert in native.certs { + if let Err(err) = roots.add(cert) { + tracing::warn!(?err, "failed to add root cert"); + } + } + + let crypto = self.builder().with_root_certificates(roots).with_no_client_auth(); + + self.build(crypto) + } + + /// Supply certificates for accepted servers instead of using root CAs. + pub fn with_server_certificates(self, certs: Vec) -> Result { + let hashes = certs.iter().map({ + let provider = self.provider.clone(); + move |cert| crypto::sha256(&provider, cert).as_ref().to_vec() + }); + + self.with_server_certificate_hashes(hashes.collect()) + } + + /// Supply sha256 hashes for accepted certificates instead of using root CAs. + pub fn with_server_certificate_hashes(self, hashes: Vec>) -> Result { + // Use a custom fingerprint verifier. + let fingerprints = Arc::new(ServerFingerprints { + provider: self.provider.clone(), + fingerprints: hashes, + }); + + // Configure the crypto client. + let crypto = self + .builder() + .dangerous() + .with_custom_certificate_verifier(fingerprints.clone()) + .with_no_client_auth(); + + self.build(crypto) + } + + /// Access dangerous configuration options. + /// + /// This method returns a builder that provides access to potentially insecure + /// TLS configurations. These options are opt-in and require explicit acknowledgment + /// through the builder pattern, making the security implications clear at the call site. + pub fn dangerous(self) -> DangerousClientBuilder { + DangerousClientBuilder { inner: self } + } + + fn builder(&self) -> rustls::ConfigBuilder { + rustls::ClientConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + } + + fn build(self, mut crypto: rustls::ClientConfig) -> Result { + crypto.alpn_protocols = vec![ALPN.as_bytes().to_vec()]; + + let client_config = QuicClientConfig::try_from(crypto).unwrap(); + let mut client_config = crate::quic::driver::ClientConfig::new(Arc::new(client_config)); + + let mut transport = crate::quic::driver::TransportConfig::default(); + if let Some(cc) = &self.congestion_controller { + transport.congestion_controller_factory(cc.clone()); + } + + client_config.transport_config(transport.into()); + + let client = crate::quic::driver::Endpoint::client("[::]:0".parse().unwrap()).unwrap(); + Ok(Client { + endpoint: client, + config: client_config, + }) + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ClientBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -170,209 +158,182 @@ impl Default for ClientBuilder { /// These options should only be used when you understand the security implications, /// such as in local development or over a secure VPN connection. pub struct DangerousClientBuilder { - inner: ClientBuilder, + inner: ClientBuilder, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl DangerousClientBuilder { - /// Disable certificate verification entirely. - /// - /// This makes the connection vulnerable to man-in-the-middle attacks. - /// Only use this in secure environments, such as in local development or over a VPN connection. - /// - /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the - /// security sense, hence the explicit `dangerous()` builder requirement. - pub fn with_no_certificate_verification(self) -> Result { - let noop = NoCertificateVerification(self.inner.provider.clone()); - - let crypto = self - .inner - .builder() - .dangerous() - .with_custom_certificate_verifier(Arc::new(noop)) - .with_no_client_auth(); - - self.inner.build(crypto) - } + /// Disable certificate verification entirely. + /// + /// This makes the connection vulnerable to man-in-the-middle attacks. + /// Only use this in secure environments, such as in local development or over a VPN connection. + /// + /// This method is safe in the Rust sense (no memory unsafety), but dangerous in the + /// security sense, hence the explicit `dangerous()` builder requirement. + pub fn with_no_certificate_verification(self) -> Result { + let noop = NoCertificateVerification(self.inner.provider.clone()); + + let crypto = self + .inner + .builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(noop)) + .with_no_client_auth(); + + self.inner.build(crypto) + } } /// A client for connecting to a WebTransport server. #[derive(Clone, Debug)] pub struct Client { - endpoint: noq::Endpoint, - config: noq::ClientConfig, + endpoint: crate::quic::driver::Endpoint, + config: crate::quic::driver::ClientConfig, } impl Client { - /// Manually create a client via a Noq endpoint and config. - /// - /// The ALPN MUST be set to [ALPN]. - pub fn new(endpoint: noq::Endpoint, config: noq::ClientConfig) -> Self { - Self { endpoint, config } - } - - /// Connect to the server. - pub async fn connect( - &self, - request: impl Into, - ) -> Result { - let request = request.into(); - - let port = request.url.port().unwrap_or(443); - - // TODO error on username:password in host - let (host, remote) = match request - .url - .host() - .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? - { - Host::Domain(domain) => { - let domain = domain.to_string(); - // Look up the DNS entry. - let mut remotes = match lookup_host((domain.clone(), port)).await { - Ok(remotes) => remotes, - Err(_) => return Err(ClientError::InvalidDnsName(domain)), - }; - - // Return the first entry. - let remote = match remotes.next() { - Some(remote) => remote, - None => return Err(ClientError::InvalidDnsName(domain)), - }; - - (domain, remote) - } - Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), - Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), - }; - - // Connect to the server using the addr we just resolved. - let conn = self - .endpoint - .connect_with(self.config.clone(), remote, &host)?; - let conn = conn.await?; - - // Connect with the connection we established. - Session::connect(conn, request).await - } + /// Manually create a client via a QUIC endpoint and config. + /// + /// The ALPN MUST be set to [ALPN]. + pub fn new(endpoint: crate::quic::driver::Endpoint, config: crate::quic::driver::ClientConfig) -> Self { + Self { endpoint, config } + } + + /// Connect to the server. + pub async fn connect(&self, request: impl Into) -> Result { + let request = request.into(); + + let port = request.url.port().unwrap_or(443); + + // TODO error on username:password in host + let (host, remote) = match request + .url + .host() + .ok_or_else(|| ClientError::InvalidDnsName("".to_string()))? + { + Host::Domain(domain) => { + let domain = domain.to_string(); + // Look up the DNS entry. + let mut remotes = match lookup_host((domain.clone(), port)).await { + Ok(remotes) => remotes, + Err(_) => return Err(ClientError::InvalidDnsName(domain)), + }; + + // Return the first entry. + let remote = match remotes.next() { + Some(remote) => remote, + None => return Err(ClientError::InvalidDnsName(domain)), + }; + + (domain, remote) + } + Host::Ipv4(ipv4) => (ipv4.to_string(), SocketAddr::new(IpAddr::V4(ipv4), port)), + Host::Ipv6(ipv6) => (ipv6.to_string(), SocketAddr::new(IpAddr::V6(ipv6), port)), + }; + + // Connect to the server using the addr we just resolved. + let conn = self.endpoint.connect_with(self.config.clone(), remote, &host)?; + let conn = conn.await?; + + // Connect with the connection we established. + Session::connect(conn, request).await + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for Client { - fn default() -> Self { - ClientBuilder::new().with_system_roots().unwrap() - } + fn default() -> Self { + ClientBuilder::new().with_system_roots().unwrap() + } } #[cfg_attr(not(any(feature = "aws-lc-rs", feature = "ring")), allow(dead_code))] #[derive(Debug)] struct ServerFingerprints { - provider: crypto::Provider, - fingerprints: Vec>, + provider: crypto::Provider, + fingerprints: Vec>, } impl ServerCertVerifier for ServerFingerprints { - fn verify_server_cert( - &self, - end_entity: &rustls::pki_types::CertificateDer<'_>, - _intermediates: &[rustls::pki_types::CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp_response: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - let cert_hash = crypto::sha256(&self.provider, end_entity); - if self - .fingerprints - .iter() - .any(|fingerprint| fingerprint == cert_hash.as_ref()) - { - return Ok(rustls::client::danger::ServerCertVerified::assertion()); - } - - Err(rustls::Error::InvalidCertificate( - rustls::CertificateError::UnknownIssuer, - )) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &rustls::pki_types::CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.provider.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.provider - .signature_verification_algorithms - .supported_schemes() - } + fn verify_server_cert( + &self, + end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + let cert_hash = crypto::sha256(&self.provider, end_entity); + if self + .fingerprints + .iter() + .any(|fingerprint| fingerprint == cert_hash.as_ref()) + { + return Ok(rustls::client::danger::ServerCertVerified::assertion()); + } + + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider.signature_verification_algorithms.supported_schemes() + } } #[derive(Debug)] pub struct NoCertificateVerification(Arc); impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer<'_>, - _intermediates: &[CertificateDer<'_>], - _server_name: &rustls::pki_types::ServerName<'_>, - _ocsp: &[u8], - _now: rustls::pki_types::UnixTime, - ) -> Result { - Ok(rustls::client::danger::ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls12_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn verify_tls13_signature( - &self, - message: &[u8], - cert: &CertificateDer<'_>, - dss: &rustls::DigitallySignedStruct, - ) -> Result { - rustls::crypto::verify_tls13_signature( - message, - cert, - dss, - &self.0.signature_verification_algorithms, - ) - } - - fn supported_verify_schemes(&self) -> Vec { - self.0.signature_verification_algorithms.supported_schemes() - } + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } } diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 4307355d3f..471f5e416c 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -6,153 +6,145 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum ConnectError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::ConnectError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::ConnectError), - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("http error status: {0}")] - ErrorStatus(http::StatusCode), + #[error("http error status: {0}")] + ErrorStatus(http::StatusCode), - #[error("server returned protocol not in request: {0}")] - ProtocolMismatch(String), + #[error("server returned protocol not in request: {0}")] + ProtocolMismatch(String), } /// An HTTP/3 CONNECT request/response for establishing a WebTransport session. pub struct Connecting { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connecting { - pub async fn accept(conn: &noq::Connection) -> Result { - // Accept the stream that will be used to send the HTTP CONNECT request. - // If they try to send any other type of HTTP request, we will error out. - let (send, mut recv) = conn.accept_bi().await?; - - let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; - tracing::debug!(?request, "received CONNECT request"); - - // The request was successfully decoded, so we can send a response. - Ok(Self { - request, - send, - recv, - }) - } - - // Called by the server to send a response to the client and establish the session. - pub async fn respond( - mut self, - response: impl Into, - ) -> Result { - let response = response.into(); - - // Validate that our protocol was in the client's request. - if let Some(protocol) = &response.protocol { - if !self.request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - tracing::debug!(?response, "sending CONNECT response"); - response.write(&mut self.send).await?; - - Ok(Connected { - request: self.request, - response, - send: self.send, - recv: self.recv, - }) - } - - pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { - let mut connect = self.respond(status).await?; - connect.send.finish().ok(); - Ok(()) - } + pub async fn accept(conn: &crate::quic::driver::Connection) -> Result { + // Accept the stream that will be used to send the HTTP CONNECT request. + // If they try to send any other type of HTTP request, we will error out. + let (send, mut recv) = conn.accept_bi().await?; + + let request = web_transport_proto::ConnectRequest::read(&mut recv).await?; + tracing::debug!(?request, "received CONNECT request"); + + // The request was successfully decoded, so we can send a response. + Ok(Self { request, send, recv }) + } + + // Called by the server to send a response to the client and establish the session. + pub async fn respond(mut self, response: impl Into) -> Result { + let response = response.into(); + + // Validate that our protocol was in the client's request. + if let Some(protocol) = &response.protocol + && !self.request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + tracing::debug!(?response, "sending CONNECT response"); + response.write(&mut self.send).await?; + + Ok(Connected { + request: self.request, + response, + send: self.send, + recv: self.recv, + }) + } + + pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { + let mut connect = self.respond(status).await?; + connect.send.finish().ok(); + Ok(()) + } } impl Deref for Connecting { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.request - } + fn deref(&self) -> &Self::Target { + &self.request + } } pub struct Connected { - // The request that was sent by the client. - pub request: ConnectRequest, + // The request that was sent by the client. + pub request: ConnectRequest, - // The response sent by the server. - pub response: ConnectResponse, + // The response sent by the server. + pub response: ConnectResponse, - // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: noq::SendStream, - pub(crate) recv: noq::RecvStream, + // A reference to the send/recv stream, so we don't close it until dropped. + pub(crate) send: crate::quic::driver::SendStream, + pub(crate) recv: crate::quic::driver::RecvStream, } impl Connected { - /// Open a new WebTransport session on the given connection for the given URL. - /// - /// You may add any number of subprotocols allowing the server to select from. - /// If the list is empty the field will be omitted in the request header. - pub async fn open( - conn: &noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Create a new stream that will be used to send the CONNECT frame. - let (mut send, mut recv) = conn.open_bi().await?; - - tracing::debug!(?request, "sending CONNECT request"); - request.write(&mut send).await?; - - let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; - tracing::debug!(?response, "received CONNECT response"); - - // Throw an error if we didn't get a 200 OK. - if response.status != http::StatusCode::OK { - return Err(ConnectError::ErrorStatus(response.status)); - } - - // Validate that the server's protocol was in our request. - if let Some(protocol) = &response.protocol { - if !request.protocols.contains(protocol) { - return Err(ConnectError::ProtocolMismatch(protocol.clone())); - } - } - - Ok(Self { - request, - response, - send, - recv, - }) - } - - // The session ID is the stream ID of the CONNECT request. - pub fn session_id(&self) -> VarInt { - // We gotta convert from the Noq VarInt to the (forked) WebTransport VarInt. - // We don't use the noq::VarInt because that would mean a noq dependency in - // web-transport-proto - let stream_id = noq::VarInt::from(self.send.id()); - VarInt::try_from(stream_id.into_inner()).unwrap() - } + /// Open a new WebTransport session on the given connection for the given URL. + /// + /// You may add any number of subprotocols allowing the server to select from. + /// If the list is empty the field will be omitted in the request header. + pub async fn open( + conn: &crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Create a new stream that will be used to send the CONNECT frame. + let (mut send, mut recv) = conn.open_bi().await?; + + tracing::debug!(?request, "sending CONNECT request"); + request.write(&mut send).await?; + + let response = web_transport_proto::ConnectResponse::read(&mut recv).await?; + tracing::debug!(?response, "received CONNECT response"); + + // Throw an error if we didn't get a 200 OK. + if response.status != http::StatusCode::OK { + return Err(ConnectError::ErrorStatus(response.status)); + } + + // Validate that the server's protocol was in our request. + if let Some(protocol) = &response.protocol + && !request.protocols.contains(protocol) + { + return Err(ConnectError::ProtocolMismatch(protocol.clone())); + } + + Ok(Self { + request, + response, + send, + recv, + }) + } + + // The session ID is the stream ID of the CONNECT request. + pub fn session_id(&self) -> VarInt { + // Convert from the QUIC VarInt to the WebTransport VarInt, since web-transport-proto + // doesn't depend on moq-quic. + let stream_id = crate::quic::driver::VarInt::from(self.send.id()); + VarInt::try_from(stream_id.into_inner()).unwrap() + } } diff --git a/rs/moq-tokio/src/quic/web/crypto.rs b/rs/moq-tokio/src/quic/web/crypto.rs index 1b8bb8ecf1..a1581975c9 100644 --- a/rs/moq-tokio/src/quic/web/crypto.rs +++ b/rs/moq-tokio/src/quic/web/crypto.rs @@ -5,8 +5,8 @@ use std::sync::Arc; -use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::crypto::CryptoProvider; +use rustls::crypto::hash::{self, HashAlgorithm}; use rustls::pki_types::CertificateDer; /// A shared reference to a crypto provider. @@ -22,25 +22,23 @@ pub type Provider = Arc; /// Panics if no provider is available. Either call `CryptoProvider::set_default()` /// or enable exactly one of the `ring` or `aws-lc-rs` features. pub fn default_provider() -> Provider { - // See - if let Some(provider) = CryptoProvider::get_default().cloned() { - return provider; - } + // See + if let Some(provider) = CryptoProvider::get_default().cloned() { + return provider; + } - #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] - { - return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); - } - #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] - { - return Arc::new(rustls::crypto::ring::default_provider()); - } - #[allow(unreachable_code)] - { - panic!( - "CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature." - ); - } + #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] + { + return Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + } + #[cfg(all(feature = "ring", not(feature = "aws-lc-rs")))] + { + return Arc::new(rustls::crypto::ring::default_provider()); + } + #[allow(unreachable_code)] + { + panic!("CryptoProvider::set_default() must be called; or only enable one ring/aws-lc-rs feature."); + } } /// Computes the SHA-256 hash of a certificate using the provided crypto provider. @@ -49,19 +47,19 @@ pub fn default_provider() -> Provider { /// /// Panics if the provider doesn't expose a SHA-256 hash algorithm. pub fn sha256(provider: &Provider, cert: &CertificateDer<'_>) -> hash::Output { - let hash_provider = provider.cipher_suites.iter().find_map(|suite| { - let hash_provider = suite.tls13()?.common.hash_provider; - if hash_provider.algorithm() == HashAlgorithm::SHA256 { - Some(hash_provider) - } else { - None - } - }); - if let Some(hash_provider) = hash_provider { - return hash_provider.hash(cert); - } + let hash_provider = provider.cipher_suites.iter().find_map(|suite| { + let hash_provider = suite.tls13()?.common.hash_provider; + if hash_provider.algorithm() == HashAlgorithm::SHA256 { + Some(hash_provider) + } else { + None + } + }); + if let Some(hash_provider) = hash_provider { + return hash_provider.hash(cert); + } - panic!( - "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." - ); + panic!( + "No SHA-256 backend available. Ensure your provider exposes SHA-256 or enable the 'ring'/'aws-lc-rs' feature." + ); } diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 654260fc4c..5697bf684e 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -2,70 +2,68 @@ use std::sync::{Arc, OnceLock}; use thiserror::Error; -use crate::{ConnectError, SettingsError}; +use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] pub enum ClientError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error: {0}")] - Connection(#[from] noq::ConnectionError), + #[error("connection error: {0}")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write: {0}")] - WriteError(#[from] noq::WriteError), + #[error("failed to write: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read: {0}")] - ReadError(#[from] noq::ReadError), + #[error("failed to read: {0}")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings: {0}")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings: {0}")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect: {0}")] - HttpError(#[from] ConnectError), + #[error("failed to exchange h3 connect: {0}")] + HttpError(#[from] ConnectError), - #[error("quic error: {0}")] - NoqError(#[from] noq::ConnectError), + #[error("quic error: {0}")] + ConnectError(#[from] crate::quic::driver::ConnectError), - #[error("invalid DNS name: {0}")] - InvalidDnsName(String), + #[error("invalid DNS name: {0}")] + InvalidDnsName(String), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } -/// An errors returned by [`crate::Session`], split based on if they are underlying QUIC errors or +/// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] pub enum SessionError { - #[error("connection error: {0}")] - ConnectionError(noq::ConnectionError), + #[error("connection error: {0}")] + ConnectionError(crate::quic::driver::ConnectionError), - #[error("webtransport error: {0}")] - WebTransportError(#[from] WebTransportError), + #[error("webtransport error: {0}")] + WebTransportError(#[from] WebTransportError), - #[error("send datagram error: {0}")] - SendDatagramError(#[from] noq::SendDatagramError), + #[error("send datagram error: {0}")] + SendDatagramError(#[from] crate::quic::driver::SendDatagramError), } -impl From for SessionError { - fn from(e: noq::ConnectionError) -> Self { - match &e { - noq::ConnectionError::ApplicationClosed(close) => { - match web_transport_proto::error_from_http3(close.error_code.into_inner()) { - Some(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - None => SessionError::ConnectionError(e), - } - } - _ => SessionError::ConnectionError(e), - } - } +impl From for SessionError { + fn from(e: crate::quic::driver::ConnectionError) -> Self { + match &e { + crate::quic::driver::ConnectionError::ApplicationClosed(close) => { + match web_transport_proto::error_from_http3(close.error_code.into_inner()) { + Some(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + None => SessionError::ConnectionError(e), + } + } + _ => SessionError::ConnectionError(e), + } + } } /// Why a session closed, shared by the session and its streams. The first close wins. @@ -74,192 +72,191 @@ impl From for SessionError { /// codes are encoded. #[derive(Debug)] pub(crate) struct CloseReason { - // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. - raw: bool, - reason: OnceLock, + // Raw QUIC carries the application code as is; HTTP/3 maps it into its own code space. + raw: bool, + reason: OnceLock, } impl CloseReason { - pub(crate) fn new(raw: bool) -> Self { - Self { - raw, - reason: OnceLock::new(), - } - } - - /// The QUIC code to reset or stop a stream with. - pub(crate) fn encode_stream_code(&self, code: u32) -> noq::VarInt { - if self.raw { - return code.into(); - } - noq::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() - } - - /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. - pub(crate) fn decode_stream_code(&self, code: noq::VarInt) -> Option { - let code = code.into_inner(); - // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that - // range, so the mapped form cannot be mistaken for a raw code. - match web_transport_proto::error_from_http3(code) { - Some(code) => Some(code), - None if self.raw => u32::try_from(code).ok(), - None => None, - } - } - - /// Record the close reason, returning false if one was already recorded. - pub(crate) fn set(&self, err: SessionError) -> bool { - self.reason.set(err).is_ok() - } - - /// Replace an error caused by the connection closing with the recorded reason, or - /// decode the peer's raw QUIC close code when none was recorded. - pub(crate) fn map(&self, err: SessionError) -> SessionError { - let conn = match &err { - SessionError::ConnectionError(conn) - | SessionError::SendDatagramError(noq::SendDatagramError::ConnectionLost(conn)) => { - Some(conn) - } - SessionError::WebTransportError(WebTransportError::Closed(..)) => None, - _ => return err, - }; - - if let Some(reason) = self.reason.get() { - return reason.clone(); - } - - match conn { - Some(noq::ConnectionError::ApplicationClosed(close)) if self.raw => { - match u32::try_from(close.error_code.into_inner()) { - Ok(code) => WebTransportError::Closed( - code, - String::from_utf8_lossy(&close.reason).into_owned(), - ) - .into(), - Err(_) => err, - } - } - _ => err, - } - } + pub(crate) fn new(raw: bool) -> Self { + Self { + raw, + reason: OnceLock::new(), + } + } + + /// The QUIC code to reset or stop a stream with. + pub(crate) fn encode_stream_code(&self, code: u32) -> crate::quic::driver::VarInt { + if self.raw { + return code.into(); + } + crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(code)).unwrap() + } + + /// Decode the peer's QUIC code for a stream reset or stop, or None if it is not one. + pub(crate) fn decode_stream_code(&self, code: crate::quic::driver::VarInt) -> Option { + let code = code.into_inner(); + // An older raw peer maps its codes into the HTTP/3 range too. No u32 reaches that + // range, so the mapped form cannot be mistaken for a raw code. + match web_transport_proto::error_from_http3(code) { + Some(code) => Some(code), + None if self.raw => u32::try_from(code).ok(), + None => None, + } + } + + /// Record the close reason, returning false if one was already recorded. + pub(crate) fn set(&self, err: SessionError) -> bool { + self.reason.set(err).is_ok() + } + + /// Replace an error caused by the connection closing with the recorded reason, or + /// decode the peer's raw QUIC close code when none was recorded. + pub(crate) fn map(&self, err: SessionError) -> SessionError { + let conn = match &err { + SessionError::ConnectionError(conn) + | SessionError::SendDatagramError(crate::quic::driver::SendDatagramError::ConnectionLost(conn)) => Some(conn), + SessionError::WebTransportError(WebTransportError::Closed(..)) => None, + _ => return err, + }; + + if let Some(reason) = self.reason.get() { + return reason.clone(); + } + + match conn { + Some(crate::quic::driver::ConnectionError::ApplicationClosed(close)) if self.raw => { + match u32::try_from(close.error_code.into_inner()) { + Ok(code) => { + WebTransportError::Closed(code, String::from_utf8_lossy(&close.reason).into_owned()).into() + } + Err(_) => err, + } + } + _ => err, + } + } } /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] pub enum WebTransportError { - #[error("closed: code={0} reason={1}")] - Closed(u32, String), + #[error("closed: code={0} reason={1}")] + Closed(u32, String), - #[error("unknown session")] - UnknownSession, + #[error("unknown session")] + UnknownSession, - #[error("read error: {0}")] - ReadError(#[from] noq::ReadExactError), + #[error("read error: {0}")] + ReadError(#[from] crate::quic::driver::ReadExactError), - #[error("write error: {0}")] - WriteError(#[from] noq::WriteError), + #[error("write error: {0}")] + WriteError(#[from] crate::quic::driver::WriteError), } -/// An error when writing to [`crate::SendStream`]. Similar to [`noq::WriteError`]. +/// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] pub enum WriteError { - #[error("STOP_SENDING: {0}")] - Stopped(u32), + #[error("STOP_SENDING: {0}")] + Stopped(u32), - #[error("invalid STOP_SENDING: {0}")] - InvalidStopped(noq::VarInt), + #[error("invalid STOP_SENDING: {0}")] + InvalidStopped(crate::quic::driver::VarInt), - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("stream closed")] - ClosedStream, + #[error("stream closed")] + ClosedStream, } -impl From for WriteError { - fn from(e: noq::WriteError) -> Self { - match e { - noq::WriteError::Stopped(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - } - } - noq::WriteError::ClosedStream => WriteError::ClosedStream, - noq::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), - noq::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for WriteError { + fn from(e: crate::quic::driver::WriteError) -> Self { + match e { + crate::quic::driver::WriteError::Stopped(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + } + } + crate::quic::driver::WriteError::ClosedStream => WriteError::ClosedStream, + crate::quic::driver::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + crate::quic::driver::WriteError::ZeroRttRejected => unreachable!("0-RTT not supported"), + } + } } -/// An error when reading from [`crate::RecvStream`]. Similar to [`noq::ReadError`]. +/// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] pub enum ReadError { - #[error("session error: {0}")] - SessionError(#[from] SessionError), + #[error("session error: {0}")] + SessionError(#[from] SessionError), - #[error("RESET_STREAM: {0}")] - Reset(u32), + #[error("RESET_STREAM: {0}")] + Reset(u32), - #[error("invalid RESET_STREAM: {0}")] - InvalidReset(noq::VarInt), + #[error("invalid RESET_STREAM: {0}")] + InvalidReset(crate::quic::driver::VarInt), - #[error("stream already closed")] - ClosedStream, + #[error("stream already closed")] + ClosedStream, } -impl From for ReadError { - fn from(value: noq::ReadError) -> Self { - match value { - noq::ReadError::Reset(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - } - } - noq::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), - noq::ReadError::ClosedStream => ReadError::ClosedStream, - noq::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), - } - } +impl From for ReadError { + fn from(value: crate::quic::driver::ReadError) -> Self { + match value { + crate::quic::driver::ReadError::Reset(code) => { + match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + } + } + crate::quic::driver::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), + crate::quic::driver::ReadError::ClosedStream => ReadError::ClosedStream, + crate::quic::driver::ReadError::ZeroRttRejected => unreachable!("0-RTT not supported"), + crate::quic::driver::ReadError::IllegalOrderedRead => { + unreachable!("WebTransport streams are only read in order") + } + } + } } -/// An error returned by [`crate::RecvStream::read_exact`]. Similar to [`noq::ReadExactError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] pub enum ReadExactError { - #[error("finished early")] - FinishedEarly(usize), + #[error("finished early")] + FinishedEarly(usize), - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadExactError { - fn from(e: noq::ReadExactError) -> Self { - match e { - noq::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), - noq::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), - } - } +impl From for ReadExactError { + fn from(e: crate::quic::driver::ReadExactError) -> Self { + match e { + crate::quic::driver::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), + crate::quic::driver::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), + } + } } -/// An error returned by [`crate::RecvStream::read_to_end`]. Similar to [`noq::ReadToEndError`]. +/// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] pub enum ReadToEndError { - #[error("too long")] - TooLong, + #[error("too long")] + TooLong, - #[error("read error: {0}")] - ReadError(#[from] ReadError), + #[error("read error: {0}")] + ReadError(#[from] ReadError), } -impl From for ReadToEndError { - fn from(e: noq::ReadToEndError) -> Self { - match e { - noq::ReadToEndError::TooLong => ReadToEndError::TooLong, - noq::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), - } - } +impl From for ReadToEndError { + fn from(e: crate::quic::driver::ReadToEndError) -> Self { + match e { + crate::quic::driver::ReadToEndError::TooLong => ReadToEndError::TooLong, + crate::quic::driver::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), + } + } } /// An error indicating the stream was already closed. @@ -267,39 +264,39 @@ impl From for ReadToEndError { #[error("stream closed")] pub struct ClosedStream; -impl From for ClosedStream { - fn from(_: noq::ClosedStream) -> Self { - ClosedStream - } +impl From for ClosedStream { + fn from(_: crate::quic::driver::ClosedStream) -> Self { + ClosedStream + } } /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] pub enum ServerError { - #[error("unexpected end of stream")] - UnexpectedEnd, + #[error("unexpected end of stream")] + UnexpectedEnd, - #[error("connection error")] - Connection(#[from] noq::ConnectionError), + #[error("connection error")] + Connection(#[from] crate::quic::driver::ConnectionError), - #[error("failed to write")] - WriteError(#[from] noq::WriteError), + #[error("failed to write")] + WriteError(#[from] crate::quic::driver::WriteError), - #[error("failed to read")] - ReadError(#[from] noq::ReadError), + #[error("failed to read")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("failed to exchange h3 settings")] - SettingsError(#[from] SettingsError), + #[error("failed to exchange h3 settings")] + SettingsError(#[from] SettingsError), - #[error("failed to exchange h3 connect")] - ConnectError(#[from] ConnectError), + #[error("failed to exchange h3 connect")] + ConnectError(#[from] ConnectError), - #[error("io error: {0}")] - IoError(Arc), + #[error("io error: {0}")] + IoError(Arc), - #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] - #[error("rustls error: {0}")] - Rustls(#[from] rustls::Error), + #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] + #[error("rustls error: {0}")] + Rustls(#[from] rustls::Error), } // #[derive(Clone, Error, Debug)] @@ -317,89 +314,89 @@ pub enum ServerError { // SessionError(#[from] SessionError), // } -// impl From for SendDatagramError { -// fn from(value: noq::SendDatagramError) -> Self { +// impl From for SendDatagramError { +// fn from(value: crate::quic::driver::SendDatagramError) -> Self { // match value { -// noq::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, -// noq::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, -// noq::SendDatagramError::TooLarge => SendDatagramError::TooLarge, -// noq::SendDatagramError::ConnectionLost(e) => +// crate::quic::driver::SendDatagramError::UnsupportedByPeer => SendDatagramError::UnsupportedPeer, +// crate::quic::driver::SendDatagramError::Disabled => SendDatagramError::DatagramSupportDisabled, +// crate::quic::driver::SendDatagramError::TooLarge => SendDatagramError::TooLarge, +// crate::quic::driver::SendDatagramError::ConnectionLost(e) => // SendDatagramError::SessionError(e.into()), } // } // } impl web_transport_trait::Error for SessionError { - fn session_error(&self) -> Option<(u32, String)> { - if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { - return Some((*code, reason.to_string())); - } + fn session_error(&self) -> Option<(u32, String)> { + if let SessionError::WebTransportError(WebTransportError::Closed(code, reason)) = self { + return Some((*code, reason.to_string())); + } - None - } + None + } } impl web_transport_trait::Error for WriteError { - fn session_error(&self) -> Option<(u32, String)> { - if let WriteError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - WriteError::Stopped(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let WriteError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + WriteError::Stopped(code) => Some(*code), + _ => None, + } + } } impl web_transport_trait::Error for ReadError { - fn session_error(&self) -> Option<(u32, String)> { - if let ReadError::SessionError(e) = self { - return e.session_error(); - } - - None - } - - fn stream_error(&self) -> Option { - match self { - ReadError::Reset(code) => Some(*code), - _ => None, - } - } + fn session_error(&self) -> Option<(u32, String)> { + if let ReadError::SessionError(e) = self { + return e.session_error(); + } + + None + } + + fn stream_error(&self) -> Option { + match self { + ReadError::Reset(code) => Some(*code), + _ => None, + } + } } #[cfg(test)] mod tests { - use super::*; - - /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its - /// range is not a WebTransport code. - #[test] - fn http3_stream_codes_stay_mapped() { - let close = CloseReason::new(false); - let mapped = close.encode_stream_code(5); - assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); - assert_eq!(close.decode_stream_code(mapped), Some(5)); - assert_eq!(close.decode_stream_code(5u32.into()), None); - } - - /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. - #[test] - fn raw_stream_codes_pass_through() { - let close = CloseReason::new(true); - assert_eq!(close.encode_stream_code(5).into_inner(), 5); - assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); - assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); - - let legacy = noq::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); - assert_eq!(close.decode_stream_code(legacy), Some(5)); - - // Past u32 and outside the HTTP/3 range: neither form. - let invalid = noq::VarInt::from_u64(1 << 40).unwrap(); - assert_eq!(close.decode_stream_code(invalid), None); - } + use super::*; + + /// WebTransport keeps the HTTP/3 mapping in both directions, and a code outside its + /// range is not a WebTransport code. + #[test] + fn http3_stream_codes_stay_mapped() { + let close = CloseReason::new(false); + let mapped = close.encode_stream_code(5); + assert_eq!(mapped.into_inner(), web_transport_proto::error_to_http3(5)); + assert_eq!(close.decode_stream_code(mapped), Some(5)); + assert_eq!(close.decode_stream_code(5u32.into()), None); + } + + /// Raw QUIC sends the code as is and reads both the raw and the legacy mapped form. + #[test] + fn raw_stream_codes_pass_through() { + let close = CloseReason::new(true); + assert_eq!(close.encode_stream_code(5).into_inner(), 5); + assert_eq!(close.decode_stream_code(5u32.into()), Some(5)); + assert_eq!(close.decode_stream_code(u32::MAX.into()), Some(u32::MAX)); + + let legacy = crate::quic::driver::VarInt::try_from(web_transport_proto::error_to_http3(5)).unwrap(); + assert_eq!(close.decode_stream_code(legacy), Some(5)); + + // Past u32 and outside the HTTP/3 range: neither form. + let invalid = crate::quic::driver::VarInt::from_u64(1 << 40).unwrap(); + assert_eq!(close.decode_stream_code(invalid), None); + } } diff --git a/rs/moq-tokio/src/quic/web/fan.rs b/rs/moq-tokio/src/quic/web/fan.rs new file mode 100644 index 0000000000..0962877fcd --- /dev/null +++ b/rs/moq-tokio/src/quic/web/fan.rs @@ -0,0 +1,140 @@ +use std::sync::{Arc, Mutex}; +use std::task::{Wake, Waker}; + +use kio::{Waiter, WaiterList}; + +/// A shared [`WaiterList`] that a [`Waker`] can wake. +/// +/// The accept futures below take one `Waker` and keep it. Handing them a caller's waker +/// would make the latest caller own the wakeup for everyone, so they are polled with +/// [`Fan::waker`] instead, which outlives every caller and wakes the whole list. +#[derive(Clone, Default)] +pub(crate) struct Fan(Arc); + +#[derive(Default)] +struct Inner(Mutex); + +#[derive(Default)] +struct State { + list: WaiterList, + /// Outstanding [`Hold`]s; a wake while any is alive is owed instead of delivered. + held: usize, + owed: bool, +} + +impl Fan { + /// Park `waiter` until the next wake. + pub(crate) fn register(&self, waiter: &Waiter) { + waiter.register(&mut self.0.0.lock().unwrap().list); + } + + /// A waker that wakes every parked waiter. + pub(crate) fn waker(&self) -> Waker { + Waker::from(self.0.clone()) + } + + /// Wake every parked waiter, or owe the wake until the last [`Hold`] drops. + pub(crate) fn wake(&self) { + self.0.notify(); + } + + /// Defer wakes until the guard drops, so none is delivered under the caller's lock. + pub(crate) fn hold(&self) -> Hold { + self.0.0.lock().unwrap().held += 1; + Hold(self.0.clone()) + } +} + +impl Inner { + fn notify(&self) { + let mut list = { + let mut state = self.0.lock().unwrap(); + if state.held > 0 { + state.owed = true; + return; + } + state.list.take() + }; + // Outside the lock: a waker may resume its task inline. + list.wake(); + } +} + +impl Wake for Inner { + fn wake(self: Arc) { + self.notify(); + } + + fn wake_by_ref(self: &Arc) { + self.notify(); + } +} + +/// Defers a [`Fan`]'s wakes while alive, delivering any owed one on drop. +pub(crate) struct Hold(Arc); + +impl Drop for Hold { + fn drop(&mut self) { + let owed = { + let mut state = self.0.0.lock().unwrap(); + state.held -= 1; + state.held == 0 && std::mem::take(&mut state.owed) + }; + if owed { + self.0.notify(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hold_defers_the_wake_until_dropped() { + let fan = Fan::default(); + let (waker, count) = counting(); + let waiter = Waiter::new(waker); + fan.register(&waiter); + + let hold = fan.hold(); + fan.waker().wake_by_ref(); + assert_eq!(count.load(), 0, "held back"); + + drop(hold); + assert_eq!(count.load(), 1, "delivered on drop"); + } + + #[test] + fn waker_wakes_every_waiter() { + let fan = Fan::default(); + let (a, count_a) = counting(); + let (b, count_b) = counting(); + // The list holds waiters weakly, so they have to outlive the wake. + let (a, b) = (Waiter::new(a), Waiter::new(b)); + fan.register(&a); + fan.register(&b); + + fan.wake(); + assert_eq!((count_a.load(), count_b.load()), (1, 1)); + } + + struct Count(std::sync::atomic::AtomicUsize); + + impl Count { + fn load(&self) -> usize { + self.0.load(std::sync::atomic::Ordering::SeqCst) + } + } + + impl Wake for Count { + fn wake(self: Arc) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + fn counting() -> (Waker, Arc) { + let count = Arc::new(Count(Default::default())); + (Waker::from(count.clone()), count) + } +} diff --git a/rs/moq-tokio/src/quic/web/mod.rs b/rs/moq-tokio/src/quic/web/mod.rs index e5f3706492..47bf299eae 100644 --- a/rs/moq-tokio/src/quic/web/mod.rs +++ b/rs/moq-tokio/src/quic/web/mod.rs @@ -3,7 +3,7 @@ //! //! WebTransport is layered on top of HTTP/3 which is then layered on top of QUIC. //! This library hides that detail and tries to expose only the QUIC API, delegating as much as -//! possible to the underlying implementation. See the [Noq documentation](https://docs.rs/moq-noq/latest/moq_noq/) for more documentation. +//! possible to the underlying implementation. See [`crate::quic::driver`] for more documentation. //! //! QUIC provides two primary APIs: //! @@ -43,12 +43,13 @@ pub use session::*; // Internal mod connect; +mod fan; mod settings; use connect::*; use settings::*; -// Required to access web_transport_moq::proto::ConnectError wrapped in ClientError +// Required to access proto::ConnectError wrapped in ClientError pub use connect::ConnectError; /// The HTTP/3 ALPN is required when negotiating a QUIC connection. @@ -57,15 +58,8 @@ pub const ALPN: &str = "h3"; /// Export our simple crypto provider. pub mod crypto; -/// Re-export the underlying QUIC implementation. -pub use noq; - /// Re-export the http crate because it's in the public API. pub use http; -/// Re-export kio, whose [`Waiter`](kio::Waiter) is how a caller of -/// [`SessionAccept`]'s poll methods holds its place in the accept queue. -pub use kio; - /// Re-export the WebTransport protocol implementation. pub use web_transport_proto as proto; diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index 4c3d188172..1e46bac784 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -1,159 +1,131 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::Bytes; -use crate::{CloseReason, ClosedStream, ReadError, ReadExactError, ReadToEndError, SessionError}; +use crate::quic::web::{CloseReason, ReadError, ReadExactError, ReadToEndError, SessionError}; -/// A stream that can be used to receive bytes. See [`noq::RecvStream`]. +/// A stream that can be used to receive bytes. See [`crate::quic::driver::RecvStream`]. #[derive(Debug)] pub struct RecvStream { - inner: noq::RecvStream, - close: Arc, + inner: crate::quic::driver::RecvStream, + close: Arc, } impl RecvStream { - pub(crate) fn new(stream: noq::RecvStream, close: Arc) -> Self { - Self { - inner: stream, - close, - } - } - - /// Decode the peer's reset code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::ReadError) -> ReadError { - match e { - noq::ReadError::Reset(code) => match self.close.decode_stream_code(code) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - }, - e => match ReadError::from(e) { - ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Tell the other end to stop sending data with the given error code. See - /// [`noq::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error - /// space with HTTP/3. A raw QUIC session sends the code as is. - pub fn stop(&mut self, code: u32) -> Result<(), noq::ClosedStream> { - let code = self.close.encode_stream_code(code); - self.inner.stop(code) - } - - // Unfortunately, we have to wrap ReadError for a bunch of functions. - - /// Read some data into the buffer and return the amount read. See [`noq::RecvStream::read`]. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - self.inner.read(buf).await.map_err(|e| self.map_error(e)) - } - - /// Fill the entire buffer with data. See [`noq::RecvStream::read_exact`]. - pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { - self.inner.read_exact(buf).await.map_err(|e| match e { - noq::ReadExactError::ReadError(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Read a chunk of data from the stream. See [`noq::RecvStream::read_chunk`]. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - self.inner - .read_chunk(max_length) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read chunks of data from the stream. See [`noq::RecvStream::read_many_chunks`]. - pub async fn read_many_chunks( - &mut self, - bufs: &mut [Bytes], - ) -> Result, ReadError> { - self.inner - .read_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Read until the end of the stream or the limit is hit. See [`noq::RecvStream::read_to_end`]. - pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { - self.inner - .read_to_end(size_limit) - .await - .map_err(|e| match e { - noq::ReadToEndError::Read(e) => self.map_error(e).into(), - e => e.into(), - }) - } - - /// Block until the stream has been reset and return the error code. See - /// [`noq::RecvStream::received_reset`]. - /// - /// Unlike Noq, this returns a SessionError, not a ResetError, because 0-RTT is not supported. - pub async fn received_reset(&mut self) -> Result, SessionError> { - match self.inner.received_reset().await { - Ok(None) => Ok(None), - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Err(noq::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), - Err(noq::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.inner.id() - } - - /// Returns the number of bytes read from this stream. - /// - /// This is the offset of the next byte to be read, i.e. the length of the contiguous - /// prefix of the stream consumed by the application. - pub fn bytes_read(&self) -> Result { - self.inner.bytes_read().map_err(|_| ClosedStream) - } - - // We purposely don't expose the 0RTT because it's not valid with WebTransport + pub(crate) fn new(stream: crate::quic::driver::RecvStream, close: Arc) -> Self { + Self { inner: stream, close } + } + + /// Decode the peer's reset code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::ReadError) -> ReadError { + match e { + crate::quic::driver::ReadError::Reset(code) => match self.close.decode_stream_code(code) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + e => match ReadError::from(e) { + ReadError::SessionError(e) => ReadError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Tell the other end to stop sending data with the given error code. See + /// [`crate::quic::driver::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. + pub fn stop(&mut self, code: u32) -> Result<(), crate::quic::driver::ClosedStream> { + let code = self.close.encode_stream_code(code); + self.inner.stop(code) + } + + // Unfortunately, we have to wrap ReadError for a bunch of functions. + + /// Read some data into the buffer and return the amount read. See [`crate::quic::driver::RecvStream::read`]. + pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { + self.inner.read(buf).await.map_err(|e| self.map_error(e)) + } + + /// Fill the entire buffer with data. See [`crate::quic::driver::RecvStream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { + self.inner.read_exact(buf).await.map_err(|e| match e { + crate::quic::driver::ReadExactError::ReadError(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Read a chunk of data from the stream. See [`crate::quic::driver::RecvStream::read_chunk`]. + pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { + self.inner + .read_chunk(max_length, true) + .await + .map(|chunk| chunk.map(|chunk| chunk.bytes)) + .map_err(|e| self.map_error(e)) + } + + /// Read until the end of the stream or the limit is hit. See [`crate::quic::driver::RecvStream::read_to_end`]. + pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { + self.inner.read_to_end(size_limit).await.map_err(|e| match e { + crate::quic::driver::ReadToEndError::Read(e) => self.map_error(e).into(), + e => e.into(), + }) + } + + /// Block until the stream has been reset and return the error code. See + /// [`crate::quic::driver::RecvStream::received_reset`]. + /// + /// Unlike the driver, this returns a SessionError, not a ResetError, because 0-RTT is not supported. + pub async fn received_reset(&mut self) -> Result, SessionError> { + match self.inner.received_reset().await { + Ok(None) => Ok(None), + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Err(crate::quic::driver::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::ResetError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.inner.id() + } + + // We purposely don't expose the 0RTT because it's not valid with WebTransport } impl tokio::io::AsyncRead for RecvStream { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut tokio::io::ReadBuf, - ) -> Poll> { - Pin::new(&mut self.inner).poll_read(cx, buf) - } + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } } impl web_transport_trait::RecvStream for RecvStream { - type Error = ReadError; + type Error = ReadError; - fn stop(&mut self, code: u32) { - Self::stop(self, code).ok(); - } + fn stop(&mut self, code: u32) { + Self::stop(self, code).ok(); + } - async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { - self.read(dst).await - } + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + self.read(dst).await + } - async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { - self.read_chunk(max).await - } + async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { + self.read_chunk(max).await + } - async fn closed(&mut self) -> Result<(), Self::Error> { - self.received_reset().await?; - Ok(()) - } + async fn closed(&mut self) -> Result<(), Self::Error> { + self.received_reset().await?; + Ok(()) + } } diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 39af0d1fc7..7f87718527 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -1,197 +1,168 @@ use std::{ - io, - pin::Pin, - sync::Arc, - task::{Context, Poll}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, }; use bytes::{Buf, Bytes}; -use crate::{CloseReason, ClosedStream, SessionError, WriteError}; +use crate::quic::web::{CloseReason, ClosedStream, SessionError, WriteError}; -/// A stream that can be used to send bytes. See [`noq::SendStream`]. +/// A stream that can be used to send bytes. See [`crate::quic::driver::SendStream`]. /// /// This wrapper is mainly needed for error codes, which is unfortunate. /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. #[derive(Debug)] pub struct SendStream { - stream: noq::SendStream, - close: Arc, + stream: crate::quic::driver::SendStream, + close: Arc, } impl SendStream { - pub(crate) fn new(stream: noq::SendStream, close: Arc) -> Self { - Self { stream, close } - } - - /// Decode the peer's stop code, and report a session error as the session's close reason. - fn map_error(&self, e: noq::WriteError) -> WriteError { - match e { - noq::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { - Some(code) => WriteError::Stopped(code), - None => WriteError::InvalidStopped(code), - }, - e => match WriteError::from(e) { - WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), - e => e, - }, - } - } - - /// Abruptly reset the stream with the provided error code. See [`noq::SendStream::reset`]. - /// This is a u32 with WebTransport because we share the error space with HTTP/3. - /// A raw QUIC session sends the code as is. - pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { - let code = self.close.encode_stream_code(code); - self.stream.reset(code).map_err(Into::into) - } - - /// Wait until the stream has been stopped and return the error code. See - /// [`noq::SendStream::stopped`]. - /// - /// Unlike Noq, this returns None if the code is not a valid WebTransport error code. - /// Also unlike Noq, this returns a SessionError, not a StoppedError, because 0-RTT is not - /// supported. - pub async fn stopped(&self) -> Result, SessionError> { - match self.stream.stopped().await { - Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), - Ok(None) => Ok(None), - Err(noq::StoppedError::ConnectionLost(conn_err)) => { - Err(self.close.map(conn_err.into())) - } - Err(noq::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), - } - } - - // Unfortunately, we have to wrap WriteError for a bunch of functions. - - /// Write some data to the stream, returning the size written. See [`noq::SendStream::write`]. - pub async fn write(&mut self, buf: &[u8]) -> Result { - self.stream.write(buf).await.map_err(|e| self.map_error(e)) - } - - /// Write all of the data to the stream. See [`noq::SendStream::write_all`]. - pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { - self.stream - .write_all(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write chunks of data to the stream, returning the number of bytes written. - /// - /// See [`noq::SendStream::write_many_chunks`]. - pub async fn write_many_chunks( - &mut self, - bufs: &mut &mut [Bytes], - ) -> Result { - self.stream - .write_many_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write a chunk of data to the stream. See [`noq::SendStream::write_chunk`]. - pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { - self.stream - .write_chunk(buf) - .await - .map_err(|e| self.map_error(e)) - } - - /// Write all of the chunks of data to the stream. See [`noq::SendStream::write_all_chunks`]. - pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { - self.stream - .write_all_chunks(bufs) - .await - .map_err(|e| self.map_error(e)) - } - - /// Mark the stream as finished, such that no more data can be written. See - /// [`noq::SendStream::finish`]. - /// - /// WARNING: This is implicitly called on Drop, but it's a common footgun in Noq. - /// If you cancel futures by dropping them you'll get incomplete writes. - pub fn finish(&mut self) -> Result<(), ClosedStream> { - self.stream.finish().map_err(Into::into) - } - - pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { - self.stream.set_priority(order).map_err(Into::into) - } - - pub fn priority(&self) -> Result { - self.stream.priority().map_err(Into::into) - } - - /// Return the underlying QUIC stream ID. - /// - /// > **Warning** - /// > - /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other - /// > sessions. - /// > The [noq::StreamId::index] might not increment by 1 like expected when using [noq]. - /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> noq::StreamId { - self.stream.id() - } + pub(crate) fn new(stream: crate::quic::driver::SendStream, close: Arc) -> Self { + Self { stream, close } + } + + /// Decode the peer's stop code, and report a session error as the session's close reason. + fn map_error(&self, e: crate::quic::driver::WriteError) -> WriteError { + match e { + crate::quic::driver::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + Some(code) => WriteError::Stopped(code), + None => WriteError::InvalidStopped(code), + }, + e => match WriteError::from(e) { + WriteError::SessionError(e) => WriteError::SessionError(self.close.map(e)), + e => e, + }, + } + } + + /// Abruptly reset the stream with the provided error code. See [`crate::quic::driver::SendStream::reset`]. + /// This is a u32 with WebTransport because we share the error space with HTTP/3. + /// A raw QUIC session sends the code as is. + pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + let code = self.close.encode_stream_code(code); + self.stream.reset(code).map_err(Into::into) + } + + /// Wait until the stream has been stopped and return the error code. See + /// [`crate::quic::driver::SendStream::stopped`]. + /// + /// Unlike the driver, this returns None if the code is not a valid WebTransport error code. + /// Also unlike the driver, this returns a SessionError, not a StoppedError, because 0-RTT is not + /// supported. + pub async fn stopped(&self) -> Result, SessionError> { + match self.stream.stopped().await { + Ok(Some(code)) => Ok(self.close.decode_stream_code(code)), + Ok(None) => Ok(None), + Err(crate::quic::driver::StoppedError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(crate::quic::driver::StoppedError::ZeroRttRejected) => unreachable!("0-RTT not supported"), + } + } + + // Unfortunately, we have to wrap WriteError for a bunch of functions. + + /// Write some data to the stream, returning the size written. See [`crate::quic::driver::SendStream::write`]. + pub async fn write(&mut self, buf: &[u8]) -> Result { + self.stream.write(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the data to the stream. See [`crate::quic::driver::SendStream::write_all`]. + pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { + self.stream.write_all(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write a chunk of data to the stream. See [`crate::quic::driver::SendStream::write_chunk`]. + pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { + self.stream.write_chunk(buf).await.map_err(|e| self.map_error(e)) + } + + /// Write all of the chunks of data to the stream. See [`crate::quic::driver::SendStream::write_all_chunks`]. + pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { + self.stream.write_all_chunks(bufs).await.map_err(|e| self.map_error(e)) + } + + /// Mark the stream as finished, such that no more data can be written. See + /// [`crate::quic::driver::SendStream::finish`]. + /// + /// WARNING: This is implicitly called on Drop, but it's a common footgun in QUIC libraries. + /// If you cancel futures by dropping them you'll get incomplete writes. + pub fn finish(&mut self) -> Result<(), ClosedStream> { + self.stream.finish().map_err(Into::into) + } + + pub fn set_priority(&self, order: i32) -> Result<(), ClosedStream> { + self.stream.set_priority(order).map_err(Into::into) + } + + pub fn priority(&self) -> Result { + self.stream.priority().map_err(Into::into) + } + + /// Return the underlying QUIC stream ID. + /// + /// > **Warning** + /// > + /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other + /// > sessions. + /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > This is why the Javascript WebTransport API does not expose the Stream ID. + pub fn quic_id(&self) -> crate::quic::driver::StreamId { + self.stream.id() + } } impl tokio::io::AsyncWrite for SendStream { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - // We have to use this syntax because noq added its own poll_write method. - tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) - } - - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_flush(cx) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_shutdown(cx) - } + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + // We have to use this syntax because the driver has its own poll_write method. + tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } } impl web_transport_trait::SendStream for SendStream { - type Error = WriteError; - - fn set_priority(&mut self, order: i32) { - Self::set_priority(self, order).ok(); - } - - fn reset(&mut self, code: u32) { - Self::reset(self, code).ok(); - } - - fn finish(&mut self) -> Result<(), Self::Error> { - Self::finish(self).map_err(|_| WriteError::ClosedStream) - } - - async fn write(&mut self, buf: &[u8]) -> Result { - Self::write(self, buf).await - } - - async fn write_buf(&mut self, buf: &mut B) -> Result { - // This can avoid making a copy when Buf is Bytes, as Noq will allocate anyway. - let size = buf.chunk().len(); - let chunk = buf.copy_to_bytes(size); - self.write_chunk(chunk).await?; - Ok(size) - } - - async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { - self.write_chunk(chunk).await - } - - async fn closed(&mut self) -> Result<(), Self::Error> { - // NOTE: This used to require &mut in an older version of Noq. - match self.stopped().await? { - Some(code) => Err(WriteError::Stopped(code)), - None => Ok(()), - } - } + type Error = WriteError; + + fn set_priority(&mut self, order: i32) { + Self::set_priority(self, order).ok(); + } + + fn reset(&mut self, code: u32) { + Self::reset(self, code).ok(); + } + + fn finish(&mut self) -> Result<(), Self::Error> { + Self::finish(self).map_err(|_| WriteError::ClosedStream) + } + + async fn write(&mut self, buf: &[u8]) -> Result { + Self::write(self, buf).await + } + + async fn write_buf(&mut self, buf: &mut B) -> Result { + // This can avoid making a copy when Buf is Bytes, as the driver will allocate anyway. + let size = buf.chunk().len(); + let chunk = buf.copy_to_bytes(size); + self.write_chunk(chunk).await?; + Ok(size) + } + + async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { + self.write_chunk(chunk).await + } + + async fn closed(&mut self) -> Result<(), Self::Error> { + match self.stopped().await? { + Some(code) => Err(WriteError::Stopped(code)), + None => Ok(()), + } + } } diff --git a/rs/moq-tokio/src/quic/web/server.rs b/rs/moq-tokio/src/quic/web/server.rs index 8af3e81c98..381ec11934 100644 --- a/rs/moq-tokio/src/quic/web/server.rs +++ b/rs/moq-tokio/src/quic/web/server.rs @@ -1,15 +1,15 @@ #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use std::sync::Arc; -use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] use rustls::pki_types::{CertificateDer, PrivateKeyDer}; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -use crate::{crypto, CongestionControl}; -use crate::{ - proto::{ConnectRequest, ConnectResponse}, - Connecting, ServerError, Session, Settings, +use crate::quic::web::{CongestionControl, crypto}; +use crate::quic::web::{ + Connecting, ServerError, Session, Settings, + proto::{ConnectRequest, ConnectResponse}, }; #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] @@ -17,184 +17,178 @@ use crate::{ /// /// This is optional; advanced users may use [Server::new] directly. pub struct ServerBuilder { - provider: crypto::Provider, - addr: std::net::SocketAddr, - congestion_controller: - Option>, + provider: crypto::Provider, + addr: std::net::SocketAddr, + congestion_controller: Option>, } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl Default for ServerBuilder { - fn default() -> Self { - Self::new() - } + fn default() -> Self { + Self::new() + } } #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] impl ServerBuilder { - /// Create a server builder with sane defaults. - pub fn new() -> Self { - Self { - provider: crypto::default_provider(), - addr: "[::]:443".parse().unwrap(), - congestion_controller: None, - } - } - - /// Listen on the specified address. - pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { - Self { addr, ..self } - } - - /// Enable the specified congestion controller. - pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { - self.congestion_controller = match algorithm { - CongestionControl::LowLatency => Some(Arc::new(noq::congestion::Bbr3Config::default())), - // TODO BBR is also higher throughput in theory. - CongestionControl::Throughput => { - Some(Arc::new(noq::congestion::CubicConfig::default())) - } - CongestionControl::Default => None, - }; - - self - } - - /// Supply a certificate used for TLS. - // TODO support multiple certs based on...? - pub fn with_certificate( - self, - chain: Vec>, - key: PrivateKeyDer<'static>, - ) -> Result { - // Standard Noq setup - let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) - .with_protocol_versions(&[&rustls::version::TLS13])? - .with_no_client_auth() - .with_single_cert(chain, key)?; - - config.alpn_protocols = vec![crate::ALPN.as_bytes().to_vec()]; // this one is important - - let config: noq::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); - let config = noq::ServerConfig::with_crypto(Arc::new(config)); - - let server = - noq::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; - - Ok(Server::new(server)) - } + /// Create a server builder with sane defaults. + pub fn new() -> Self { + Self { + provider: crypto::default_provider(), + addr: "[::]:443".parse().unwrap(), + congestion_controller: None, + } + } + + /// Listen on the specified address. + pub fn with_addr(self, addr: std::net::SocketAddr) -> Self { + Self { addr, ..self } + } + + /// Enable the specified congestion controller. + pub fn with_congestion_control(mut self, algorithm: CongestionControl) -> Self { + self.congestion_controller = match algorithm { + CongestionControl::LowLatency => Some(Arc::new(crate::quic::driver::congestion::Bbr3Config::default())), + // TODO BBR is also higher throughput in theory. + CongestionControl::Throughput => Some(Arc::new(crate::quic::driver::congestion::CubicConfig::default())), + CongestionControl::Default => None, + }; + + self + } + + /// Supply a certificate used for TLS. + // TODO support multiple certs based on...? + pub fn with_certificate( + self, + chain: Vec>, + key: PrivateKeyDer<'static>, + ) -> Result { + // Standard QUIC endpoint setup + let mut config = rustls::ServerConfig::builder_with_provider(self.provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_no_client_auth() + .with_single_cert(chain, key)?; + + config.alpn_protocols = vec![crate::quic::web::ALPN.as_bytes().to_vec()]; // this one is important + + let config: crate::quic::driver::crypto::rustls::QuicServerConfig = config.try_into().unwrap(); + let config = crate::quic::driver::ServerConfig::with_crypto(Arc::new(config)); + + let server = + crate::quic::driver::Endpoint::server(config, self.addr).map_err(|e| ServerError::IoError(e.into()))?; + + Ok(Server::new(server)) + } } /// A WebTransport server that accepts new sessions. pub struct Server { - endpoint: noq::Endpoint, - accept: FuturesUnordered>>, + endpoint: crate::quic::driver::Endpoint, + accept: FuturesUnordered>>, } impl core::ops::Deref for Server { - type Target = noq::Endpoint; + type Target = crate::quic::driver::Endpoint; - fn deref(&self) -> &Self::Target { - &self.endpoint - } + fn deref(&self) -> &Self::Target { + &self.endpoint + } } impl Server { - /// Manually create a new server with a manually constructed Endpoint. - /// - /// NOTE: The ALPN must be set to `crate::ALPN` for WebTransport to work. - pub fn new(endpoint: noq::Endpoint) -> Self { - Self { - endpoint, - accept: Default::default(), - } - } - - /// Accept a new WebTransport session Request from a client. - pub async fn accept(&mut self) -> Option { - loop { - tokio::select! { - res = self.endpoint.accept() => { - let conn = res?; - self.accept.push(Box::pin(async move { - let conn = conn.await?; - Request::accept(conn).await - })); - } - Some(res) = self.accept.next() => { - if let Ok(session) = res { - return Some(session) - } - } - } - } - } + /// Manually create a new server with a manually constructed Endpoint. + /// + /// NOTE: The ALPN must be set to `crate::quic::web::ALPN` for WebTransport to work. + pub fn new(endpoint: crate::quic::driver::Endpoint) -> Self { + Self { + endpoint, + accept: Default::default(), + } + } + + /// Accept a new WebTransport session Request from a client. + pub async fn accept(&mut self) -> Option { + loop { + tokio::select! { + res = self.endpoint.accept() => { + let conn = res?; + self.accept.push(Box::pin(async move { + let conn = conn.await?; + Request::accept(conn).await + })); + } + Some(res) = self.accept.next() => { + if let Ok(session) = res { + return Some(session) + } + } + } + } + } } /// A mostly complete WebTransport handshake, just awaiting the server's decision on whether to /// accept or reject the session based on the URL. pub struct Request { - conn: noq::Connection, - settings: Settings, - connect: Connecting, + conn: crate::quic::driver::Connection, + settings: Settings, + connect: Connecting, } impl Request { - /// Accept a new WebTransport session from a client. - pub async fn accept(conn: noq::Connection) -> Result { - // Perform the H3 handshake by sending/reciving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Accept the CONNECT request but don't send a response yet. - let connect = Connecting::accept(&conn).await?; - - // Return the resulting request with a reference to the settings/connect streams. - Ok(Self { - conn, - settings, - connect, - }) - } - - pub async fn ok(self) -> Result { - self.respond(ConnectResponse::OK).await - } - - /// Reply to the session with the given response, usually 200 OK. - /// - /// [ConnectResponse::with_protocol] can be used to select a subprotocol. - pub async fn respond( - self, - response: impl Into, - ) -> Result { - let response = response.into(); - let connect = self.connect.respond(response).await?; - Ok(Session::new(self.conn, self.settings, connect)) - } - - /// Reject the session with the given status code. - pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { - self.connect.reject(status).await?; - Ok(()) - } - - /// Returns the underlying QUIC connection. - pub fn conn(&self) -> &noq::Connection { - &self.conn - } - - /// Returns the CONNECT request that was sent by the client. - /// - /// DEPRECATED: You can access this via the Deref impl. - pub fn connect(&self) -> &ConnectRequest { - &self.connect - } + /// Accept a new WebTransport session from a client. + pub async fn accept(conn: crate::quic::driver::Connection) -> Result { + // Perform the H3 handshake by sending/reciving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Accept the CONNECT request but don't send a response yet. + let connect = Connecting::accept(&conn).await?; + + // Return the resulting request with a reference to the settings/connect streams. + Ok(Self { + conn, + settings, + connect, + }) + } + + pub async fn ok(self) -> Result { + self.respond(ConnectResponse::OK).await + } + + /// Reply to the session with the given response, usually 200 OK. + /// + /// [ConnectResponse::with_protocol] can be used to select a subprotocol. + pub async fn respond(self, response: impl Into) -> Result { + let response = response.into(); + let connect = self.connect.respond(response).await?; + Ok(Session::new(self.conn, self.settings, connect)) + } + + /// Reject the session with the given status code. + pub async fn reject(self, status: http::StatusCode) -> Result<(), ServerError> { + self.connect.reject(status).await?; + Ok(()) + } + + /// Returns the underlying QUIC connection. + pub fn conn(&self) -> &crate::quic::driver::Connection { + &self.conn + } + + /// Returns the CONNECT request that was sent by the client. + /// + /// DEPRECATED: You can access this via the Deref impl. + pub fn connect(&self) -> &ConnectRequest { + &self.connect + } } impl core::ops::Deref for Request { - type Target = ConnectRequest; + type Target = ConnectRequest; - fn deref(&self) -> &Self::Target { - &self.connect - } + fn deref(&self) -> &Self::Target { + &self.connect + } } diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index 82a9149d84..ad0fc8c2d6 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -1,639 +1,609 @@ use std::{ - fmt, - future::Future, - io::Cursor, - ops::Deref, - pin::Pin, - sync::{Arc, Mutex, OnceLock}, - task::{Context, Poll, Waker}, - time::Duration, + fmt, + future::Future, + io::Cursor, + ops::Deref, + pin::Pin, + sync::{Arc, Mutex, OnceLock}, + task::{Context, Poll, Waker}, + time::Duration, }; use bytes::{Bytes, BytesMut}; use futures::stream::{FuturesUnordered, Stream, StreamExt}; -use kio::{Fan, Waiter}; +use kio::Waiter; -use crate::{ - proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, - ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, - WebTransportError, +use crate::quic::web::fan::Fan; +use crate::quic::web::{ + ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, WebTransportError, + proto::{ConnectRequest, ConnectResponse, Frame, StreamUni, VarInt}, }; /// The ALPN the QUIC handshake negotiated, or `None` if there was none, the handshake /// has not completed, or it is not valid UTF-8. /// -/// Noq only exposes the ALPN via a downcast of the boxed handshake data, so the result +/// The driver only exposes the ALPN via a downcast of the boxed handshake data, so the result /// is owned rather than borrowed from the connection. #[cfg(any(feature = "aws-lc-rs", feature = "ring"))] -fn negotiated_alpn(conn: &noq::Connection) -> Option { - let data = conn.handshake_data()?; - let data = data.downcast::().ok()?; +fn negotiated_alpn(conn: &crate::quic::driver::Connection) -> Option { + let data = conn.handshake_data()?; + let data = data + .downcast::() + .ok()?; - String::from_utf8(data.protocol?).ok() + String::from_utf8(data.protocol?).ok() } -/// `noq::crypto::rustls` only exists once a TLS backend is enabled, so there is no +/// `crate::quic::driver::crypto::rustls` only exists once a TLS backend is enabled, so there is no /// handshake data type to name here. /// /// This crate's own builders are gated the same way, so normally there is no connection -/// to read an ALPN from either. A caller that enables a provider on `noq` directly and +/// to read an ALPN from either. A caller that enables a provider on `moq-quic` directly and /// brings its own endpoint can still establish one, and [`Session::protocol`] reports /// `None` for those raw sessions. Enable this crate's `aws-lc-rs` or `ring` feature to /// get the ALPN. #[cfg(not(any(feature = "aws-lc-rs", feature = "ring")))] -fn negotiated_alpn(_conn: &noq::Connection) -> Option { - None +fn negotiated_alpn(_conn: &crate::quic::driver::Connection) -> Option { + None } /// An established WebTransport session, acting like a full QUIC connection. See -/// [`noq::Connection`]. +/// [`crate::quic::driver::Connection`]. /// /// It is important to remember that WebTransport is layered on top of QUIC: /// 1. Each stream starts with a few bytes identifying the stream type and session ID. /// 2. Errors codes are encoded with the session ID, so they aren't full QUIC error codes. /// 3. Stream IDs may have gaps in them, used by HTTP/3 transparent to the application. /// -/// Deref is used to expose non-overloaded methods on [`noq::Connection`]. +/// Deref is used to expose non-overloaded methods on [`crate::quic::driver::Connection`]. /// These should be safe to use with WebTransport, but file a PR if you find one that isn't. #[derive(Clone)] pub struct Session { - conn: noq::Connection, - - // The session ID, as determined by the stream ID of the connect request. - session_id: Option, - - // The accept logic is stateful, so use an Arc to share it. - accept: Option>>, - - // Cache the headers in front of each stream we open. - header_uni: Vec, - header_bi: Vec, - header_datagram: Vec, - - // Keep a reference to the settings and connect stream to avoid closing them until dropped. - #[allow(dead_code)] - settings: Option>, - - // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. - // Wrapped in Arc>> so close() can take it exactly once. - connect_send: Arc>>, - - // Why the session closed: set once by a local close() or the background task - // when a remote CloseWebTransportSession capsule is received. - close: Arc, - - // The request sent by the client, or None for a raw QUIC session. - request: Option, - - // The response sent by the server, or None for a raw QUIC session. - response: Option, - - // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. Noq only - // exposes it behind an allocating downcast, so `protocol()` has nothing in the - // connection to borrow from and needs somewhere to keep the result. - // - // Resolved on first read rather than at construction: `raw()` accepts any - // connection, including one whose handshake has not completed and so has no ALPN - // yet. Only a successful read is latched, so an early call cannot pin `None` for - // the life of the session. Shared across clones. - // - // Unused by HTTP/3 sessions, which read the subprotocol out of the response. - alpn: Arc>, + conn: crate::quic::driver::Connection, + + // The session ID, as determined by the stream ID of the connect request. + session_id: Option, + + // The accept logic is stateful, so use an Arc to share it. + accept: Option>>, + + // Cache the headers in front of each stream we open. + header_uni: Vec, + header_bi: Vec, + header_datagram: Vec, + + // Keep a reference to the settings and connect stream to avoid closing them until dropped. + #[allow(dead_code)] + settings: Option>, + + // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. + // Wrapped in Arc>> so close() can take it exactly once. + connect_send: Arc>>, + + // Why the session closed: set once by a local close() or the background task + // when a remote CloseWebTransportSession capsule is received. + close: Arc, + + // The request sent by the client, or None for a raw QUIC session. + request: Option, + + // The response sent by the server, or None for a raw QUIC session. + response: Option, + + // The ALPN negotiated by the QUIC handshake, for raw QUIC sessions. The driver only + // exposes it behind an allocating downcast, so `protocol()` has nothing in the + // connection to borrow from and needs somewhere to keep the result. + // + // Resolved on first read rather than at construction: `raw()` accepts any + // connection, including one whose handshake has not completed and so has no ALPN + // yet. Only a successful read is latched, so an early call cannot pin `None` for + // the life of the session. Shared across clones. + // + // Unused by HTTP/3 sessions, which read the subprotocol out of the response. + alpn: Arc>, } impl Session { - pub(crate) fn new(conn: noq::Connection, settings: Settings, connect: Connected) -> Self { - // The session ID is the stream ID of the CONNECT request. - let session_id = connect.session_id(); - - // Cache the tiny header we write in front of each stream we open. - let mut header_uni = Vec::new(); - StreamUni::WEBTRANSPORT.encode(&mut header_uni); - session_id.encode(&mut header_uni); - - let mut header_bi = Vec::new(); - Frame::WEBTRANSPORT.encode(&mut header_bi); - session_id.encode(&mut header_bi); - - let mut header_datagram = Vec::new(); - session_id.encode(&mut header_datagram); - - let close = Arc::new(CloseReason::new(false)); - - // Accept logic is stateful, so use an Arc to share it. - let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); - - let this = Self { - conn, - accept: Some(Arc::new(Mutex::new(accept))), - session_id: Some(session_id), - header_uni, - header_bi, - header_datagram, - settings: Some(Arc::new(settings)), - connect_send: Arc::new(Mutex::new(Some(connect.send))), - close: close.clone(), - request: Some(connect.request.clone()), - response: Some(connect.response.clone()), - alpn: Default::default(), - }; - - // Run a background task to read capsules from the CONNECT recv stream. - let conn2 = this.conn.clone(); - tokio::spawn(Self::run_recv(conn2, connect.recv, close)); - - this - } - - // Read capsules from the CONNECT recv stream until it's closed, - // then record the close error and tear down the connection. - async fn run_recv(conn: noq::Connection, recv: noq::RecvStream, close: Arc) { - let close_info = Self::read_capsules(recv).await; - let code = close_info.as_ref().map_or(0, |(c, _)| *c); - - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Try to record the remote close error. If close() already set - // the error, it owns the connection teardown, so we bail out. - match close_info { - Some((code, reason)) => { - let err = WebTransportError::Closed(code, reason.clone()); - if !close.set(err.into()) { - return; - } - conn.close(http3_code, reason.as_bytes()); - } - None => { - let err = noq::ConnectionError::LocallyClosed.into(); - if !close.set(err) { - return; - } - conn.close(http3_code, b""); - } - }; - } - - // Keep reading capsules from the CONNECT recv stream until it's closed. - // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, - // or None if the stream closed without a capsule. - async fn read_capsules(recv: noq::RecvStream) -> Option<(u32, String)> { - let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); - loop { - match reader.read().await { - Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { - code, - reason, - })) => return Some((code, reason)), - Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} - Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { - tracing::warn!(%typ, size = payload.len(), "unknown capsule"); - } - Ok(None) => return None, - Err(e) => { - tracing::warn!(?e, "failed to read capsule"); - return None; - } - } - } - } - - /// Connect using an established QUIC connection if you want to create the connection yourself. - /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. - pub async fn connect( - conn: noq::Connection, - request: impl Into, - ) -> Result { - let request = request.into(); - - // Perform the H3 handshake by sending/receiving SETTINGS frames. - let settings = Settings::connect(&conn).await?; - - // Send the HTTP/3 CONNECT request. - let connect = Connected::open(&conn, request).await?; - - // Return the resulting session with a reference to the control/connect streams. - // If either stream is closed, then the session will be closed, so we need to keep them - // around. - let session = Session::new(conn, settings, connect); - - Ok(session) - } - - /// Accept a new unidirectional stream. See [`noq::Connection::accept_uni`]. - pub async fn accept_uni(&self) -> Result { - if let Some(accept) = &self.accept { - // `kio::wait` owns the waiter, so dropping this future — a `timeout` that - // expires, say — also drops its registration in `SessionAccept`. - kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let recv = self - .conn - .accept_uni() - .await - .map_err(|e| self.map_error(e))?; - Ok(RecvStream::new(recv, self.close.clone())) - } - } - - /// Accept a new bidirectional stream. See [`noq::Connection::accept_bi`]. - pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - if let Some(accept) = &self.accept { - kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) - .await - .map_err(|e| self.map_error(e)) - } else { - let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - } - - /// Open a new unidirectional stream. See [`noq::Connection::open_uni`]. - pub async fn open_uni(&self) -> Result { - let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_uni) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(SendStream::new(send, self.close.clone())) - } - - /// Open a new bidirectional stream. See [`noq::Connection::open_bi`]. - pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { - let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; - - // Set the stream priority to max and then write the stream header. - // Otherwise the application could write data with lower priority than the header, resulting - // in queuing. Also the header is very important for determining the session ID - // without reliable reset. - send.set_priority(i32::MAX).ok(); - Self::write_full(&mut send, &self.header_bi) - .await - .map_err(|e| self.map_error(e))?; - - // Reset the stream priority back to the default of 0. - send.set_priority(0).ok(); - Ok(( - SendStream::new(send, self.close.clone()), - RecvStream::new(recv, self.close.clone()), - )) - } - - /// Asynchronously receives an application datagram from the remote peer. - /// - /// This method is used to receive an application datagram sent by the remote - /// peer over the connection. - /// It waits for a datagram to become available and returns the received bytes. - pub async fn read_datagram(&self) -> Result { - let mut datagram = self - .conn - .read_datagram() - .await - .map_err(|e| self.map_error(e))?; - - let mut cursor = Cursor::new(&datagram); - - if let Some(session_id) = self.session_id { - // We have to check and strip the session ID from the datagram. - let actual_id = - VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; - if actual_id != session_id { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // Return the datagram without the session ID. - let datagram = datagram.split_off(cursor.position() as usize); - - Ok(datagram) - } - - /// Sends an application datagram to the remote peer. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram(buf.into()) - } else { - self.conn.send_datagram(data) - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Sends an application datagram to the remote peer, waiting for buffer space if needed. - /// - /// Datagrams are unreliable and may be dropped or delivered out of order. - /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // Unfortunately, we need to allocate/copy each datagram because of the Noq API. - // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram_wait(buf.into()).await - } else { - self.conn.send_datagram_wait(data).await - }; - - result.map_err(|e| self.map_error(e))?; - Ok(()) - } - - /// Computes the maximum size of datagrams that may be passed to - /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. - pub fn max_datagram_size(&self) -> usize { - self.conn - .max_datagram_size() - .unwrap_or(0) - .saturating_sub(self.header_datagram.len()) - } - - /// Returns the available buffer space for sending datagrams. - /// - /// This value is adjusted to account for the WebTransport session header. - pub fn datagram_send_buffer_space(&self) -> usize { - self.conn - .datagram_send_buffer_space() - .saturating_sub(self.header_datagram.len()) - } - - /// Close the session with an error code and reason. - /// - /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` - /// capsule is written on the CONNECT stream before the QUIC connection is closed. - /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. - /// - /// The capsule write and connection close happen asynchronously in a spawned task - /// that keeps the session alive until then, so the caller may drop it right away. - /// `await` [`Session::closed()`] to learn when the connection is gone. Session - /// operations will fail once the QUIC connection is closed. - pub fn close(&self, code: u32, reason: &[u8]) { - // First close wins: a no-op if the connection already closed, the background - // task recorded a remote close, or close() was already called. - if let Some(err) = self.conn.close_reason() { - self.close.set(self.map_error(err)); - return; - } - let err = SessionError::ConnectionError(noq::ConnectionError::LocallyClosed); - if !self.close.set(err) { - return; - } - - if self.session_id.is_some() { - // Take the send stream for the capsule write. - let send = self.connect_send.lock().unwrap().take(); - - if let Some(send) = send { - let reason = String::from_utf8_lossy(reason).into_owned(); - let capsule = - web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .unwrap_or(Duration::from_millis(100)); - let timeout = (rtt * 3).max(Duration::from_millis(100)); - - // The task owns a clone of the whole session, not just the connection: - // dropping the caller's last handle would otherwise end the HTTP/3 - // control and QPACK streams before the capsule is delivered, which a - // browser treats as a connection error, losing the code and reason. - let session = self.clone(); - tokio::spawn(async move { - session - .close_with_capsule(send, capsule, code, timeout) - .await; - }); - } - } else { - // Raw QUIC mode: no capsule needed. - self.conn.close(code.into(), reason); - } - } - - /// Deliver the close capsule and FIN, then give the peer its teardown grace. - async fn close_with_capsule( - &self, - mut send: noq::SendStream, - capsule: web_transport_proto::Capsule, - code: u32, - timeout: std::time::Duration, - ) { - let http3_code: noq::VarInt = web_transport_proto::error_to_http3(code) - .try_into() - .unwrap(); - - // Encode the capsule, then wrap it in an HTTP/3 DATA frame. - // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT - // stream (RFC 9297 Section 3.2). - let mut capsule_bytes = Vec::new(); - capsule.encode(&mut capsule_bytes); - - let mut frame = Vec::new(); - Frame::DATA.encode(&mut frame); - let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { - tracing::warn!("capsule too large to encode as DATA frame"); - self.conn.close(http3_code, b""); - return; - }; - len.encode(&mut frame); - frame.extend_from_slice(&capsule_bytes); - - // Write the DATA frame to the CONNECT send stream. - if let Err(e) = send.write_all(&frame).await { - tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); - self.conn.close(http3_code, b""); - return; - } - - // FIN the send stream so the peer knows no more capsules are coming. - if let Err(e) = send.finish() { - tracing::warn!(?e, "failed to finish CONNECT send stream"); - self.conn.close(http3_code, b""); - return; - } - - // A queued write is not delivery: tearing down QUIC before the DATA and FIN - // are acknowledged can discard the capsule during loss recovery. Transport - // failure (including endpoint shutdown) also completes this wait. - match send.stopped().await { - Ok(None) => {} - Ok(Some(code)) => { - tracing::debug!(?code, "peer stopped the CONNECT stream during close"); - self.conn.close(http3_code, b""); - return; - } - Err(e) => { - tracing::debug!( - ?e, - "connection ended before the close capsule was acknowledged" - ); - return; - } - } - - // Wait for the peer to close the CONNECT stream after receiving the capsule. - if tokio::time::timeout(timeout, self.conn.closed()) - .await - .is_err() - { - tracing::debug!("timeout waiting for peer to close; force-closing connection"); - self.conn.close(http3_code, b""); - } - } - - /// Wait until the session is closed, returning the error. See [`noq::Connection::closed`]. - /// - /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be - /// [`WebTransportError::Closed`] with the code and reason from the capsule. - /// - /// Unlike [`noq::Connection::closed`], this does **not** return early when - /// [`close()`](Self::close) has been called. It waits for the underlying QUIC - /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has - /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. - pub async fn closed(&self) -> SessionError { - self.map_error(self.conn.closed().await) - } - - /// Return why the session was closed, or None if it's not closed. See - /// [`noq::Connection::close_reason`]. - pub fn close_reason(&self) -> Option { - self.conn.close_reason().map(|e| self.map_error(e)) - } - - /// Report an error caused by the connection closing as the session's close reason. - fn map_error(&self, e: impl Into) -> SessionError { - self.close.map(e.into()) - } - - async fn write_full(send: &mut noq::SendStream, buf: &[u8]) -> Result<(), SessionError> { - match send.write_all(buf).await { - Ok(_) => Ok(()), - Err(noq::WriteError::ConnectionLost(err)) => Err(err.into()), - Err(err) => Err(WebTransportError::WriteError(err).into()), - } - } - - /// Create a new session from a raw QUIC connection. - /// - /// This is used to pretend like a QUIC connection is a WebTransport session, - /// making it easier to support WebTransport and raw QUIC simultaneously. - /// - /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both - /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC - /// handshake, read from `conn` on demand — a connection that is still handshaking - /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. - pub fn raw(conn: noq::Connection) -> Self { - Self { - conn, - session_id: None, - header_uni: Default::default(), - header_bi: Default::default(), - header_datagram: Default::default(), - accept: None, - settings: None, - connect_send: Arc::new(Mutex::new(None)), - close: Arc::new(CloseReason::new(true)), - request: None, - response: None, - alpn: Default::default(), - } - } - - /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn request(&self) -> Option<&ConnectRequest> { - self.request.as_ref() - } - - /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, - /// or `None` for a raw QUIC session. - pub fn response(&self) -> Option<&ConnectResponse> { - self.response.as_ref() - } - - /// Returns the application protocol negotiated for this session. - /// - /// For an HTTP/3 session this is the subprotocol the server selected via - /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. - /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw - /// connection is still handshaking. - pub fn protocol(&self) -> Option<&str> { - if let Some(response) = &self.response { - return response.protocol.as_deref(); - } - - if let Some(alpn) = self.alpn.get() { - return Some(alpn); - } - - // Latch only a successful read, so a call made while the connection is still - // handshaking cannot pin `None` for the life of the session. - let alpn = negotiated_alpn(&self.conn)?; - Some(self.alpn.get_or_init(|| alpn)) - } - - /// Return connection-level statistics. - pub fn stats(&self) -> SessionStats { - let path = self.conn.path_stats(noq::PathId::ZERO); - let rtt = self - .conn - .rtt(noq::PathId::ZERO) - .or_else(|| path.map(|p| p.rtt)); - SessionStats { - stats: self.conn.stats(), - path, - rtt, - } - } + pub(crate) fn new(conn: crate::quic::driver::Connection, settings: Settings, connect: Connected) -> Self { + // The session ID is the stream ID of the CONNECT request. + let session_id = connect.session_id(); + + // Cache the tiny header we write in front of each stream we open. + let mut header_uni = Vec::new(); + StreamUni::WEBTRANSPORT.encode(&mut header_uni); + session_id.encode(&mut header_uni); + + let mut header_bi = Vec::new(); + Frame::WEBTRANSPORT.encode(&mut header_bi); + session_id.encode(&mut header_bi); + + let mut header_datagram = Vec::new(); + session_id.encode(&mut header_datagram); + + let close = Arc::new(CloseReason::new(false)); + + // Accept logic is stateful, so use an Arc to share it. + let accept = SessionAccept::new(conn.clone(), session_id, close.clone()); + + let this = Self { + conn, + accept: Some(Arc::new(Mutex::new(accept))), + session_id: Some(session_id), + header_uni, + header_bi, + header_datagram, + settings: Some(Arc::new(settings)), + connect_send: Arc::new(Mutex::new(Some(connect.send))), + close: close.clone(), + request: Some(connect.request.clone()), + response: Some(connect.response.clone()), + alpn: Default::default(), + }; + + // Run a background task to read capsules from the CONNECT recv stream. + let conn2 = this.conn.clone(); + tokio::spawn(Self::run_recv(conn2, connect.recv, close)); + + this + } + + // Read capsules from the CONNECT recv stream until it's closed, + // then record the close error and tear down the connection. + async fn run_recv( + conn: crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + close: Arc, + ) { + let close_info = Self::read_capsules(recv).await; + let code = close_info.as_ref().map_or(0, |(c, _)| *c); + + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Try to record the remote close error. If close() already set + // the error, it owns the connection teardown, so we bail out. + match close_info { + Some((code, reason)) => { + let err = WebTransportError::Closed(code, reason.clone()); + if !close.set(err.into()) { + return; + } + conn.close(http3_code, reason.as_bytes()); + } + None => { + let err = crate::quic::driver::ConnectionError::LocallyClosed.into(); + if !close.set(err) { + return; + } + conn.close(http3_code, b""); + } + }; + } + + // Keep reading capsules from the CONNECT recv stream until it's closed. + // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, + // or None if the stream closed without a capsule. + async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); + loop { + match reader.read().await { + Ok(Some(web_transport_proto::Capsule::CloseWebTransportSession { code, reason })) => { + return Some((code, reason)); + } + Ok(Some(web_transport_proto::Capsule::Grease { .. })) => {} + Ok(Some(web_transport_proto::Capsule::Unknown { typ, payload })) => { + tracing::warn!(%typ, size = payload.len(), "unknown capsule"); + } + Ok(None) => return None, + Err(e) => { + tracing::warn!(?e, "failed to read capsule"); + return None; + } + } + } + } + + /// Connect using an established QUIC connection if you want to create the connection yourself. + /// This will only work with a brand new QUIC connection using the HTTP/3 ALPN. + pub async fn connect( + conn: crate::quic::driver::Connection, + request: impl Into, + ) -> Result { + let request = request.into(); + + // Perform the H3 handshake by sending/receiving SETTINGS frames. + let settings = Settings::connect(&conn).await?; + + // Send the HTTP/3 CONNECT request. + let connect = Connected::open(&conn, request).await?; + + // Return the resulting session with a reference to the control/connect streams. + // If either stream is closed, then the session will be closed, so we need to keep them + // around. + let session = Session::new(conn, settings, connect); + + Ok(session) + } + + /// Accept a new unidirectional stream. See [`crate::quic::driver::Connection::accept_uni`]. + pub async fn accept_uni(&self) -> Result { + if let Some(accept) = &self.accept { + // `kio::wait` owns the waiter, so dropping this future — a `timeout` that + // expires, say — also drops its registration in `SessionAccept`. + kio::wait(|waiter| poll_accept_uni_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let recv = self.conn.accept_uni().await.map_err(|e| self.map_error(e))?; + Ok(RecvStream::new(recv, self.close.clone())) + } + } + + /// Accept a new bidirectional stream. See [`crate::quic::driver::Connection::accept_bi`]. + pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + if let Some(accept) = &self.accept { + kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) + .await + .map_err(|e| self.map_error(e)) + } else { + let (send, recv) = self.conn.accept_bi().await.map_err(|e| self.map_error(e))?; + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + } + + /// Open a new unidirectional stream. See [`crate::quic::driver::Connection::open_uni`]. + pub async fn open_uni(&self) -> Result { + let mut send = self.conn.open_uni().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_uni) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(SendStream::new(send, self.close.clone())) + } + + /// Open a new bidirectional stream. See [`crate::quic::driver::Connection::open_bi`]. + pub async fn open_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { + let (mut send, recv) = self.conn.open_bi().await.map_err(|e| self.map_error(e))?; + + // Set the stream priority to max and then write the stream header. + // Otherwise the application could write data with lower priority than the header, resulting + // in queuing. Also the header is very important for determining the session ID + // without reliable reset. + send.set_priority(i32::MAX).ok(); + Self::write_full(&mut send, &self.header_bi) + .await + .map_err(|e| self.map_error(e))?; + + // Reset the stream priority back to the default of 0. + send.set_priority(0).ok(); + Ok(( + SendStream::new(send, self.close.clone()), + RecvStream::new(recv, self.close.clone()), + )) + } + + /// Asynchronously receives an application datagram from the remote peer. + /// + /// This method is used to receive an application datagram sent by the remote + /// peer over the connection. + /// It waits for a datagram to become available and returns the received bytes. + pub async fn read_datagram(&self) -> Result { + let mut datagram = self.conn.read_datagram().await.map_err(|e| self.map_error(e))?; + + let mut cursor = Cursor::new(&datagram); + + if let Some(session_id) = self.session_id { + // We have to check and strip the session ID from the datagram. + let actual_id = VarInt::decode(&mut cursor).map_err(|_| WebTransportError::UnknownSession)?; + if actual_id != session_id { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // Return the datagram without the session ID. + let datagram = datagram.split_off(cursor.position() as usize); + + Ok(datagram) + } + + /// Sends an application datagram to the remote peer. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub fn send_datagram(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram(buf.into()) + } else { + self.conn.send_datagram(data) + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Sends an application datagram to the remote peer, waiting for buffer space if needed. + /// + /// Datagrams are unreliable and may be dropped or delivered out of order. + /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). + pub async fn send_datagram_wait(&self, data: Bytes) -> Result<(), SessionError> { + let result = if !self.header_datagram.is_empty() { + // Unfortunately, we need to allocate/copy each datagram because of the driver API. + // Pls go +1 if you care: https://github.com/quinn-rs/quinn/issues/1724 + let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); + + // Prepend the datagram with the header indicating the session ID. + buf.extend_from_slice(&self.header_datagram); + buf.extend_from_slice(&data); + + self.conn.send_datagram_wait(buf.into()).await + } else { + self.conn.send_datagram_wait(data).await + }; + + result.map_err(|e| self.map_error(e))?; + Ok(()) + } + + /// Computes the maximum size of datagrams that may be passed to + /// [`send_datagram`](Self::send_datagram), or 0 when the peer does not accept datagrams. + pub fn max_datagram_size(&self) -> usize { + self.conn + .max_datagram_size() + .unwrap_or(0) + .saturating_sub(self.header_datagram.len()) + } + + /// Returns the available buffer space for sending datagrams. + /// + /// This value is adjusted to account for the WebTransport session header. + pub fn datagram_send_buffer_space(&self) -> usize { + self.conn + .datagram_send_buffer_space() + .saturating_sub(self.header_datagram.len()) + } + + /// Close the session with an error code and reason. + /// + /// When there is a session ID (WebTransport over HTTP/3), a `CloseWebTransportSession` + /// capsule is written on the CONNECT stream before the QUIC connection is closed. + /// This allows browser clients to receive the close code and reason via `WebTransport.closed`. + /// + /// The capsule write and connection close happen asynchronously in a spawned task + /// that keeps the session alive until then, so the caller may drop it right away. + /// `await` [`Session::closed()`] to learn when the connection is gone. Session + /// operations will fail once the QUIC connection is closed. + pub fn close(&self, code: u32, reason: &[u8]) { + // First close wins: a no-op if the connection already closed, the background + // task recorded a remote close, or close() was already called. + if let Some(err) = self.conn.close_reason() { + self.close.set(self.map_error(err)); + return; + } + let err = SessionError::ConnectionError(crate::quic::driver::ConnectionError::LocallyClosed); + if !self.close.set(err) { + return; + } + + if self.session_id.is_some() { + // Take the send stream for the capsule write. + let send = self.connect_send.lock().unwrap().take(); + + if let Some(send) = send { + let reason = String::from_utf8_lossy(reason).into_owned(); + let capsule = web_transport_proto::Capsule::CloseWebTransportSession { code, reason }; + let timeout = (self.conn.rtt() * 3).max(Duration::from_millis(100)); + + // The task owns a clone of the whole session, not just the connection: + // dropping the caller's last handle would otherwise end the HTTP/3 + // control and QPACK streams before the capsule is delivered, which a + // browser treats as a connection error, losing the code and reason. + let session = self.clone(); + tokio::spawn(async move { + session.close_with_capsule(send, capsule, code, timeout).await; + }); + } + } else { + // Raw QUIC mode: no capsule needed. + self.conn.close(code.into(), reason); + } + } + + /// Deliver the close capsule and FIN, then give the peer its teardown grace. + async fn close_with_capsule( + &self, + mut send: crate::quic::driver::SendStream, + capsule: web_transport_proto::Capsule, + code: u32, + timeout: std::time::Duration, + ) { + let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); + + // Encode the capsule, then wrap it in an HTTP/3 DATA frame. + // In HTTP/3, capsule data is carried inside DATA frames on the CONNECT + // stream (RFC 9297 Section 3.2). + let mut capsule_bytes = Vec::new(); + capsule.encode(&mut capsule_bytes); + + let mut frame = Vec::new(); + Frame::DATA.encode(&mut frame); + let Ok(len) = VarInt::try_from(capsule_bytes.len()) else { + tracing::warn!("capsule too large to encode as DATA frame"); + self.conn.close(http3_code, b""); + return; + }; + len.encode(&mut frame); + frame.extend_from_slice(&capsule_bytes); + + // Write the DATA frame to the CONNECT send stream. + if let Err(e) = send.write_all(&frame).await { + tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); + self.conn.close(http3_code, b""); + return; + } + + // FIN the send stream so the peer knows no more capsules are coming. + if let Err(e) = send.finish() { + tracing::warn!(?e, "failed to finish CONNECT send stream"); + self.conn.close(http3_code, b""); + return; + } + + // A queued write is not delivery: tearing down QUIC before the DATA and FIN + // are acknowledged can discard the capsule during loss recovery. Transport + // failure (including endpoint shutdown) also completes this wait. + match send.stopped().await { + Ok(None) => {} + Ok(Some(code)) => { + tracing::debug!(?code, "peer stopped the CONNECT stream during close"); + self.conn.close(http3_code, b""); + return; + } + Err(e) => { + tracing::debug!(?e, "connection ended before the close capsule was acknowledged"); + return; + } + } + + // Wait for the peer to close the CONNECT stream after receiving the capsule. + if tokio::time::timeout(timeout, self.conn.closed()).await.is_err() { + tracing::debug!("timeout waiting for peer to close; force-closing connection"); + self.conn.close(http3_code, b""); + } + } + + /// Wait until the session is closed, returning the error. See [`crate::quic::driver::Connection::closed`]. + /// + /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be + /// [`WebTransportError::Closed`] with the code and reason from the capsule. + /// + /// Unlike [`crate::quic::driver::Connection::closed`], this does **not** return early when + /// [`close()`](Self::close) has been called. It waits for the underlying QUIC + /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has + /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. + pub async fn closed(&self) -> SessionError { + self.map_error(self.conn.closed().await) + } + + /// Return why the session was closed, or None if it's not closed. See + /// [`crate::quic::driver::Connection::close_reason`]. + pub fn close_reason(&self) -> Option { + self.conn.close_reason().map(|e| self.map_error(e)) + } + + /// Report an error caused by the connection closing as the session's close reason. + fn map_error(&self, e: impl Into) -> SessionError { + self.close.map(e.into()) + } + + async fn write_full(send: &mut crate::quic::driver::SendStream, buf: &[u8]) -> Result<(), SessionError> { + match send.write_all(buf).await { + Ok(_) => Ok(()), + Err(crate::quic::driver::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(err) => Err(WebTransportError::WriteError(err).into()), + } + } + + /// Create a new session from a raw QUIC connection. + /// + /// This is used to pretend like a QUIC connection is a WebTransport session, + /// making it easier to support WebTransport and raw QUIC simultaneously. + /// + /// There is no HTTP/3 exchange, so [`Self::request`] and [`Self::response`] both + /// return `None`. [`Self::protocol`] reports the ALPN negotiated by the QUIC + /// handshake, read from `conn` on demand — a connection that is still handshaking + /// (from `into_0rtt`, say) is fine, it just has no ALPN to report until it is done. + pub fn raw(conn: crate::quic::driver::Connection) -> Self { + Self { + conn, + session_id: None, + header_uni: Default::default(), + header_bi: Default::default(), + header_datagram: Default::default(), + accept: None, + settings: None, + connect_send: Arc::new(Mutex::new(None)), + close: Arc::new(CloseReason::new(true)), + request: None, + response: None, + alpn: Default::default(), + } + } + + /// Returns the [`ConnectRequest`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn request(&self) -> Option<&ConnectRequest> { + self.request.as_ref() + } + + /// Returns the [`ConnectResponse`] if this session was established over HTTP/3, + /// or `None` for a raw QUIC session. + pub fn response(&self) -> Option<&ConnectResponse> { + self.response.as_ref() + } + + /// Returns the application protocol negotiated for this session. + /// + /// For an HTTP/3 session this is the subprotocol the server selected via + /// `WT-Available-Protocols`; for a raw QUIC session it is the negotiated ALPN. + /// `None` if neither was negotiated, the ALPN is not valid UTF-8, or the raw + /// connection is still handshaking. + pub fn protocol(&self) -> Option<&str> { + if let Some(response) = &self.response { + return response.protocol.as_deref(); + } + + if let Some(alpn) = self.alpn.get() { + return Some(alpn); + } + + // Latch only a successful read, so a call made while the connection is still + // handshaking cannot pin `None` for the life of the session. + let alpn = negotiated_alpn(&self.conn)?; + Some(self.alpn.get_or_init(|| alpn)) + } + + /// Return connection-level statistics. + pub fn stats(&self) -> SessionStats { + SessionStats(self.conn.stats()) + } } impl Deref for Session { - type Target = noq::Connection; + type Target = crate::quic::driver::Connection; - fn deref(&self) -> &Self::Target { - &self.conn - } + fn deref(&self) -> &Self::Target { + &self.conn + } } impl fmt::Debug for Session { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.conn.fmt(f) - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.conn.fmt(f) + } } impl PartialEq for Session { - fn eq(&self, other: &Self) -> bool { - self.conn.stable_id() == other.conn.stable_id() - } + fn eq(&self, other: &Self) -> bool { + self.conn.stable_id() == other.conn.stable_id() + } } impl Eq for Session {} @@ -645,687 +615,656 @@ impl Eq for Session {} // inline, and the first thing a resumed accepter does is take this same lock. An // arrival or a failure is exactly what the others are parked waiting to retry // after, so `Ready` is the signal. -fn poll_accept_uni_shared( - accept: &Mutex, - waiter: &Waiter, -) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.uni_waiters.clone(); - - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_uni(waiter); - - (result, waiters, hold) - }; - - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); - - if result.is_ready() { - waiters.wake(); - } - - result +fn poll_accept_uni_shared(accept: &Mutex, waiter: &Waiter) -> Poll> { + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.uni_waiters.clone(); + + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_uni(waiter); + + (result, waiters, hold) + }; + + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); + + if result.is_ready() { + waiters.wake(); + } + + result } fn poll_accept_bi_shared( - accept: &Mutex, - waiter: &Waiter, + accept: &Mutex, + waiter: &Waiter, ) -> Poll> { - let (result, waiters, hold) = { - let mut accept = accept.lock().unwrap(); - let waiters = accept.bi_waiters.clone(); + let (result, waiters, hold) = { + let mut accept = accept.lock().unwrap(); + let waiters = accept.bi_waiters.clone(); - // The poll below drives the shared accept futures with this list's waker, and one - // of them may wake it inline. `Fan` holds those back while this guard is alive. - let hold = waiters.hold(); - let result = accept.poll_accept_bi(waiter); + // The poll below drives the shared accept futures with this list's waker, and one + // of them may wake it inline. `Fan` holds those back while this guard is alive. + let hold = waiters.hold(); + let result = accept.poll_accept_bi(waiter); - (result, waiters, hold) - }; + (result, waiters, hold) + }; - // Dropped after the accept lock, never before: that is where a held-back wake is - // delivered, and delivering it under the lock is the hazard the hold exists for. - drop(hold); + // Dropped after the accept lock, never before: that is where a held-back wake is + // delivered, and delivering it under the lock is the hazard the hold exists for. + drop(hold); - if result.is_ready() { - waiters.wake(); - } + if result.is_ready() { + waiters.wake(); + } - result + result } // Type aliases just so clippy doesn't complain about the complexity. -type AcceptUni = dyn Stream> + Send; -type AcceptBi = - dyn Stream> + Send; -type PendingUni = dyn Future> + Send; -type PendingBi = - dyn Future, SessionError>> + Send; +type AcceptUni = + dyn Stream> + Send; +type AcceptBi = dyn Stream< + Item = Result< + (crate::quic::driver::SendStream, crate::quic::driver::RecvStream), + crate::quic::driver::ConnectionError, + >, + > + Send; +type PendingUni = dyn Future> + Send; +type PendingBi = dyn Future, SessionError>> + + Send; // Logic just for accepting streams, which is annoying because of the stream header. pub struct SessionAccept { - session_id: VarInt, - - // Shared close reason for propagation to accepted streams. - close: Arc, - - // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them. Again, this is just so they don't get closed until we drop the session. - qpack_encoder: Option, - qpack_decoder: Option, - - accept_uni: Pin>, - accept_bi: Pin>, - - // Keep track of work being done to read/write the WebTransport stream header. - pending_uni: FuturesUnordered>>, - pending_bi: FuturesUnordered>>, - - // Waiters from concurrent callers of accept_bi / accept_uni. - // Every clone of the session polls this one struct, so an arrival has to be fanned - // out: each caller registers here and all of them are woken when a stream lands — - // by the caller that saw it, once it has released the lock on this struct. - bi_waiters: Fan, - uni_waiters: Fan, - - // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with - // the same waker every time. That waker outlives every caller, so an accepter that - // drops its future cannot take the wakeup path with it, and the layer below holds - // one registration rather than one per caller. - bi_waker: Waker, - uni_waker: Waker, + session_id: VarInt, + + // Shared close reason for propagation to accepted streams. + close: Arc, + + // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them. Again, this is just so they don't get closed until we drop the session. + qpack_encoder: Option, + qpack_decoder: Option, + + accept_uni: Pin>, + accept_bi: Pin>, + + // Keep track of work being done to read/write the WebTransport stream header. + pending_uni: FuturesUnordered>>, + pending_bi: FuturesUnordered>>, + + // Waiters from concurrent callers of accept_bi / accept_uni. + // Every clone of the session polls this one struct, so an arrival has to be fanned + // out: each caller registers here and all of them are woken when a stream lands — + // by the caller that saw it, once it has released the lock on this struct. + bi_waiters: Fan, + uni_waiters: Fan, + + // `Waker::from(waiters.clone())`, cached so the inner accept futures are polled with + // the same waker every time. That waker outlives every caller, so an accepter that + // drops its future cannot take the wakeup path with it, and the layer below holds + // one registration rather than one per caller. + bi_waker: Waker, + uni_waker: Waker, } impl SessionAccept { - pub(crate) fn new(conn: noq::Connection, session_id: VarInt, close: Arc) -> Self { - // Create a stream that just outputs new streams, so it's easy to call from poll. - let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { - Some((conn.accept_uni().await, conn)) - })); - - let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { - Some((conn.accept_bi().await, conn)) - })); - - let bi_waiters = Fan::new(); - let uni_waiters = Fan::new(); - let bi_waker = bi_waiters.waker(); - let uni_waker = uni_waiters.waker(); - - Self { - session_id, - close, - - qpack_decoder: None, - qpack_encoder: None, - - accept_uni, - accept_bi, - - pending_uni: FuturesUnordered::new(), - pending_bi: FuturesUnordered::new(), - - bi_waiters, - uni_waiters, - bi_waker, - uni_waker, - } - } - - /// Poll for the next unidirectional WebTransport stream. - /// - /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops - /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive - /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this - /// with [`kio::wait`], which holds the waiter inside the future it builds. - /// - /// A `Ready` here means every *other* parked accepter should be woken so it can - /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes - /// them once the lock on this struct is released. - // - // Poll-based because we accept and decode streams in parallel. In async land this - // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is - // runtime-agnostic. - pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { - // Register before polling, not on the way out: the shared waker can fire from the - // layer below at any point here, and a wake that lands before the caller is on - // the list would be lost. - self.uni_waiters.register(waiter); - - let waker = self.uni_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let recv = match res { - Ok(recv) => recv, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_uni(recv, self.session_id); - self.pending_uni.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "unidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - // Decide if we keep looping based on the type. - match typ { - StreamUni::WEBTRANSPORT => { - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok(recv)); - } - StreamUni::QPACK_DECODER => { - self.qpack_decoder = Some(recv); - } - StreamUni::QPACK_ENCODER => { - self.qpack_encoder = Some(recv); - } - _ => { - // ignore unknown streams - tracing::debug!(?typ, "ignoring unknown unidirectional stream"); - } - } - } - } - - // Reads the stream header, returning the stream type. - async fn decode_uni( - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result<(StreamUni, noq::RecvStream), SessionError> { - // Read the VarInt at the start of the stream. - let typ = StreamUni(read_varint(&mut recv).await?); - - if typ == StreamUni::WEBTRANSPORT { - // Read the session_id and validate it - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - } - - // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates - // them, so return everything. - Ok((typ, recv)) - } - - /// Poll for the next bidirectional WebTransport stream. - /// - /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` - /// registration is weak and owned by the caller, and a `Ready` is what the other - /// parked accepters need to be woken for. - pub fn poll_accept_bi( - &mut self, - waiter: &Waiter, - ) -> Poll> { - // Register before polling; see `poll_accept_uni`. - self.bi_waiters.register(waiter); - - let waker = self.bi_waker.clone(); - let cx = &mut Context::from_waker(&waker); - - loop { - // Accept any new streams. - if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { - // Start decoding the header and add the future to the list of pending streams. - let (send, recv) = match res { - Ok(pair) => pair, - Err(e) => { - return Poll::Ready(Err(e.into())); - } - }; - let pending = Self::decode_bi(send, recv, self.session_id); - self.pending_bi.push(Box::pin(pending)); - - continue; - } - - // Poll the list of pending streams. - let res = match self.pending_bi.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(res))) => res, - Poll::Ready(Some(Err(err))) => { - log_header_error(err, "bidirectional"); - continue; - } - Poll::Ready(None) | Poll::Pending => return Poll::Pending, - }; - - if let Some((send, recv)) = res { - // Wrap the streams in our own types for correct error codes. - let send = SendStream::new(send, self.close.clone()); - let recv = RecvStream::new(recv, self.close.clone()); - return Poll::Ready(Ok((send, recv))); - } - - // Keep looping if it's a stream we want to ignore. - } - } - - // Reads the stream header, returning Some if it's a WebTransport stream. - async fn decode_bi( - send: noq::SendStream, - mut recv: noq::RecvStream, - expected_session: VarInt, - ) -> Result, SessionError> { - let typ = read_varint(&mut recv).await?; - if Frame(typ) != Frame::WEBTRANSPORT { - tracing::debug!(?typ, "ignoring unknown bidirectional stream"); - return Ok(None); - } - - // Read the session ID and validate it. - let session_id = read_varint(&mut recv).await?; - if session_id != expected_session { - return Err(WebTransportError::UnknownSession.into()); - } - - Ok(Some((send, recv))) - } + pub(crate) fn new(conn: crate::quic::driver::Connection, session_id: VarInt, close: Arc) -> Self { + // Create a stream that just outputs new streams, so it's easy to call from poll. + let accept_uni = Box::pin(futures::stream::unfold(conn.clone(), |conn| async { + Some((conn.accept_uni().await, conn)) + })); + + let accept_bi = Box::pin(futures::stream::unfold(conn, |conn| async { + Some((conn.accept_bi().await, conn)) + })); + + let bi_waiters = Fan::default(); + let uni_waiters = Fan::default(); + let bi_waker = bi_waiters.waker(); + let uni_waker = uni_waiters.waker(); + + Self { + session_id, + close, + + qpack_decoder: None, + qpack_encoder: None, + + accept_uni, + accept_bi, + + pending_uni: FuturesUnordered::new(), + pending_bi: FuturesUnordered::new(), + + bi_waiters, + uni_waiters, + bi_waker, + uni_waker, + } + } + + /// Poll for the next unidirectional WebTransport stream. + /// + /// `waiter` is parked until a stream arrives, the accept fails, or the caller drops + /// it. The registration is weak and owned by the caller: keep the [`Waiter`] alive + /// until it is woken, or it will be reclaimed and nothing will wake you. Drive this + /// with [`kio::wait`], which holds the waiter inside the future it builds. + /// + /// A `Ready` here means every *other* parked accepter should be woken so it can + /// retry. This does not do that itself — see `poll_accept_uni_shared`, which wakes + /// them once the lock on this struct is released. + // + // Poll-based because we accept and decode streams in parallel. In async land this + // would be a `tokio::JoinSet`, but that needs a runtime; `FuturesUnordered` is + // runtime-agnostic. + pub fn poll_accept_uni(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling, not on the way out: the shared waker can fire from the + // layer below at any point here, and a wake that lands before the caller is on + // the list would be lost. + self.uni_waiters.register(waiter); + + let waker = self.uni_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_uni.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let recv = match res { + Ok(recv) => recv, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_uni(recv, self.session_id); + self.pending_uni.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let (typ, recv) = match self.pending_uni.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "unidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + // Decide if we keep looping based on the type. + match typ { + StreamUni::WEBTRANSPORT => { + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok(recv)); + } + StreamUni::QPACK_DECODER => { + self.qpack_decoder = Some(recv); + } + StreamUni::QPACK_ENCODER => { + self.qpack_encoder = Some(recv); + } + _ => { + // ignore unknown streams + tracing::debug!(?typ, "ignoring unknown unidirectional stream"); + } + } + } + } + + // Reads the stream header, returning the stream type. + async fn decode_uni( + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result<(StreamUni, crate::quic::driver::RecvStream), SessionError> { + // Read the VarInt at the start of the stream. + let typ = StreamUni(read_varint(&mut recv).await?); + + if typ == StreamUni::WEBTRANSPORT { + // Read the session_id and validate it + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + } + + // We need to keep a reference to the qpack streams if the endpoint (incorrectly) creates + // them, so return everything. + Ok((typ, recv)) + } + + /// Poll for the next bidirectional WebTransport stream. + /// + /// The same contract as [`poll_accept_uni`](Self::poll_accept_uni): the `waiter` + /// registration is weak and owned by the caller, and a `Ready` is what the other + /// parked accepters need to be woken for. + pub fn poll_accept_bi(&mut self, waiter: &Waiter) -> Poll> { + // Register before polling; see `poll_accept_uni`. + self.bi_waiters.register(waiter); + + let waker = self.bi_waker.clone(); + let cx = &mut Context::from_waker(&waker); + + loop { + // Accept any new streams. + if let Poll::Ready(Some(res)) = self.accept_bi.poll_next_unpin(cx) { + // Start decoding the header and add the future to the list of pending streams. + let (send, recv) = match res { + Ok(pair) => pair, + Err(e) => { + return Poll::Ready(Err(e.into())); + } + }; + let pending = Self::decode_bi(send, recv, self.session_id); + self.pending_bi.push(Box::pin(pending)); + + continue; + } + + // Poll the list of pending streams. + let res = match self.pending_bi.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(res))) => res, + Poll::Ready(Some(Err(err))) => { + log_header_error(err, "bidirectional"); + continue; + } + Poll::Ready(None) | Poll::Pending => return Poll::Pending, + }; + + if let Some((send, recv)) = res { + // Wrap the streams in our own types for correct error codes. + let send = SendStream::new(send, self.close.clone()); + let recv = RecvStream::new(recv, self.close.clone()); + return Poll::Ready(Ok((send, recv))); + } + + // Keep looping if it's a stream we want to ignore. + } + } + + // Reads the stream header, returning Some if it's a WebTransport stream. + async fn decode_bi( + send: crate::quic::driver::SendStream, + mut recv: crate::quic::driver::RecvStream, + expected_session: VarInt, + ) -> Result, SessionError> { + let typ = read_varint(&mut recv).await?; + if Frame(typ) != Frame::WEBTRANSPORT { + tracing::debug!(?typ, "ignoring unknown bidirectional stream"); + return Ok(None); + } + + // Read the session ID and validate it. + let session_id = read_varint(&mut recv).await?; + if session_id != expected_session { + return Err(WebTransportError::UnknownSession.into()); + } + + Ok(Some((send, recv))) + } } // Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any // failure as a truncation, which would hide a stream the peer reset before its header // arrived. -async fn read_varint(recv: &mut noq::RecvStream) -> Result { - let mut buf = [0u8; 8]; - recv.read_exact(&mut buf[..1]).await?; +async fn read_varint(recv: &mut crate::quic::driver::RecvStream) -> Result { + let mut buf = [0u8; 8]; + recv.read_exact(&mut buf[..1]).await?; - // The first two bits encode the length. - let size = 1 << (buf[0] >> 6); - recv.read_exact(&mut buf[1..size]).await?; + // The first two bits encode the length. + let size = 1 << (buf[0] >> 6); + recv.read_exact(&mut buf[1..size]).await?; - Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) + Ok(VarInt::decode(&mut &buf[..size]).expect("a complete varint")) } // Without reliable reset, a stream the peer resets early loses its header with it, which // moq does routinely. Anything else is a misbehaving peer. fn log_header_error(err: SessionError, direction: &'static str) { - match err { - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError( - noq::ReadError::Reset(_) | noq::ReadError::ConnectionLost(_), - ), - )) => tracing::debug!(?err, direction, "stream closed before its header"), - _ => tracing::warn!(?err, direction, "failed to decode stream header"), - } + match err { + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError( + crate::quic::driver::ReadError::Reset(_) | crate::quic::driver::ReadError::ConnectionLost(_), + ), + )) => tracing::debug!(?err, direction, "stream closed before its header"), + _ => tracing::warn!(?err, direction, "failed to decode stream header"), + } } -pub struct SessionStats { - stats: noq::ConnectionStats, - path: Option, - rtt: Option, -} +pub struct SessionStats(crate::quic::driver::ConnectionStats); impl web_transport_trait::Stats for SessionStats { - fn bytes_sent(&self) -> Option { - Some(self.stats.udp_tx.bytes) - } - - fn bytes_received(&self) -> Option { - Some(self.stats.udp_rx.bytes) - } - - fn bytes_lost(&self) -> Option { - self.path.map(|path| path.lost_bytes) - } - - fn packets_sent(&self) -> Option { - Some(self.stats.udp_tx.datagrams) - } - - fn packets_received(&self) -> Option { - Some(self.stats.udp_rx.datagrams) - } - - fn packets_lost(&self) -> Option { - self.path.map(|path| path.lost_packets) - } - - fn rtt(&self) -> Option { - self.rtt - } - - fn estimated_send_rate(&self) -> Option { - let path = self.path?; - let rtt = self.rtt?; - let rtt_secs = rtt.as_secs_f64(); - if path.cwnd > 0 && rtt_secs > 0.0 { - Some((path.cwnd as f64 * 8.0 / rtt_secs) as u64) - } else { - None - } - } + fn bytes_sent(&self) -> Option { + Some(self.0.udp_tx.bytes) + } + + fn bytes_received(&self) -> Option { + Some(self.0.udp_rx.bytes) + } + + fn bytes_lost(&self) -> Option { + Some(self.0.path.lost_bytes) + } + + fn packets_sent(&self) -> Option { + Some(self.0.udp_tx.datagrams) + } + + fn packets_received(&self) -> Option { + Some(self.0.udp_rx.datagrams) + } + + fn packets_lost(&self) -> Option { + Some(self.0.path.lost_packets) + } + + fn rtt(&self) -> Option { + Some(self.0.path.rtt) + } + + /// The congestion controller's bandwidth estimate, in bits per second. + fn estimated_send_rate(&self) -> Option { + self.0.path.bandwidth_estimate.map(|bytes| bytes.saturating_mul(8)) + } } impl web_transport_trait::Session for Session { - type SendStream = SendStream; - type RecvStream = RecvStream; - type Error = SessionError; - - async fn accept_uni(&self) -> Result { - Self::accept_uni(self).await - } - - async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::accept_bi(self).await - } - - async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::open_bi(self).await - } - - async fn open_uni(&self) -> Result { - Self::open_uni(self).await - } - - fn close(&self, code: u32, reason: &str) { - Self::close(self, code, reason.as_bytes()); - } - - async fn closed(&self) -> Self::Error { - Self::closed(self).await - } - - fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { - Self::send_datagram(self, data) - } - - async fn recv_datagram(&self) -> Result { - Self::read_datagram(self).await - } - - fn max_datagram_size(&self) -> usize { - Self::max_datagram_size(self) - } - - fn protocol(&self) -> Option<&str> { - Self::protocol(self) - } - - #[allow(refining_impl_trait)] - fn stats(&self) -> SessionStats { - Self::stats(self) - } + type SendStream = SendStream; + type RecvStream = RecvStream; + type Error = SessionError; + + async fn accept_uni(&self) -> Result { + Self::accept_uni(self).await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::accept_bi(self).await + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + Self::open_bi(self).await + } + + async fn open_uni(&self) -> Result { + Self::open_uni(self).await + } + + fn close(&self, code: u32, reason: &str) { + Self::close(self, code, reason.as_bytes()); + } + + async fn closed(&self) -> Self::Error { + Self::closed(self).await + } + + fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { + Self::send_datagram(self, data) + } + + async fn recv_datagram(&self) -> Result { + Self::read_datagram(self).await + } + + fn max_datagram_size(&self) -> usize { + Self::max_datagram_size(self) + } + + fn protocol(&self) -> Option<&str> { + Self::protocol(self) + } + + #[allow(refining_impl_trait)] + fn stats(&self) -> SessionStats { + Self::stats(self) + } } #[cfg(all(test, any(feature = "aws-lc-rs", feature = "ring")))] mod tests { - use std::{ - io::{self, IoSliceMut}, - net::{Ipv4Addr, SocketAddr}, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - }; - - use rcgen::{CertifiedKey, KeyPair}; - use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; - - use super::*; - - const SESSION: VarInt = VarInt::from_u32(0); - - #[derive(Debug, Default)] - struct Gate { - blocked: AtomicBool, - attempted: tokio::sync::Notify, - wakers: Mutex>, - } - - #[derive(Debug)] - struct Socket { - inner: Box, - gate: Arc, - } - - #[derive(Debug)] - struct Sender { - inner: Pin>, - gate: Arc, - } - - impl noq::UdpSender for Sender { - fn poll_send( - self: Pin<&mut Self>, - transmit: &noq::udp::Transmit<'_>, - cx: &mut Context<'_>, - ) -> Poll> { - let this = self.get_mut(); - let mut wakers = this.gate.wakers.lock().unwrap(); - if this.gate.blocked.load(Ordering::SeqCst) { - if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { - wakers.push(cx.waker().clone()); - } - this.gate.attempted.notify_one(); - return Poll::Pending; - } - drop(wakers); - this.inner.as_mut().poll_send(transmit, cx) - } - } - - impl noq::AsyncUdpSocket for Socket { - fn create_sender(&self) -> Pin> { - Box::pin(Sender { - inner: self.inner.create_sender(), - gate: self.gate.clone(), - }) - } - fn poll_recv( - &mut self, - cx: &mut Context<'_>, - bufs: &mut [IoSliceMut<'_>], - meta: &mut [noq::udp::RecvMeta], - ) -> Poll> { - self.inner.poll_recv(cx, bufs, meta) - } - fn local_addr(&self) -> io::Result { - self.inner.local_addr() - } - } - - /// A connected client and server QUIC connection, plus the endpoints that drive them. - async fn pair() -> (noq::Connection, noq::Connection, [noq::Endpoint; 2]) { - let (client, server, endpoints, _) = gated_pair().await; - (client, server, endpoints) - } - - async fn gated_pair() -> ( - noq::Connection, - noq::Connection, - [noq::Endpoint; 2], - Arc, - ) { - #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - - let CertifiedKey { cert, signing_key } = - rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); - let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der( - &signing_key, - ))); - - let server_config = - noq::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); - let runtime: Arc = Arc::new(noq::TokioRuntime); - let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); - let gate = Arc::new(Gate::default()); - let socket = Socket { - inner: runtime.wrap_udp_socket(udp).unwrap(), - gate: gate.clone(), - }; - let server = noq::Endpoint::new_with_abstract_socket( - Default::default(), - Some(server_config), - Box::new(socket), - runtime, - ) - .unwrap(); - - let mut roots = rustls::RootCertStore::empty(); - roots.add(cert.der().clone()).unwrap(); - let client_config = noq::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); - let client = noq::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); - - let connecting = client - .connect_with(client_config, server.local_addr().unwrap(), "localhost") - .unwrap(); - let accept = async { server.accept().await.unwrap().await.unwrap() }; - let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); - - (client_conn, server_conn, [client, server], gate) - } - - /// A queued capsule must survive a stalled socket beyond the teardown grace. - #[tokio::test] - async fn close_waits_for_capsule_ack() { - close_with_stalled_socket(false).await; - } - - /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. - #[tokio::test] - async fn close_ack_wait_ends_with_connection() { - close_with_stalled_socket(true).await; - } - - async fn close_with_stalled_socket(cancel: bool) { - let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; - let (client, server) = tokio::join!( - Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), - async { - crate::Request::accept(server_conn) - .await - .unwrap() - .ok() - .await - }, - ); - let client = client.unwrap(); - let server = server.unwrap(); - let conn = server.conn.clone(); - let settings = Arc::downgrade(server.settings.as_ref().unwrap()); - tokio::time::pause(); - gate.blocked.store(true, Ordering::SeqCst); - server.close(2, b"unauthorized"); - drop(server); - gate.attempted.notified().await; - tokio::time::advance(Duration::from_secs(1)).await; - tokio::task::yield_now().await; - assert!( - conn.close_reason().is_none(), - "closed before the capsule was acknowledged" - ); - - if cancel { - conn.close(0u32.into(), b"drain deadline"); - conn.closed().await; - tokio::task::yield_now().await; - assert_eq!( - settings.strong_count(), - 0, - "close task retained the HTTP/3 streams after cancellation" - ); - return; - } - - gate.blocked.store(false, Ordering::SeqCst); - for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { - waker.wake(); - } - tokio::time::resume(); - let err = client.closed().await; - assert!( - matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), - "{err:?}" - ); - conn.closed().await; - } - - fn assert_reset(err: SessionError) { - assert!( - matches!( - &err, - SessionError::WebTransportError(WebTransportError::ReadError( - noq::ReadExactError::ReadError(noq::ReadError::Reset(code)) - )) if code.into_inner() == 7 - ), - "{err:?}" - ); - } - - /// A uni stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn uni_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - // The stream type alone, so the reset also cuts off the session ID. - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54]).await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A bi stream reset before its header arrived reports the reset, not a foreign session. - #[tokio::test] - async fn bi_reset_before_header() { - let (client, server, _endpoints) = pair().await; - - let (mut send, _recv) = client.open_bi().await.unwrap(); - send.reset(7u32.into()).unwrap(); - - let (send, recv) = server.accept_bi().await.unwrap(); - let err = SessionAccept::decode_bi(send, recv, SESSION) - .await - .err() - .unwrap(); - assert_reset(err); - } - - /// A stream that names another session is still reported as `UnknownSession`. - #[tokio::test] - async fn uni_other_session() { - let (client, server, _endpoints) = pair().await; - - let mut send = client.open_uni().await.unwrap(); - send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); - send.finish().unwrap(); - - let recv = server.accept_uni().await.unwrap(); - let err = SessionAccept::decode_uni(recv, SESSION) - .await - .err() - .unwrap(); - assert!( - matches!( - err, - SessionError::WebTransportError(WebTransportError::UnknownSession) - ), - "{err:?}" - ); - } + use std::{ + io::{self, IoSliceMut}, + net::{Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + }; + + use rcgen::{CertifiedKey, KeyPair}; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + use super::*; + + const SESSION: VarInt = VarInt::from_u32(0); + + #[derive(Debug, Default)] + struct Gate { + blocked: AtomicBool, + attempted: tokio::sync::Notify, + wakers: Mutex>, + } + + #[derive(Debug)] + struct Socket { + inner: Box, + gate: Arc, + } + + #[derive(Debug)] + struct Sender { + inner: Pin>, + gate: Arc, + } + + impl crate::quic::driver::UdpSender for Sender { + fn poll_send( + self: Pin<&mut Self>, + transmit: &moq_sock::udp::Transmit<'_>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + let mut wakers = this.gate.wakers.lock().unwrap(); + if this.gate.blocked.load(Ordering::SeqCst) { + if !wakers.iter().any(|waker| waker.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + this.gate.attempted.notify_one(); + return Poll::Pending; + } + drop(wakers); + this.inner.as_mut().poll_send(transmit, cx) + } + } + + impl crate::quic::driver::AsyncUdpSocket for Socket { + fn create_sender(&self) -> Pin> { + Box::pin(Sender { + inner: self.inner.create_sender(), + gate: self.gate.clone(), + }) + } + fn poll_recv( + &mut self, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [moq_sock::udp::RecvMeta], + ) -> Poll> { + self.inner.poll_recv(cx, bufs, meta) + } + fn local_addr(&self) -> io::Result { + self.inner.local_addr() + } + } + + /// A connected client and server QUIC connection, plus the endpoints that drive them. + async fn pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + ) { + let (client, server, endpoints, _) = gated_pair().await; + (client, server, endpoints) + } + + async fn gated_pair() -> ( + crate::quic::driver::Connection, + crate::quic::driver::Connection, + [crate::quic::driver::Endpoint; 2], + Arc, + ) { + #[cfg(all(feature = "aws-lc-rs", feature = "ring"))] + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(KeyPair::serialize_der(&signing_key))); + + let server_config = crate::quic::driver::ServerConfig::with_single_cert(vec![cert.der().clone()], key).unwrap(); + let udp = std::net::UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let gate = Arc::new(Gate::default()); + let socket = Socket { + inner: crate::quic::driver::wrap_udp_socket(udp).unwrap(), + gate: gate.clone(), + }; + let server = crate::quic::driver::Endpoint::new_with_abstract_socket( + Default::default(), + Some(server_config), + Box::new(socket), + ) + .unwrap(); + + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let client_config = crate::quic::driver::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); + let client = crate::quic::driver::Endpoint::client((Ipv4Addr::LOCALHOST, 0).into()).unwrap(); + + let connecting = client + .connect_with(client_config, server.local_addr().unwrap(), "localhost") + .unwrap(); + let accept = async { server.accept().await.unwrap().await.unwrap() }; + let (client_conn, server_conn) = tokio::join!(async { connecting.await.unwrap() }, accept); + + (client_conn, server_conn, [client, server], gate) + } + + /// A queued capsule must survive a stalled socket beyond the teardown grace. + #[tokio::test] + async fn close_waits_for_capsule_ack() { + close_with_stalled_socket(false).await; + } + + /// Endpoint/drain cancellation must release a task waiting for a capsule ACK. + #[tokio::test] + async fn close_ack_wait_ends_with_connection() { + close_with_stalled_socket(true).await; + } + + async fn close_with_stalled_socket(cancel: bool) { + let (client_conn, server_conn, _endpoints, gate) = gated_pair().await; + let (client, server) = tokio::join!( + Session::connect(client_conn, url::Url::parse("https://localhost/").unwrap()), + async { crate::quic::web::Request::accept(server_conn).await.unwrap().ok().await }, + ); + let client = client.unwrap(); + let server = server.unwrap(); + let conn = server.conn.clone(); + let settings = Arc::downgrade(server.settings.as_ref().unwrap()); + tokio::time::pause(); + gate.blocked.store(true, Ordering::SeqCst); + server.close(2, b"unauthorized"); + drop(server); + gate.attempted.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert!( + conn.close_reason().is_none(), + "closed before the capsule was acknowledged" + ); + + if cancel { + conn.close(0u32.into(), b"drain deadline"); + conn.closed().await; + tokio::task::yield_now().await; + assert_eq!( + settings.strong_count(), + 0, + "close task retained the HTTP/3 streams after cancellation" + ); + return; + } + + gate.blocked.store(false, Ordering::SeqCst); + for waker in std::mem::take(&mut *gate.wakers.lock().unwrap()) { + waker.wake(); + } + tokio::time::resume(); + let err = client.closed().await; + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::Closed(2, ref reason)) if reason == "unauthorized"), + "{err:?}" + ); + conn.closed().await; + } + + fn assert_reset(err: SessionError) { + assert!( + matches!( + &err, + SessionError::WebTransportError(WebTransportError::ReadError( + crate::quic::driver::ReadExactError::ReadError(crate::quic::driver::ReadError::Reset(code)) + )) if code.into_inner() == 7 + ), + "{err:?}" + ); + } + + /// A uni stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn uni_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + // The stream type alone, so the reset also cuts off the session ID. + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54]).await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A bi stream reset before its header arrived reports the reset, not a foreign session. + #[tokio::test] + async fn bi_reset_before_header() { + let (client, server, _endpoints) = pair().await; + + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.reset(7u32.into()).unwrap(); + + let (send, recv) = server.accept_bi().await.unwrap(); + let err = SessionAccept::decode_bi(send, recv, SESSION).await.err().unwrap(); + assert_reset(err); + } + + /// A stream that names another session is still reported as `UnknownSession`. + #[tokio::test] + async fn uni_other_session() { + let (client, server, _endpoints) = pair().await; + + let mut send = client.open_uni().await.unwrap(); + send.write_all(&[0x40, 0x54, 0x04]).await.unwrap(); + send.finish().unwrap(); + + let recv = server.accept_uni().await.unwrap(); + let err = SessionAccept::decode_uni(recv, SESSION).await.err().unwrap(); + assert!( + matches!(err, SessionError::WebTransportError(WebTransportError::UnknownSession)), + "{err:?}" + ); + } } diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index 08ea9a53a1..fdee95f23c 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -4,67 +4,67 @@ use thiserror::Error; #[derive(Error, Debug, Clone)] pub enum SettingsError { - #[error("quic stream was closed early")] - UnexpectedEnd, + #[error("quic stream was closed early")] + UnexpectedEnd, - #[error("protocol error: {0}")] - ProtoError(#[from] web_transport_proto::SettingsError), + #[error("protocol error: {0}")] + ProtoError(#[from] web_transport_proto::SettingsError), - #[error("WebTransport is not supported")] - WebTransportUnsupported, + #[error("WebTransport is not supported")] + WebTransportUnsupported, - #[error("connection error")] - ConnectionError(#[from] noq::ConnectionError), + #[error("connection error")] + ConnectionError(#[from] crate::quic::driver::ConnectionError), - #[error("read error")] - ReadError(#[from] noq::ReadError), + #[error("read error")] + ReadError(#[from] crate::quic::driver::ReadError), - #[error("write error")] - WriteError(#[from] noq::WriteError), + #[error("write error")] + WriteError(#[from] crate::quic::driver::WriteError), } pub struct Settings { - // A reference to the send/recv stream, so we don't close it until dropped. - #[allow(dead_code)] - send: noq::SendStream, + // A reference to the send/recv stream, so we don't close it until dropped. + #[allow(dead_code)] + send: crate::quic::driver::SendStream, - #[allow(dead_code)] - recv: noq::RecvStream, + #[allow(dead_code)] + recv: crate::quic::driver::RecvStream, } impl Settings { - // Establish the H3 connection. - pub async fn connect(conn: &noq::Connection) -> Result { - let recv = Self::accept(conn); - let send = Self::open(conn); + // Establish the H3 connection. + pub async fn connect(conn: &crate::quic::driver::Connection) -> Result { + let recv = Self::accept(conn); + let send = Self::open(conn); - // Run both tasks concurrently until one errors or they both complete. - let (send, recv) = try_join!(send, recv)?; - Ok(Self { send, recv }) - } + // Run both tasks concurrently until one errors or they both complete. + let (send, recv) = try_join!(send, recv)?; + Ok(Self { send, recv }) + } - async fn accept(conn: &noq::Connection) -> Result { - let mut recv = conn.accept_uni().await?; - let settings = web_transport_proto::Settings::read(&mut recv).await?; + async fn accept(conn: &crate::quic::driver::Connection) -> Result { + let mut recv = conn.accept_uni().await?; + let settings = web_transport_proto::Settings::read(&mut recv).await?; - tracing::debug!(?settings, "received SETTINGS frame"); + tracing::debug!(?settings, "received SETTINGS frame"); - if settings.supports_webtransport() == 0 { - return Err(SettingsError::WebTransportUnsupported); - } + if settings.supports_webtransport() == 0 { + return Err(SettingsError::WebTransportUnsupported); + } - Ok(recv) - } + Ok(recv) + } - async fn open(conn: &noq::Connection) -> Result { - let mut settings = web_transport_proto::Settings::default(); - settings.enable_webtransport(1); + async fn open(conn: &crate::quic::driver::Connection) -> Result { + let mut settings = web_transport_proto::Settings::default(); + settings.enable_webtransport(1); - tracing::debug!(?settings, "sending SETTINGS frame"); + tracing::debug!(?settings, "sending SETTINGS frame"); - let mut send = conn.open_uni().await?; - settings.write(&mut send).await?; + let mut send = conn.open_uni().await?; + settings.write(&mut send).await?; - Ok(send) - } + Ok(send) + } } diff --git a/rs/moq-tokio/src/resolve.rs b/rs/moq-tokio/src/resolve.rs index 3f58d119fa..2ce9408e11 100644 --- a/rs/moq-tokio/src/resolve.rs +++ b/rs/moq-tokio/src/resolve.rs @@ -289,7 +289,7 @@ impl Candidates { /// are handed out anyway so the dial surfaces the OS error instead of a /// confusing "no DNS entries". See /// for the Windows failure this family matching originally fixed. - #[cfg(any(feature = "noq", test))] + #[cfg(any(feature = "quic", test))] pub(crate) fn with_local(mut self, local: SocketAddr, dual_stack: bool) -> Self { self.local = Some((local, dual_stack)); self diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 3c57e4b287..754d40ff57 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -16,7 +16,7 @@ use url::Url; // Only the transports that finish their handshake in a spawned future need `.boxed()`; // the stream listeners hand back an already-built `Request`. -#[cfg(any(feature = "noq", feature = "iroh", feature = "websocket"))] +#[cfg(any(feature = "quic", feature = "iroh", feature = "websocket"))] use futures::FutureExt; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; @@ -86,7 +86,7 @@ impl crate::listen::Config { } /// Default bind address used when [`crate::listen::Config::bind`] is not set. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) const DEFAULT_BIND: net::SocketAddr = net::SocketAddr::V6(net::SocketAddrV6::new(net::Ipv6Addr::UNSPECIFIED, 443, 0, 0)); @@ -109,7 +109,7 @@ pub(crate) enum Parts { /// The QUIC listener only, as one member of a `SO_REUSEPORT` group. Folding /// the member in here is what stops a group member and a stream-only server /// from being asked for at once. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] Member(crate::listen::Socket), } @@ -130,7 +130,7 @@ impl Parts { /// This server's socket in a complete group, when it is a member of one. /// Consuming, because only one backend may own the serving handle. - #[cfg_attr(not(feature = "noq"), expect(dead_code, reason = "no QUIC backend is compiled in"))] + #[cfg_attr(not(feature = "quic"), expect(dead_code, reason = "no QUIC backend is compiled in"))] fn member(self) -> Option { match self { Self::Member(member) => Some(member), @@ -197,7 +197,7 @@ impl Config { } /// Copy the settings one QUIC worker owns. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn worker(&self) -> Self { Self { listen: self.listen.clone(), @@ -232,8 +232,8 @@ pub struct Server { streams: StreamListeners, #[cfg(feature = "iroh")] iroh: Option, - #[cfg(feature = "noq")] - noq: Option, + #[cfg(feature = "quic")] + quic: Option, #[cfg(feature = "websocket")] websocket: Option, } @@ -250,10 +250,10 @@ pub struct Server { /// only caller. #[derive(Clone, Debug)] #[allow(dead_code)] -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) struct SocketRetainer { - #[cfg(feature = "noq")] - noq: Option>, + #[cfg(feature = "quic")] + quic: Option>, } impl Server { @@ -314,9 +314,9 @@ impl Server { // `parts.quic()` and not `build_quic`: a caller that asked for streams only // is not asking for a backend, while leaving everything else configured is // still the error it always was. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] if config.bind.is_some() && parts.quic() { - return Err(Error::NoBackend("--listen requires the noq feature")); + return Err(Error::NoBackend("--listen requires the quic feature")); } // Only the QUIC backend verifies client certificates; the qmux listeners @@ -327,8 +327,8 @@ impl Server { return Err(Error::MtlsUnsupported); } if parts.quic() && !config.tls.root.is_empty() { - #[cfg(not(feature = "noq"))] - return Err(Error::NoBackend("--listen-tls-root requires the noq feature")); + #[cfg(not(feature = "quic"))] + return Err(Error::NoBackend("--listen-tls-root requires the quic feature")); } #[cfg(feature = "_certs")] @@ -337,7 +337,7 @@ impl Server { return Err(crate::tls::Error::ConflictingServe.into()); } #[allow(unused_mut)] - let mut needed = cfg!(feature = "noq") && build_quic; + let mut needed = cfg!(feature = "quic") && build_quic; #[cfg(feature = "tcp")] { needed |= build_streams && config.tcp.bind.is_some() && config.tcp.tls == Some(true); @@ -362,13 +362,13 @@ impl Server { // The member is a serving handle released by a complete reuseport group, // so the backend may take it only once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let member = parts.member(); - #[cfg(feature = "noq")] - let noq = build_quic + #[cfg(feature = "quic")] + let quic = build_quic .then(|| { - crate::noq::NoqServer::new(crate::noq::Options { + crate::quic::Server::new(crate::quic::Options { config: config.clone(), quic: quic.clone(), member, @@ -437,8 +437,8 @@ impl Server { streams, #[cfg(feature = "iroh")] iroh, - #[cfg(feature = "noq")] - noq, + #[cfg(feature = "quic")] + quic, #[cfg(feature = "websocket")] websocket, }) @@ -510,9 +510,9 @@ impl Server { /// /// Empty when no TLS-bearing backend is configured (e.g. a stream-only server). pub fn certificates(&self) -> crate::tls::Certificates { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return noq.certificates(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return quic.certificates(); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`): no certificates. crate::tls::Certificates::empty() @@ -522,17 +522,17 @@ impl Server { /// group after this server is gone. /// /// Crate-private: only the worker group retains sockets this way. A worker - /// member serves the same noq backend as the tokio listener. - #[cfg(feature = "noq")] + /// member serves the same QUIC backend as the tokio listener. + #[cfg(feature = "quic")] pub(crate) fn retain(&self) -> SocketRetainer { SocketRetainer { - #[cfg(feature = "noq")] - noq: self.noq.as_ref().and_then(|server| server.retain()), + #[cfg(feature = "quic")] + quic: self.quic.as_ref().and_then(|server| server.retain()), } } #[cfg(not(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -624,7 +624,7 @@ impl Server { /// The body of [`Listener::accept`]. Stream accept loops start here if /// [`Server::bind`] left them stopped; [`Server::listen`] already started them. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -641,22 +641,22 @@ impl Server { } loop { // The QUIC endpoint address, reported as a QUIC session's local side. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] let local = self.local_addr().ok(); // tokio::select! does not support cfg directives on arms, so we need to create the futures here. // `Some(None)` once a configured endpoint stops, which only losing its // driver does while we accept. - #[cfg(feature = "noq")] - let noq_accept = async { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_mut() { - return Some(noq.accept().await); + #[cfg(feature = "quic")] + let quic_accept = async { + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_mut() { + return Some(quic.accept().await); } None }; - #[cfg(not(feature = "noq"))] - let noq_accept = async { None::<()> }; + #[cfg(not(feature = "quic"))] + let quic_accept = async { None::<()> }; #[cfg(feature = "iroh")] let iroh_accept = async { @@ -714,8 +714,8 @@ impl Server { #[cfg(not(any(feature = "tcp", all(feature = "uds", unix))))] let _ = request; } - Some(_conn) = noq_accept => { - #[cfg(feature = "noq")] + Some(_conn) = quic_accept => { + #[cfg(feature = "quic")] { // The process would keep its other listeners and silently drop // QUIC, so stop accepting at all and let the owner fail loudly. @@ -734,13 +734,13 @@ impl Server { // Accept the transport (capturing url + mTLS identity) and exchange the // MoQ SETUP up front, so path/role are known before the caller authorizes // (like the stream bindings). - let Accepted { session, url, identity, authority, mut link } = within(deadline, super::noq::accept(_conn, alpns)).await?; + let Accepted { session, url, identity, authority, mut link } = within(deadline, crate::quic::accept(_conn, alpns)).await?; link.local = local; let binding = match url { Some(_) => moq_net::transport::Binding::Uri, None => moq_net::transport::Binding::InBand }; let (request, deadline) = accept_request(&setup.server, session, binding, deadline).await?; // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; - Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline, active }) + Ok(Request { transport, url, identity, authority, link, kind: RequestKind::Quic(Box::new(request)), deadline, active }) }.boxed()); } } @@ -809,9 +809,9 @@ impl Server { /// Errors with [`Error::NoBackend`] on a stream-only server, which has no /// QUIC listener. pub fn local_addr(&self) -> crate::Result { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - return Ok(noq.local_addr()?); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + return Ok(quic.local_addr()?); } // No QUIC backend (e.g. a stream-only `--listen-tcp-bind`). Err(Error::NoBackend("no QUIC listener configured")) @@ -828,9 +828,9 @@ impl Server { #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] self.streams.shutdown().await; - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.take() { - noq.shutdown().await; + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.take() { + quic.shutdown().await; } #[cfg(feature = "iroh")] if let Some(iroh) = self.iroh.take() { @@ -844,9 +844,9 @@ impl Server { /// Close the QUIC connections without waiting for the socket, as a drop must. fn close(&mut self) { - #[cfg(feature = "noq")] - if let Some(noq) = self.noq.as_ref() { - noq.close(); + #[cfg(feature = "quic")] + if let Some(quic) = self.quic.as_ref() { + quic.close(); } } } @@ -943,7 +943,7 @@ async fn serve_session(request: Request) -> crate::Result<()> { /// The MoQ server an accepted connection handshakes against, and how long it has. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -957,7 +957,7 @@ struct Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -972,7 +972,7 @@ impl Setup { } #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1035,7 +1035,7 @@ struct Deadline { impl Deadline { #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1078,7 +1078,7 @@ impl Deadline { /// Nothing is established yet that could carry a close code, so an expired /// handshake is dropped, which closes its connection. #[cfg(any( - feature = "noq", + feature = "quic", feature = "iroh", feature = "websocket", feature = "tcp", @@ -1526,8 +1526,8 @@ async fn stream_request( type PendingRequest = moq_net::server::Handshake; pub(crate) enum RequestKind { - #[cfg(feature = "noq")] - Noq(Box>>), + #[cfg(feature = "quic")] + Quic(Box>>), #[cfg(feature = "iroh")] Iroh(Box>>), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1537,7 +1537,7 @@ pub(crate) enum RequestKind { /// The transport-level facts a backend captures while accepting a connection, before the /// MoQ SETUP. Grouped so the shared accept loop builds a [`Request`] from named fields /// rather than a wide tuple. -#[cfg(any(feature = "noq", feature = "iroh"))] +#[cfg(any(feature = "quic", feature = "iroh"))] pub(crate) struct Accepted { pub session: S, pub url: Option, @@ -1606,8 +1606,8 @@ pub enum Reject { macro_rules! request_ref { ($self:expr, $r:ident => $body:expr) => { match &$self.kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1620,8 +1620,8 @@ macro_rules! request_ref { macro_rules! request_into { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => $body, + #[cfg(feature = "quic")] + RequestKind::Quic($r) => $body, #[cfg(feature = "iroh")] RequestKind::Iroh($r) => $body, #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1634,8 +1634,8 @@ macro_rules! request_into { macro_rules! request_map { ($kind:expr, $r:ident => $body:expr) => { match $kind { - #[cfg(feature = "noq")] - RequestKind::Noq($r) => RequestKind::Noq(Box::new($body)), + #[cfg(feature = "quic")] + RequestKind::Quic($r) => RequestKind::Quic(Box::new($body)), #[cfg(feature = "iroh")] RequestKind::Iroh($r) => RequestKind::Iroh(Box::new($body)), #[cfg(any(feature = "tcp", all(feature = "uds", unix), feature = "websocket"))] @@ -1729,7 +1729,7 @@ impl Request { /// transport carries no host (iroh, stream bindings). /// /// Reported as offered: rustls sends no SNI for an IP-literal dial (RFC 6066), - /// so noq reports `None`. + /// so the QUIC backend reports `None`. /// /// Not the moq-net IETF SETUP `Authority` parameter. Client-asserted and not authenticated, /// so authorize on the token or [`Self::peer_identity`] rather than on this value. @@ -1824,7 +1824,7 @@ impl Request { /// The client certificate chain the peer presented, if any, validated /// against a configured [`crate::tls::Listen::root`] during the handshake. /// - /// Captured at the transport handshake (before the SETUP). Noq supports mTLS; + /// Captured at the transport handshake (before the SETUP). The QUIC backend supports mTLS; /// other transports always return `None`. Use it to grant /// elevated access or to close the session once the certificate expires (see /// [`crate::tls::PeerIdentity::expiry`]). @@ -1859,7 +1859,7 @@ mod tests { assert!(matches!(died, Some(Err(err)) if err.is_panic())); } - #[cfg(all(not(feature = "noq"), feature = "_certs"))] + #[cfg(all(not(feature = "quic"), feature = "_certs"))] #[test] fn no_quic_backend_needs_no_served_identity() { Server::new(Config::default()).expect("an absent QUIC backend must not load a TLS identity"); @@ -2654,7 +2654,7 @@ mod tests { /// Closing a listener must release its UDP socket before returning, even while /// an accepted session still holds its QUIC connection, so a restart can rebind /// the port at once. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn close_releases_quic_socket() { let config = crate::listen::Config { @@ -2709,7 +2709,7 @@ mod tests { } /// An explicit QUIC bind cannot be honored without a QUIC backend. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn quic_bind_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2729,7 +2729,7 @@ mod tests { /// A client CA on the default QUIC listener names the missing backend, not a /// stream-only server the caller never configured. - #[cfg(not(feature = "noq"))] + #[cfg(not(feature = "quic"))] #[test] fn client_ca_without_a_quic_backend_is_rejected() { let config = crate::listen::Config { @@ -2752,7 +2752,7 @@ mod tests { /// A QUIC-only server reports nothing. It multiplexes over one UDP socket and /// never calls `accept`, so a zero counter would be a watch that cannot fire. - #[cfg(all(feature = "noq", not(feature = "tcp")))] + #[cfg(all(feature = "quic", not(feature = "tcp")))] #[test] fn accept_health_is_empty_without_a_stream_listener() { let server = crate::listen::Config::default() @@ -2773,7 +2773,7 @@ mod tests { /// Building the endpoint needs a runtime, and `certificates()` must stay /// readable without one (no guard escapes to the caller). - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn certificates_expose_generated_fingerprints() { let mut config = crate::listen::Config { diff --git a/rs/moq-tokio/src/tcp.rs b/rs/moq-tokio/src/tcp.rs index 1c19e4310f..2ff15e6778 100644 --- a/rs/moq-tokio/src/tcp.rs +++ b/rs/moq-tokio/src/tcp.rs @@ -376,7 +376,7 @@ mod tests { listen.tcp.tls = Some(true); listen.tls.generate = vec!["localhost".into()]; // Client-certificate policy is refused without a QUIC listener to read it. - if !cfg!(feature = "noq") { + if !cfg!(feature = "quic") { continue; } listen.bind = Some("127.0.0.1:0".parse().unwrap()); diff --git a/rs/moq-tokio/src/tls.rs b/rs/moq-tokio/src/tls.rs index 79f56ecef6..a28dcfcd7c 100644 --- a/rs/moq-tokio/src/tls.rs +++ b/rs/moq-tokio/src/tls.rs @@ -24,7 +24,7 @@ use rustls::pki_types::PrivatePkcs8KeyDer; /// Errors loading or generating TLS certificates and keys. /// -/// Shared by the client TLS config and noq server. +/// Shared by the client TLS config and QUIC server. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { @@ -791,7 +791,7 @@ pub(crate) enum Verification { Fingerprints(Vec<[u8; 32]>), /// Standard CA verification. When `system` is set the platform/default trust - /// store is trusted too; noq uses the OS platform verifier. + /// store is trusted too; the QUIC backend uses the OS platform verifier. /// `custom` are extra PEM roots trusted in addition. Roots { custom: CustomRoots, system: bool }, } @@ -955,7 +955,7 @@ impl Connect { /// plaintext fetch, and there is nothing to bootstrap when verification is /// disabled. With CA roots (the default), `http://` is the deliberate /// per-connection way to pin a self-signed relay, so it is allowed. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn allows_http_bootstrap(&self) -> bool { self.fingerprint.is_empty() && !self.insecure.unwrap_or_default() } @@ -1634,7 +1634,7 @@ impl PeerIdentity { /// Wrap the type-erased identity from the QUIC connection. /// Returns `None` if the peer presented no certificate or the identity is /// not a certificate chain. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] pub(crate) fn from_any(identity: Option>) -> Option { let chain = identity?.downcast::>>().ok()?; Some(Self { chain: *chain }) @@ -2092,7 +2092,7 @@ impl rustls::client::danger::ServerCertVerifier for FingerprintVerifier { } #[cfg(test)] -#[cfg(all(feature = "noq", feature = "aws-lc-rs"))] +#[cfg(all(feature = "quic", feature = "aws-lc-rs"))] mod tests { #[tokio::test] async fn resolved_config_keeps_reload_owner_alive() { @@ -2266,7 +2266,7 @@ mod tests { params.self_signed(&key).unwrap().into() } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_expiry_reads_not_after() { // notAfter at a whole second so the round-trip is exact. @@ -2277,7 +2277,7 @@ mod tests { params.not_after = not_after; let cert: CertificateDer<'static> = params.self_signed(&key).unwrap().into(); - // noq hand back the chain as a boxed Vec. + // The QUIC backend hands back the chain as a boxed Vec. let identity: Box = Box::new(vec![cert]); let parsed = PeerIdentity::from_any(Some(identity)).expect("chain parsed"); let expiry = parsed.expiry().expect("expiry parsed"); @@ -2287,7 +2287,7 @@ mod tests { ); } - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn peer_identity_none_without_chain() { assert!(PeerIdentity::from_any(None).is_none()); @@ -2390,7 +2390,7 @@ mod tests { /// keeps its task, an OS directory watch, and the certificate keys alive for the /// rest of the process. The `Arc` is the observable half: while the task lives it /// holds one, so a `Weak` that still upgrades is a watcher that never stopped. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[tokio::test] async fn dropping_the_reload_guard_stops_the_watcher() { let key = rcgen::KeyPair::generate().unwrap(); @@ -2525,7 +2525,7 @@ mod tests { /// A resumed handshake skips certificate verification, so a listener with a /// live peer set must disable resumption for removals to take effect. - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn removed_peers_cannot_resume() { let server_identity = Identity::generate(["localhost"]).unwrap(); @@ -2559,7 +2559,7 @@ mod tests { } /// A plain-TLS embedder receives a working config from the shared identity. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[tokio::test] async fn served_config_handshakes() { let server = Listen { @@ -2685,7 +2685,7 @@ mod tests { (file, path) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn signed_certificates() -> ( String, CertificateDer<'static>, @@ -2722,7 +2722,7 @@ mod tests { ) } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] fn handshake_kinds( client: Arc, server: Arc, @@ -2754,7 +2754,7 @@ mod tests { panic!("TLS handshake did not settle"); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_roots_disable_session_resumption() { use std::io::Write; @@ -2805,7 +2805,7 @@ mod tests { ); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn reloadable_client_roots_disable_server_resumption() { let (ca_a, server_cert, server_key, client_cert, client_key) = signed_certificates(); @@ -2884,7 +2884,7 @@ mod tests { assert!(handshake_kinds(reloadable_client, reloadable).is_err()); } - #[cfg(all(feature = "watch", feature = "noq"))] + #[cfg(all(feature = "watch", feature = "quic"))] #[test] fn custom_roots_reload_for_new_client_and_server_handshakes() { let (ca_a, server_a, _, client_a, _) = signed_certificates(); @@ -3296,7 +3296,7 @@ mod legacy_tests { /// The accept side, where a dropped `--server-tls-root` takes the mTLS client /// CAs with it and leaves the listener accepting unauthenticated peers. /// moq-cli's `export hls` flattens this type directly, so it is a live path. - #[cfg(feature = "noq")] + #[cfg(feature = "quic")] #[test] fn the_server_builders_refuse_a_released_spelling() { let tls = parse(&["--server-tls-root", "/tmp/ca.pem"]).listen; diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index a87ec2f1c1..5ab97f52a9 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -1,7 +1,7 @@ //! Adapts the async transport interface to the poll one moq-net requires. //! //! moq-net accepts its own `moq_net::transport::poll` traits. -//! A backend that offers only the async interface (qmux, iroh, noq) is wrapped +//! A backend that offers only the async interface (qmux, iroh, quic) is wrapped //! in [`Session`] here, at the edge where async already lives. The wrapping //! costs one allocation per operation and one copy per write, and its //! closed-watch emulation is weaker than a native implementation (see diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 609de2be9f..90b2e67fd1 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -184,7 +184,7 @@ fn won_key(url: &Url) -> Result<(String, u16)> { } /// Forget that WebSocket won for `url`, giving QUIC its head start again. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) fn forget(url: &Url) { if let Ok(key) = won_key(url) { WEBSOCKET_WON.lock().unwrap().remove(&key); @@ -192,13 +192,13 @@ pub(crate) fn forget(url: &Url) { } /// Record that WebSocket won for `url`, as a fallback dial that lands does. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn remember(url: &Url) { WEBSOCKET_WON.lock().unwrap().insert(won_key(url).unwrap()); } /// Whether the next dial to `url` skips QUIC's head start. -#[cfg(all(test, feature = "noq"))] +#[cfg(all(test, feature = "quic"))] pub(crate) fn won(url: &Url) -> bool { won_key(url).is_ok_and(|key| WEBSOCKET_WON.lock().unwrap().contains(&key)) } @@ -339,7 +339,7 @@ pub struct Resolved { /// The fallback arm of the QUIC-vs-WebSocket race, so only compiled when there is a /// QUIC dial to race against. A WebSocket-only build calls [`connect`] directly. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub(crate) async fn race_handle( config: &Config, tls: &rustls::ClientConfig, diff --git a/rs/moq-tokio/src/worker/group.rs b/rs/moq-tokio/src/worker/group.rs index 282a1db116..c98642d752 100644 --- a/rs/moq-tokio/src/worker/group.rs +++ b/rs/moq-tokio/src/worker/group.rs @@ -94,7 +94,7 @@ impl Workers { // hands out one member per slot in the order the kernel numbers them by. let mut forming = moq_sock::shard::Group::acquire(requested, config.count).map_err(|err| match err { moq_sock::shard::Error::Count { count, max } => Error::WorkerCount { count, max }, - moq_sock::shard::Error::Resolve(err) => crate::noq::Error::BindSocket(err).into(), + moq_sock::shard::Error::Resolve(err) => crate::quic::Error::BindSocket(err).into(), // The port lock is the only other way to lose the address, and it is // held by exactly one thing: another group of this UID. _ => Error::WorkerOverlap { addr: requested }, @@ -102,16 +102,16 @@ impl Workers { let count = forming.count(); let mut claims = Vec::with_capacity(count as usize); while let Some(member) = forming.member() { - claims.push(member.bind().map_err(crate::noq::Error::BindSocket)?); + claims.push(member.bind().map_err(crate::quic::Error::BindSocket)?); } - let mut group = forming.complete(claims).map_err(crate::noq::Error::BindSocket)?; + let mut group = forming.complete(claims).map_err(crate::quic::Error::BindSocket)?; let shared = Arc::new(Shared::default()); let mut workers = Vec::with_capacity(count as usize); let mut certificates = None; - while let Some(member) = group.member().map_err(crate::noq::Error::BindSocket)? { + while let Some(member) = group.member().map_err(crate::quic::Error::BindSocket)? { let index = member.shard().index(); // `max(1)` because an empty core list means pinning is off, not that // there are no workers. diff --git a/rs/moq-tokio/src/worker/mod.rs b/rs/moq-tokio/src/worker/mod.rs index 403835a4c0..1cb9991806 100644 --- a/rs/moq-tokio/src/worker/mod.rs +++ b/rs/moq-tokio/src/worker/mod.rs @@ -20,10 +20,10 @@ // Everything but the knobs: a group binds and serves QUIC, so it needs a backend // to bind with. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] mod group; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub use group::{Group, Member, Spawner, Workers}; /// How many QUIC workers to run, and whether to pin them. diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 6097344197..161ebce324 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -1,4 +1,4 @@ -//! Integration tests for noq QUIC and the iroh transport. +//! Integration tests for QUIC and the iroh transport. use std::time::Duration; @@ -10,7 +10,7 @@ fn timed() -> moq_net::track::Info { const TIMEOUT: Duration = Duration::from_secs(10); /// Inputs for [`connect_test`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct ConnectTest<'a> { /// URL scheme to dial (`moqt` for raw QUIC, `https` for WebTransport). scheme: &'a str, @@ -39,7 +39,7 @@ struct ConnectTest<'a> { /// /// Dials `localhost`, so the client sends an SNI. Use [`no_sni_test`] to cover /// the SNI-less path. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn backend_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -60,7 +60,7 @@ async fn backend_test(scheme: &str) { /// Raw QUIC (`moqt`/`moql`) has no request URI, so the whole request target has to /// ride the SETUP; WebTransport carries it in the CONNECT URL instead. Either way the /// server reports the same route and query through [`moq_tokio::server::Request`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn path_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -80,7 +80,7 @@ async fn path_test(scheme: &str) { /// in the server name). Raw QUIC has no in-band request URL, so this exercises /// the accept path with an empty server name, which must still establish rather /// than reject. Binds the loopback IP directly to avoid dual-stack flakiness. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn no_sni_test(scheme: &str) { connect_test(ConnectTest { scheme, @@ -98,7 +98,7 @@ async fn no_sni_test(scheme: &str) { /// Publish a broadcast on the server bound to `bind`, subscribe on a client that /// dials `authority`, and verify the data arrives over the requested scheme. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn connect_test(config: ConnectTest<'_>) { let ConnectTest { scheme, @@ -225,7 +225,7 @@ async fn connect_test(config: ConnectTest<'_>) { /// Write a self-signed PEM cert + key for `name` to `dir`, prefixed by `stem`. /// /// Returns the two paths, in the order the `cert` and `key` lists want them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::path::PathBuf, std::path::PathBuf) { use std::io::Write; @@ -253,7 +253,7 @@ fn write_self_signed(dir: &std::path::Path, stem: &str, name: &str) -> (std::pat /// when the server picked the certificate the SNI asked for. Without SNI /// selection every client would get the first configured certificate, so the /// `alt.localhost` pin below would never match. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn sni_test() { let dir = tempfile::tempdir().expect("tempdir"); let (first_cert, first_key) = write_self_signed(dir.path(), "first", "localhost"); @@ -310,7 +310,7 @@ async fn sni_test() { } /// A generated certificate joins the file-backed ones rather than replacing them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn cert_sources_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -335,7 +335,7 @@ async fn cert_sources_test() { /// Rotate the certificate files under a running listener and assert the served /// set follows, without the listener being rebuilt. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn reload_test() { let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -383,7 +383,7 @@ async fn reload_test() { /// Generate a CA, a server cert + key, and a client cert + key (all PEM, the /// leaf certs signed by the CA) written to a tempdir. Returns the dir plus the /// five paths so the caller can wire them into the TLS configs. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { use rcgen::{ BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, KeyUsagePurpose, @@ -436,7 +436,7 @@ fn generate_mtls_certs() -> (tempfile::TempDir, MtlsPaths) { } /// Filesystem paths to the PEM material produced by [`generate_mtls_certs`]. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] struct MtlsPaths { ca: std::path::PathBuf, server_cert: std::path::PathBuf, @@ -447,7 +447,7 @@ struct MtlsPaths { /// Connect with a client certificate signed by a CA the server trusts, and /// assert the server observes the validated peer certificate via mTLS. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn mtls_test(scheme: &str, reject: bool) { let (_dir, paths) = generate_mtls_certs(); @@ -736,14 +736,14 @@ async fn iroh_connect_test(version: Option<&str>) { .expect("server task failed"); } -// ── Noq backend ───────────────────────────────────────────────────── +// ── QUIC backend ──────────────────────────────────────────────────── /// A client that closes before its runtime stops tells the server at once, instead of /// leaving it to the idle timeout, which is what a process exiting on a signal does. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_reaches_server() { +async fn quic_client_close_reaches_server() { // An idle timeout inside TIMEOUT would hide a lost close. let mut quic = moq_tokio::quic::Config::default(); quic.idle_timeout = Some(TIMEOUT * 3); @@ -802,10 +802,10 @@ async fn noq_client_close_reaches_server() { /// A client that finishes its track and closes before its runtime stops still delivers /// the queued group and the track's finish, instead of the close discarding them. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_finished_track() { +async fn quic_client_close_drains_finished_track() { // Small, since the debug build logging every packet is slow and the drain has // one second. Queued right before the close, it is still unacknowledged then. let payload: Vec = (0..1024).map(|i| i as u8).collect(); @@ -916,10 +916,10 @@ async fn noq_client_close_drains_finished_track() { /// A GOAWAY leaves the old session serving its subscriptions after the replacement /// connects, and a close then drains that predecessor too rather than dropping it. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_drains_migrated_predecessor() { +async fn quic_client_close_drains_migrated_predecessor() { // Several congestion windows, so it is still in flight when the close starts. let payload: Vec = (0..64 * 1024).map(|i| i as u8).collect(); @@ -1064,10 +1064,10 @@ async fn noq_client_close_drains_migrated_predecessor() { /// A close cuts a predecessor that cannot drain off at its handover deadline, /// rather than holding it for the full one second close window. -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_client_close_keeps_predecessor_handover() { +async fn quic_client_close_keeps_predecessor_handover() { let quic = moq_tokio::quic::Config::default(); let mut server_config = moq_tokio::listen::Config::default(); server_config.bind = Some("127.0.0.1:0".parse().unwrap()); @@ -1173,73 +1173,73 @@ async fn noq_client_close_keeps_predecessor_handover() { ); } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic() { +async fn quic_raw_quic() { backend_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_no_sni() { +async fn quic_raw_quic_no_sni() { no_sni_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_path() { +async fn quic_raw_quic_path() { path_test("moqt").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_raw_quic_moql_path() { +async fn quic_raw_quic_moql_path() { path_test("moql").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport_path() { +async fn quic_webtransport_path() { path_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_webtransport() { +async fn quic_webtransport() { backend_test("https").await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_mtls() { +async fn quic_mtls() { mtls_test("https", false).await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_sni_certificate() { +async fn quic_sni_certificate() { sni_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_sources() { +async fn quic_cert_sources() { cert_sources_test().await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tracing_test::traced_test] #[tokio::test] -async fn noq_cert_reload() { +async fn quic_cert_reload() { reload_test().await; } @@ -1249,8 +1249,8 @@ async fn noq_cert_reload() { /// files it left behind. /// /// Both ends write into one directory, so this covers the client and server paths -/// at once. Noq writes one file per connection. -#[cfg(all(feature = "qlog", feature = "noq"))] +/// at once. Each endpoint writes one file. +#[cfg(all(feature = "qlog", feature = "quic"))] async fn qlog_test(scheme: &str) -> Vec { let dir = tempfile::tempdir().expect("failed to create tempdir"); @@ -1276,7 +1276,7 @@ async fn qlog_test(scheme: &str) -> Vec { .collect(); for trace in &traces { - // Noq writes JSON-SEQ (RFC 7464): each record is a 0x1e separator then + // qlog streams JSON-SEQ (RFC 7464): each record is a 0x1e separator then // JSON, the first being the qlog header. Checking the bytes rather than just the // length catches a writer that was buffered and never flushed. let raw = std::fs::read(trace).expect("failed to read trace"); @@ -1303,9 +1303,9 @@ async fn qlog_test(scheme: &str) -> Vec { traces } -#[cfg(all(feature = "qlog", feature = "noq"))] +#[cfg(all(feature = "qlog", feature = "quic"))] #[tokio::test] -async fn noq_qlog() { +async fn quic_qlog() { let traces = qlog_test("moqt").await; assert!(!traces.is_empty(), "expected at least one trace"); } @@ -1334,7 +1334,7 @@ async fn connect_once( /// The size is the point: a frame that fits inside one window would pass whether or /// not the setting ever reached the backend. /// -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] async fn window_test(scheme: &str) { let mut quic = moq_tokio::quic::Config::default(); quic.receive_window = Some(64 * 1024); @@ -1357,9 +1357,9 @@ async fn window_test(scheme: &str) { .await; } -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] #[tokio::test] -async fn noq_windows() { +async fn quic_windows() { window_test("moqt").await; } diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 7cb70653e9..4ee4044de5 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -2481,7 +2481,7 @@ async fn quic_driver_task_inherits_connection_span() { .expect("group closed prematurely"); assert_eq!(&frame.payload[..], b"hello"); - // Other tasks (e.g. noq's own internal per-connection IO loop) legitimately + // Other tasks (e.g. the QUIC driver's per-connection IO loop) legitimately // carry the "conn" span too, since they're spawned while it's entered. So it's // not enough for the marker to appear *somewhere* in the logs; it must be on // the exact line moq-net emits from inside the driver task the runtime diff --git a/rs/moq-tokio/tests/close_code.rs b/rs/moq-tokio/tests/close_code.rs index 9d5f287e97..c3149c1f77 100644 --- a/rs/moq-tokio/tests/close_code.rs +++ b/rs/moq-tokio/tests/close_code.rs @@ -3,7 +3,7 @@ //! Each case runs several fresh connections, since the losses this guards against were //! races between the peer's close and whatever the transport reported next. -#![cfg(all(feature = "noq", feature = "websocket"))] +#![cfg(all(feature = "quic", feature = "websocket"))] use std::time::Duration; diff --git a/rs/moq-tokio/tests/stream_code.rs b/rs/moq-tokio/tests/stream_code.rs index 922fe36488..508a56db8d 100644 --- a/rs/moq-tokio/tests/stream_code.rs +++ b/rs/moq-tokio/tests/stream_code.rs @@ -6,11 +6,11 @@ //! subscription it never made with STOP_SENDING, and the client reads that code off the //! wire. moq-transport codes take the same raw session. -#![cfg(feature = "noq")] +#![cfg(feature = "quic")] use std::{sync::Arc, time::Duration}; -use web_transport_moq::noq; +use moq_tokio::quic::driver; /// moq-lite sends CANCEL (0x1) for a group it has no subscription for. const CANCEL: u32 = 0x1; @@ -68,7 +68,7 @@ async fn serve() -> Server { /// Dial the server over raw QUIC with `alpn`, send an empty SETUP and a GROUP for an /// unknown subscription, and return the code the server stopped the GROUP with. -async fn refused_code(server: &Server, alpn: &str) -> Option { +async fn refused_code(server: &Server, alpn: &str) -> Option { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let mut roots = rustls::RootCertStore::empty(); @@ -77,12 +77,12 @@ async fn refused_code(server: &Server, alpn: &str) -> Option { .with_root_certificates(roots) .with_no_client_auth(); tls.alpn_protocols = vec![alpn.as_bytes().to_vec()]; - let quic = noq::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); + let quic = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic config"); - let endpoint = noq::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); + let endpoint = driver::Endpoint::client("127.0.0.1:0".parse().unwrap()).expect("endpoint"); let conn = endpoint .connect_with( - noq::ClientConfig::new(Arc::new(quic)), + driver::ClientConfig::new(Arc::new(quic)), ([127, 0, 0, 1], server.port).into(), "localhost", ) diff --git a/rs/moq-tokio/tests/worker.rs b/rs/moq-tokio/tests/worker.rs index c31196628b..bce5b6607e 100644 --- a/rs/moq-tokio/tests/worker.rs +++ b/rs/moq-tokio/tests/worker.rs @@ -3,7 +3,7 @@ //! Linux-only, because the mode is: every worker binds the listen address with //! `SO_REUSEPORT`, and no other platform load-balances a unicast UDP port across //! the group. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] use std::net::UdpSocket; diff --git a/rs/moq-transcode/Cargo.toml b/rs/moq-transcode/Cargo.toml index 1f5a900982..39ef2b9d42 100644 --- a/rs/moq-transcode/Cargo.toml +++ b/rs/moq-transcode/Cargo.toml @@ -44,7 +44,7 @@ tracing = { workspace = true } [dev-dependencies] anyhow = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs", "websocket"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs", "websocket"] } # test-util unlocks tokio::time::pause for the deterministic live-feed tests. tokio = { workspace = true, features = ["full", "test-util"] } url = { workspace = true } diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index e373fe2e65..a0e1f2513e 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -13,13 +13,13 @@ keywords = ["quic", "io-uring", "networking", "linux"] categories = ["asynchronous", "network-programming", "os::linux-apis"] [features] -default = ["noq"] -# moq-noq-proto, with TLS through rustls. -noq = ["dep:moq-noq-proto", "dep:rustls", "dep:rustls-platform-verifier"] +default = ["quic"] +# QUIC and WebTransport on moq-quic, with TLS through rustls. +quic = ["dep:moq-quic", "dep:moq-clock", "dep:rustls", "dep:rustls-platform-verifier"] # Capture qlog traces (`quic::Transport::qlog`). Off by default: it compiles the -# qlog crate and its serde_json encoder into noq, which a production build has +# qlog crate and its serde_json encoder into moq-quic, which a production build has # no use for. -qlog = ["moq-noq-proto?/qlog"] +qlog = ["moq-quic?/qlog"] [target.'cfg(target_os = "linux")'.dependencies] bytes = "1" @@ -27,9 +27,10 @@ http = "1" io-uring = { workspace = true } kio = { workspace = true } libc = { workspace = true } +# The QUIC driver hands moq-quic its clock's instants. +moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } -# default-features off so the crypto provider is ours to pick. -moq-noq-proto = { workspace = true, features = ["bloom", "tracing-log", "rustls-aws-lc-rs"], optional = true } +moq-quic = { workspace = true, features = ["bloom", "rustls-aws-lc-rs"], optional = true } moq-sock = { workspace = true } rand = "0.10" rustc-hash = { workspace = true } @@ -56,7 +57,7 @@ web-transport-trait = { workspace = true } [target.'cfg(target_os = "linux")'.dev-dependencies] io-uring = { workspace = true } libc = { workspace = true } -moq-tokio = { path = "../moq-tokio", default-features = false, features = ["noq", "aws-lc-rs"] } +moq-tokio = { path = "../moq-tokio", default-features = false, features = ["quic", "aws-lc-rs"] } nix = { workspace = true } rcgen = { version = "0.14", default-features = false, features = ["pem", "aws_lc_rs"] } rustls = "0.23" @@ -65,7 +66,6 @@ rustls = "0.23" serde_json = { workspace = true } tempfile = { workspace = true } tracing = "0.1" -web-transport-moq = { workspace = true } [[bench]] name = "udp_tokio" @@ -76,10 +76,9 @@ name = "udp_uring" harness = false [[bench]] -name = "echo_noq" -path = "benches/echo_noq.rs" +name = "echo_quic" harness = false -required-features = ["noq"] +required-features = ["quic"] [[bench]] name = "session_lite" diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index af817d5e93..db5a5a8aec 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -71,35 +71,35 @@ the tokio stack. ## Backends -The `quic` module uses the sans-IO [moq-noq-proto](https://github.com/moq-dev/noq) -stack with rustls. The `noq` feature is enabled by default and remains optional -so the worker, timers, and UDP socket can be built without QUIC. +The `quic` module uses the sans-IO [moq-quic](../moq-quic) stack with rustls. +The `quic` feature is enabled by default and remains optional so the worker, +timers, and UDP socket can be built without QUIC. | Feature | Stack | TLS | |---|---|---| -| `noq` (default) | [moq-noq-proto](https://github.com/moq-dev/noq) | rustls | +| `quic` (default) | [moq-quic](../moq-quic) | rustls | Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features noq +cargo test -p moq-uring --features quic ``` `moq-relay` enables it with `--features io-uring`. -The `qlog` feature turns on noq's capture support. It writes one file per -connection. +The `qlog` feature turns on moq-quic's capture support. It writes one file per +endpoint on the accept side and per dial on the connect side. ## Validation -`tests/echo.rs` runs an echo against a tokio noq peer over the worker: -handshake, half a megabyte each way, and timers driven by noq's timeout. +`tests/echo.rs` runs an echo against a tokio QUIC peer over the worker: +handshake, half a megabyte each way, and timers driven by moq-quic's timeout. `tests/session.rs` runs full moq-lite sessions through `quic::Endpoint` (including two clients demuxed on one server socket), and `tests/endpoint.rs` covers the endpoint mechanics (dial+accept on one socket, version negotiation, the dial-only refusal), `tests/workers.rs` runs a steered two-worker reuseport group serving one port across threads, and -`tests/web.rs` is WebTransport interop against `web-transport-moq`: stream and +`tests/web.rs` is WebTransport interop against moq-tokio's: stream and datagram echo through the H3 framing, close codes through the capsule, and a full moq-lite session over WebTransport. All of them skip (loudly) below the kernel floor, which @@ -109,8 +109,8 @@ includes GitHub-hosted CI runners. `udp_tokio` and `udp_uring` are the disposable syscall-level matrices from the first spike (recv batching x GRO x GSO, epoll vs io\_uring); see git history -for their methodology. `echo_noq` is the ablation matrix over the real -worker: the same noq echo with receive batching, GRO, and GSO toggled one +for their methodology. `echo_quic` is the ablation matrix over the real +worker: the same QUIC echo with receive batching, GRO, and GSO toggled one at a time. `worker_turns` sweeps socket counts and datagrams per burst independently to measure drive-loop batching without QUIC. Its smoke cases run in the Rust nightly benchmark group. diff --git a/rs/moq-uring/benches/echo_noq.rs b/rs/moq-uring/benches/echo_quic.rs similarity index 93% rename from rs/moq-uring/benches/echo_noq.rs rename to rs/moq-uring/benches/echo_quic.rs index e9c0866f1c..a7fbc1008d 100644 --- a/rs/moq-uring/benches/echo_noq.rs +++ b/rs/moq-uring/benches/echo_quic.rs @@ -1,4 +1,4 @@ -//! Echo a tokio noq peer through the io_uring worker while ablating UDP features. +//! Echo a tokio QUIC peer through the io_uring worker while ablating UDP features. use criterion::{criterion_group, criterion_main}; @@ -50,7 +50,7 @@ mod linux { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -72,11 +72,11 @@ mod linux { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); @@ -154,7 +154,7 @@ mod linux { ("all-off", none), ]; - let mut group = c.benchmark_group("echo_noq"); + let mut group = c.benchmark_group("echo_quic"); group.throughput(Throughput::Bytes(PAYLOAD as u64)); for (name, udp_config) in ablations { diff --git a/rs/moq-uring/benches/session_lite.rs b/rs/moq-uring/benches/session_lite.rs index cc629a98b6..1b4bd616e0 100644 --- a/rs/moq-uring/benches/session_lite.rs +++ b/rs/moq-uring/benches/session_lite.rs @@ -1,7 +1,7 @@ //! The PR-4 ablation matrix at session level: a full moq-lite session pair on //! one worker, publisher to subscriber over loopback QUIC, toggling receive //! batching, GRO, and GSO exactly like the raw echo. Each iteration delivers -//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_noq`'s unit), so +//! one group of 32 x 32 KiB frames (1 MiB, matching `echo_quic`'s unit), so //! the two matrices are directly comparable: the difference is the moq-net //! machine and container framing on top of the same wire path. //! @@ -9,11 +9,11 @@ use criterion::{criterion_group, criterion_main}; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] #[path = "../tests/support.rs"] mod support; -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] mod linux { use std::net::UdpSocket; use std::time::Instant; @@ -235,10 +235,10 @@ mod linux { } } -#[cfg(all(target_os = "linux", feature = "noq"))] +#[cfg(all(target_os = "linux", feature = "quic"))] use linux::benchmark; -#[cfg(not(all(target_os = "linux", feature = "noq")))] +#[cfg(not(all(target_os = "linux", feature = "quic")))] fn benchmark(_: &mut criterion::Criterion) {} criterion_group!(benches, benchmark); diff --git a/rs/moq-uring/src/lib.rs b/rs/moq-uring/src/lib.rs index da3c2be544..09f75226f6 100644 --- a/rs/moq-uring/src/lib.rs +++ b/rs/moq-uring/src/lib.rs @@ -21,7 +21,7 @@ //! member of a steered reuseport group ([`udp::Bound`]) issues connection ids //! that steer back to it. [`Handle::run`] supplies time and schedules driver //! wakeups; callers run the returned drivers with [`Handle::spawn`]. The stack -//! underneath is enabled by the `noq` feature; a build without it leaves the +//! underneath is enabled by the `quic` feature; a build without it leaves the //! module out. //! //! [`metrics::Metrics`] is how the worker's own health leaves its thread: @@ -39,7 +39,7 @@ mod error; pub mod metrics; mod park; -#[cfg(feature = "noq")] +#[cfg(feature = "quic")] pub mod quic; mod shared; mod timer; diff --git a/rs/moq-uring/src/quic/noq/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs similarity index 87% rename from rs/moq-uring/src/quic/noq/connection.rs rename to rs/moq-uring/src/quic/driver/connection.rs index 9def53c5ee..fb70a5a112 100644 --- a/rs/moq-uring/src/quic/noq/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -7,7 +7,7 @@ use std::task::{Context, Poll}; use std::time::Instant; use bytes::Bytes; -use moq_noq_proto::{ConnectionHandle, Dir, StreamId, VarInt}; +use moq_quic::{ConnectionHandle, Dir, StreamId, VarInt}; use rustc_hash::FxHashMap; use super::super::{Error, SEGMENT}; @@ -28,7 +28,7 @@ const TRAIN_SEGMENTS: usize = 63; /// operation can mutate the connection, drop that borrow, and then register a /// waiter without ever holding both. pub(crate) struct Inner { - pub(crate) conn: RefCell, + pub(crate) conn: RefCell, pub(crate) state: RefCell, /// The worker driving this connection, which anything layered on it /// (the WebTransport handshake, say) runs on too. @@ -44,7 +44,7 @@ pub(crate) struct State { established: bool, establish_waiters: kio::WaiterList, - /// Whoever is waiting for a peer-initiated stream. noq-proto hands the + /// Whoever is waiting for a peer-initiated stream. moq-quic hands the /// ids out in order, so the queue is its own. accept_bi_waiters: kio::WaiterList, accept_uni_waiters: kio::WaiterList, @@ -55,7 +55,7 @@ pub(crate) struct State { /// Per-stream read/write parking, keyed by stream id. /// /// FxHash rather than SipHash on all four of these: they sit on the - /// driver's per-event path, and noq-proto assigns the ids. + /// driver's per-event path, and moq-quic assigns the ids. readable: FxHashMap, writable: FxHashMap, /// Send streams waiting for their end: a FIN the peer acknowledged, or a @@ -83,7 +83,7 @@ pub(crate) struct State { dead: bool, /// The close this side asked for, until the driver has put it on the wire. - /// noq raises no event for it, so this is what the terminal error is + /// moq-quic raises no event for it, so this is what the terminal error is /// built from. local_close: Option<(u64, String)>, } @@ -191,13 +191,13 @@ impl Inner { /// Start tracking send stream `id`, which a handle now owns. /// - /// Seeded from noq rather than empty: a peer can open a bidirectional + /// Seeded from moq-quic rather than empty: a peer can open a bidirectional /// stream and stop it before the application ever accepts it, and that - /// event arrives with no handle to record it against. noq still knows, + /// event arrives with no handle to record it against. moq-quic still knows, /// so a `poll_closed` on the fresh handle reports the stop instead of /// waiting for an event that has already been and gone. pub(crate) fn track(&self, id: StreamId) { - // Err means noq has no send half for the id, which is nothing to + // Err means moq-quic has no send half for the id, which is nothing to // report either way. let stopped = self.conn.borrow_mut().send_stream(id).stopped().ok().flatten(); self.state @@ -242,11 +242,11 @@ impl Inner { /// space, which needs the whole varint range. pub(crate) fn close_code(&self, code: u64, reason: &str) { self.conn.borrow_mut().close( - Instant::now(), + moq_clock::Instant::now(), VarInt::from_u64(code).unwrap_or(VarInt::MAX), Bytes::copy_from_slice(reason.as_bytes()), ); - // noq raises no event for a close the application asked for, so the + // moq-quic raises no event for a close the application asked for, so the // terminal error is ours to publish. Not here though: the driver // publishes it once it has staged the CONNECTION_CLOSE, so a caller // that stops driving the worker the moment `poll_closed` resolves has @@ -355,7 +355,7 @@ impl Clone for Connection { /// Build a connection's shared state and its driver. /// /// The driver future does timers, event sweeps, and egress; ingress arrives -/// from the endpoint's demux task, which feeds noq-proto directly and +/// from the endpoint's demux task, which feeds moq-quic directly and /// [kicks](Inner::kick) the driver. The caller spawns the future and reclaims /// the connection's bookkeeping once it resolves. pub(crate) fn launch( @@ -363,7 +363,7 @@ pub(crate) fn launch( socket: Rc, endpoint: Weak, key: ConnectionHandle, - conn: moq_noq_proto::Connection, + conn: moq_quic::Connection, ) -> (Shared, impl Future + use<>) { let shared = Rc::new(Inner { conn: RefCell::new(conn), @@ -443,7 +443,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu let handshake = conn .crypto_session() .handshake_data() - .and_then(|data| data.downcast::().ok()); + .and_then(|data| data.downcast::().ok()); let (alpn, server_name) = match handshake { Some(data) => ( data.protocol.map(|proto| String::from_utf8_lossy(&proto).into_owned()), @@ -451,11 +451,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu ), None => (None, None), }; - // Multipath is never negotiated here, so the first path is the only one. - let remote = conn - .network_path(moq_noq_proto::PathId::ZERO) - .expect("an established connection has its first path") - .remote(); + let remote = conn.remote_address(); (alpn, server_name, remote) }; @@ -563,7 +559,7 @@ impl web_transport_trait::poll::Session for Connection { Poll::Ready(Ok(())) } // The send queue is full; a flush frees space. - Err(moq_noq_proto::SendDatagramError::Blocked(_)) => { + Err(moq_quic::SendDatagramError::Blocked(_)) => { let mut state = self.shared.state.borrow_mut(); waiter.register(&mut state.datagram_send_waiters); Poll::Pending @@ -608,20 +604,16 @@ impl web_transport_trait::poll::Session for Connection { } fn stats(&self) -> impl web_transport_trait::Stats { - let (stats, path) = { - let mut conn = self.shared.conn.borrow_mut(); - let stats = conn.stats(); - let path = conn.path_stats(moq_noq_proto::PathId::ZERO).unwrap_or_default(); - (stats, path) - }; + let stats = self.shared.conn.borrow().stats(); Stats { bytes_sent: stats.udp_tx.bytes, bytes_received: stats.udp_rx.bytes, - bytes_lost: stats.lost_bytes, + bytes_lost: stats.path.lost_bytes, packets_sent: stats.udp_tx.datagrams, packets_received: stats.udp_rx.datagrams, - packets_lost: stats.lost_packets, - rtt: path.rtt, + packets_lost: stats.path.lost_packets, + rtt: stats.path.rtt, + bandwidth: stats.path.bandwidth_estimate, } } } @@ -632,7 +624,7 @@ impl std::fmt::Debug for Connection { } } -/// A snapshot of noq-proto's counters in [`web_transport_trait::Stats`] +/// A snapshot of moq-quic's counters in [`web_transport_trait::Stats`] /// shape. struct Stats { bytes_sent: u64, @@ -642,6 +634,8 @@ struct Stats { packets_received: u64, packets_lost: u64, rtt: std::time::Duration, + /// The congestion controller's bandwidth estimate in bytes per second. + bandwidth: Option, } impl web_transport_trait::Stats for Stats { @@ -673,24 +667,22 @@ impl web_transport_trait::Stats for Stats { Some(self.rtt) } - /// Nothing, because noq-proto exposes no delivery or pacing rate. + /// The congestion controller's bandwidth estimate, in bits per second. /// - /// The congestion window over the RTT is not a stand-in for one: BBR - /// deliberately holds about twice the bandwidth-delay product (nearly - /// three times it while starting up), so that number is a multiple of - /// what the path will carry, and moq-net feeds this straight into its - /// bandwidth allocator and PROBE. A missing sample is a state the model - /// already handles (`stats.estimated_send_rate` is an `Option`, and no - /// bandwidth producer is created without one); an inflated one is a rate - /// an encoder will chase. + /// BBR estimates the delivery rate; CUBIC offers none. The congestion + /// window over the RTT is not a stand-in: BBR deliberately holds about + /// twice the bandwidth-delay product, so that number is a multiple of what + /// the path will carry, and moq-net feeds this straight into its bandwidth + /// allocator and PROBE. A missing sample is a state the model already + /// handles; an inflated one is a rate an encoder will chase. fn estimated_send_rate(&self) -> Option { - None + self.bandwidth.map(|bytes| bytes.saturating_mul(8)) } } /// The per-connection task: endpoint events, application events, timers, and /// packets out. Packets in come from the endpoint's demux task, which feeds -/// noq-proto directly and kicks this driver. +/// moq-quic directly and kicks this driver. struct Driver { shared: Shared, socket: Rc, @@ -700,11 +692,11 @@ struct Driver { endpoint: Weak, key: ConnectionHandle, deadline: crate::Timer, - /// Egress staging: noq-proto writes into a `Vec`, so a train is built + /// Egress staging: moq-quic writes into a `Vec`, so a train is built /// here and copied into the socket's registered buffer. scratch: Vec, /// A CONNECTION_CLOSE has been handed to the socket. Not implied by a - /// flush that came back empty: noq paces the close like any other packet, + /// flush that came back empty: moq-quic paces the close like any other packet, /// so it can sit behind a pacing timer after the application asked for it. close_staged: bool, } @@ -732,9 +724,9 @@ impl Driver { } // A closed connection still owes the peer its CONNECTION_CLOSE // (and a retransmit for each packet that arrives after), so the - // driver runs until noq says the drain is over. + // driver runs until moq-quic says the drain is over. if self.shared.conn.borrow().is_drained() { - // A close noq never managed to send (the server's + // A close moq-quic never managed to send (the server's // anti-amplification limit can withhold it) ends here too. self.publish_close(); return Poll::Ready(()); @@ -752,11 +744,12 @@ impl Driver { // polling before the set would leave the firing to wake nobody // (fatal on a dial nobody answers, where no ingress ever re-polls // us). - self.deadline.set(self.shared.conn.borrow_mut().poll_timeout()); + self.deadline + .set(self.shared.conn.borrow_mut().poll_timeout().map(super::to_std)); if self.deadline.poll(waiter).is_pending() { return Poll::Pending; } - self.shared.conn.borrow_mut().handle_timeout(Instant::now()); + self.shared.conn.borrow_mut().handle_timeout(moq_clock::Instant::now()); } } @@ -791,25 +784,24 @@ impl Driver { let mut state = self.shared.state.borrow_mut(); match event { - moq_noq_proto::Event::Connected => { + moq_quic::Event::Connected => { state.established = true; state.establish_waiters.wake(); } - moq_noq_proto::Event::ConnectionLost { reason } => state.fail(reason.into()), - moq_noq_proto::Event::DatagramReceived => state.datagram_recv_waiters.wake(), - moq_noq_proto::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), - moq_noq_proto::Event::HandshakeDataReady => {} - moq_noq_proto::Event::Stream(event) => sweep_stream(&mut state, event), - moq_noq_proto::Event::HandshakeConfirmed - | moq_noq_proto::Event::Path(_) - | moq_noq_proto::Event::NatTraversal(_) => {} + moq_quic::Event::ConnectionLost { reason } => state.fail(reason.into()), + moq_quic::Event::DatagramReceived => state.datagram_recv_waiters.wake(), + moq_quic::Event::DatagramsUnblocked => state.datagram_send_waiters.wake(), + moq_quic::Event::HandshakeDataReady => {} + moq_quic::Event::Stream(event) => sweep_stream(&mut state, event), + // Non-exhaustive upstream; nothing else needs a wakeup. + _ => {} } } } /// Publish the terminal error for a close this side asked for. /// - /// noq raises no event for it, so the driver is what reports it, and + /// moq-quic raises no event for it, so the driver is what reports it, and /// only once the CONNECTION_CLOSE is staged (or the drain is over), since /// an application is free to stop driving the worker the moment /// `poll_closed` resolves. Staged is not delivered: the send is @@ -832,7 +824,7 @@ impl Driver { // Backpressure, or nothing left to stage. Poll::Pending => return Poll::Pending, } - // Requeue behind the other ready tasks. If noq is drained, the next + // Requeue behind the other ready tasks. If moq-quic is drained, the next // poll costs one empty acquire and then parks normally. waiter.waker().wake_by_ref(); Poll::Pending @@ -858,27 +850,26 @@ impl Driver { tx.len() )))); } - let segments = std::num::NonZeroUsize::new(segments).expect("segments was checked above"); self.scratch.clear(); let (transmit, closing) = { let mut conn = self.shared.conn.borrow_mut(); // Nothing to send, or paced; the buffer returns to the pool on drop. - let Some(transmit) = conn.poll_transmit(Instant::now(), segments, &mut self.scratch) else { + let Some(transmit) = conn.poll_transmit(moq_clock::Instant::now(), segments, &mut self.scratch) else { return Poll::Pending; }; - // Once closed, noq transmits nothing but the CONNECTION_CLOSE. + // Once closed, moq-quic transmits nothing but the CONNECTION_CLOSE. (transmit, conn.is_closed()) }; tx[..transmit.size].copy_from_slice(&self.scratch[..transmit.size]); // A lone datagram is its own segment size, and the socket's GSO - // stride has to match what noq actually packed. + // stride has to match what moq-quic actually packed. let transmit = udp::Transmit { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { return Poll::Ready(Err(Error::Io(err.to_string()))); @@ -913,25 +904,25 @@ fn end(state: &mut State, id: StreamId, end: End) { } /// Apply one stream event to the parking tables. -fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { +fn sweep_stream(state: &mut State, event: moq_quic::StreamEvent) { // Waking removes the entry: a still-interested poller re-registers on its // next poll, so the maps only hold streams somebody is parked on. match event { - moq_noq_proto::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), - moq_noq_proto::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), - moq_noq_proto::StreamEvent::Available { .. } => state.open_waiters.wake(), - moq_noq_proto::StreamEvent::Readable { id } => { + moq_quic::StreamEvent::Opened { dir: Dir::Bi } => state.accept_bi_waiters.wake(), + moq_quic::StreamEvent::Opened { dir: Dir::Uni } => state.accept_uni_waiters.wake(), + moq_quic::StreamEvent::Available { .. } => state.open_waiters.wake(), + moq_quic::StreamEvent::Readable { id } => { if let Some(mut waiters) = state.readable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Writable { id } => { + moq_quic::StreamEvent::Writable { id } => { if let Some(mut waiters) = state.writable.remove(&id) { waiters.wake(); } } - moq_noq_proto::StreamEvent::Finished { id } => end(state, id, End::Delivered), - moq_noq_proto::StreamEvent::Stopped { id, error_code } => { + moq_quic::StreamEvent::Finished { id } => end(state, id, End::Delivered), + moq_quic::StreamEvent::Stopped { id, error_code } => { end(state, id, End::Stopped(error_code.into_inner())); // A writer blocked on capacity has to learn it will never come. if let Some(mut waiters) = state.writable.remove(&id) { @@ -943,7 +934,7 @@ fn sweep_stream(state: &mut State, event: moq_noq_proto::StreamEvent) { #[cfg(test)] mod tests { - use moq_noq_proto::Side; + use moq_quic::Side; use super::*; diff --git a/rs/moq-uring/src/quic/noq/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs similarity index 92% rename from rs/moq-uring/src/quic/noq/endpoint.rs rename to rs/moq-uring/src/quic/driver/endpoint.rs index 2a7ff9d8e5..2246efed3e 100644 --- a/rs/moq-uring/src/quic/noq/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -1,6 +1,6 @@ -//! The noq endpoint: one socket, many noq-proto connections. +//! The endpoint: one socket, many moq-quic connections. //! -//! noq-proto's own [`moq_noq_proto::Endpoint`] is the routing table: it parses +//! moq-quic's own [`moq_quic::Endpoint`] is the routing table: it parses //! each datagram, hands it to the connection its destination id names, mints //! and retires ids as peers consume them, and answers an unsupported version //! itself. What is left for us is the socket, the accept backlog, and a driver @@ -17,7 +17,7 @@ use std::task::Poll; use std::time::Instant; use bytes::BytesMut; -use moq_noq_proto::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; +use moq_quic::{ConnectionHandle, DatagramEvent, Incoming, Transmit}; use rustc_hash::FxHashMap; use super::super::{Error, endpoint::Config}; @@ -42,11 +42,11 @@ pub(crate) struct Inner { socket: Rc, local: SocketAddr, /// The routing table, and the server configuration it accepts with. - endpoint: RefCell, + endpoint: RefCell, accepting: RefCell>, /// Every live connection, looked up per received datagram. /// - /// FxHash rather than SipHash: noq-proto hands out the handle, and it + /// FxHash rather than SipHash: moq-quic hands out the handle, and it /// owns connection-id routing (and its hasher choices) itself, so nothing /// a peer picks reaches this map. conns: RefCell>, @@ -91,7 +91,7 @@ impl Endpoint { let server = match &config.server { Some(server) => { let mut server = super::server_config(server)?; - // noq buffers half-open handshakes itself, so it enforces + // moq-quic buffers half-open handshakes itself, so it enforces // the same bound the accept queue does. server.max_incoming(config.backlog); Some(Arc::new(server)) @@ -101,7 +101,7 @@ impl Endpoint { let accepting = server.is_some().then(|| Accepting { queue: VecDeque::new() }); // MTU discovery is off (the GSO pool sends fixed SEGMENT datagrams), // so the endpoint has no reason to allow it either. - let endpoint = moq_noq_proto::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); + let endpoint = moq_quic::Endpoint::new(super::endpoint_config(socket.shard())?, server, false); let inner = Rc::new(Inner { activity: config.activity, @@ -165,7 +165,7 @@ impl Endpoint { .inner .endpoint .borrow_mut() - .connect(Instant::now(), client, config.peer, &config.server_name) + .connect(moq_clock::Instant::now(), client, config.peer, &config.server_name) .map_err(|err| Error::Quic(err.to_string()))?; let shared = self.inner.launch(key, conn); // Flush the Initial flight; the demux task takes it from here. @@ -246,7 +246,7 @@ impl Inner { /// Route every datagram in one receive, then kick the connections fed. fn demux(self: &Rc, packet: &mut udp::Packet) { let from = packet.from(); - let ecn = packet.ecn().map(super::ecn_to_noq); + let ecn = packet.ecn().map(super::ecn_to_quic); // Where the endpoint writes its own answers (version negotiation, // retry, a refusal), reused across the whole receive. let mut buf = Vec::new(); @@ -261,8 +261,9 @@ impl Inner { // arrived on, so the four-tuple stays half known, exactly as it // does on a platform without `IP_PKTINFO`. let event = self.endpoint.borrow_mut().handle( - Instant::now(), - from.into(), + moq_clock::Instant::now(), + from, + None, ecn, BytesMut::from(&segment[..]), &mut buf, @@ -339,9 +340,14 @@ impl Inner { return None; }; buf.clear(); - let now = Instant::now(); - let deadline = self.handshake.timeout.and_then(|timeout| now.checked_add(timeout)); - let accepted = self.endpoint.borrow_mut().accept(incoming, now, buf, None); + let deadline = self + .handshake + .timeout + .and_then(|timeout| Instant::now().checked_add(timeout)); + let accepted = self + .endpoint + .borrow_mut() + .accept(incoming, moq_clock::Instant::now(), buf, None); let (key, conn) = match accepted { Ok(accepted) => accepted, Err(err) => { @@ -402,7 +408,7 @@ impl Inner { to: transmit.destination, len: transmit.size, segment: transmit.segment_size.unwrap_or(transmit.size), - ecn: transmit.ecn.map(super::ecn_from_noq), + ecn: transmit.ecn.map(super::ecn_from_quic), }; if let Err(err) = tx.send(transmit) { tracing::debug!(%err, "failed to send an endpoint response"); @@ -410,7 +416,7 @@ impl Inner { } /// Register `conn`, spawn its driver, and arrange its teardown. - fn launch(self: &Rc, key: ConnectionHandle, conn: moq_noq_proto::Connection) -> connection::Shared { + fn launch(self: &Rc, key: ConnectionHandle, conn: moq_quic::Connection) -> connection::Shared { let (shared, driver) = connection::launch(&self.owner, self.socket.clone(), Rc::downgrade(self), key, conn); self.conns.borrow_mut().insert(key, shared.clone()); @@ -427,8 +433,8 @@ impl Inner { pub(crate) fn on_connection_event( &self, key: ConnectionHandle, - event: moq_noq_proto::EndpointEvent, - ) -> Option { + event: moq_quic::EndpointEvent, + ) -> Option { self.endpoint.borrow_mut().handle_event(key, event) } diff --git a/rs/moq-uring/src/quic/noq/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs similarity index 79% rename from rs/moq-uring/src/quic/noq/mod.rs rename to rs/moq-uring/src/quic/driver/mod.rs index 00d457bcdb..b1dc8fc86d 100644 --- a/rs/moq-uring/src/quic/noq/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -1,13 +1,13 @@ -//! The rustls-backed noq-proto stack on the worker's UDP path. +//! The rustls-backed moq-quic stack on the worker's UDP path. //! //! TLS goes through rustls, which is what the rest of this workspace speaks, //! so a build selecting this backend links one TLS stack instead of two. //! -//! [`moq_noq_proto::Endpoint`] holds -//! the connection-id routing table, mints and retires ids, answers unsupported -//! versions, and buffers half-open handshakes. So [`Endpoint`] here is mostly -//! the socket plumbing around it, and the parts that are ours (the accept -//! backlog, shard steering, and the driver task per connection. +//! [`moq_quic::Endpoint`] holds the connection-id routing table, mints and +//! retires ids, answers unsupported versions, and buffers half-open +//! handshakes. So [`Endpoint`] here is mostly the socket plumbing around it, +//! plus the parts that are ours: the accept backlog, shard steering, and the +//! driver task per connection. mod connection; mod endpoint; @@ -21,7 +21,7 @@ pub(crate) use connection::{End, Shared}; use std::sync::Arc; -use moq_noq_proto::crypto::rustls::{QuicClientConfig, QuicServerConfig}; +use moq_quic::crypto::rustls::{QuicClientConfig, QuicServerConfig}; use rustls::pki_types::pem::PemObject; use rustls::pki_types::{CertificateDer, PrivateKeyDer}; @@ -37,25 +37,34 @@ const CONNECTION_WINDOW: u32 = 16 * 1024 * 1024; /// How many datagrams to buffer in each direction. const DATAGRAM_WINDOW: usize = 64 * SEGMENT; -fn ecn_to_noq(ecn: udp::Ecn) -> moq_noq_proto::EcnCodepoint { +fn ecn_to_quic(ecn: udp::Ecn) -> moq_quic::EcnCodepoint { match ecn { - udp::Ecn::Ect0 => moq_noq_proto::EcnCodepoint::Ect0, - udp::Ecn::Ect1 => moq_noq_proto::EcnCodepoint::Ect1, - udp::Ecn::Ce => moq_noq_proto::EcnCodepoint::Ce, + udp::Ecn::Ect0 => moq_quic::EcnCodepoint::Ect0, + udp::Ecn::Ect1 => moq_quic::EcnCodepoint::Ect1, + udp::Ecn::Ce => moq_quic::EcnCodepoint::Ce, } } -fn ecn_from_noq(ecn: moq_noq_proto::EcnCodepoint) -> udp::Ecn { +fn ecn_from_quic(ecn: moq_quic::EcnCodepoint) -> udp::Ecn { match ecn { - moq_noq_proto::EcnCodepoint::Ect0 => udp::Ecn::Ect0, - moq_noq_proto::EcnCodepoint::Ect1 => udp::Ecn::Ect1, - moq_noq_proto::EcnCodepoint::Ce => udp::Ecn::Ce, + moq_quic::EcnCodepoint::Ect0 => udp::Ecn::Ect0, + moq_quic::EcnCodepoint::Ect1 => udp::Ecn::Ect1, + moq_quic::EcnCodepoint::Ce => udp::Ecn::Ce, } } -impl From for Error { - fn from(err: moq_noq_proto::ConnectionError) -> Self { - use moq_noq_proto::ConnectionError; +/// Map a moq-quic deadline onto the worker's timers. +/// +/// moq-quic counts on moq-clock's instants while the worker's timers still take +/// std's, until the worker becomes a moq-clock backend. A deadline already past +/// maps to now, which fires at once either way. +fn to_std(at: moq_clock::Instant) -> std::time::Instant { + std::time::Instant::now() + at.saturating_duration_since(moq_clock::Instant::now()) +} + +impl From for Error { + fn from(err: moq_quic::ConnectionError) -> Self { + use moq_quic::ConnectionError; match err { ConnectionError::ApplicationClosed(close) => Self::App { code: close.error_code.into_inner(), @@ -91,10 +100,8 @@ fn provider() -> Arc { /// The endpoint-wide configuration: how connection ids are minted, and the /// largest datagram we tell peers we can receive. -pub(crate) fn endpoint_config( - shard: Option, -) -> Result, Error> { - let mut config = moq_noq_proto::EndpointConfig::default(); +pub(crate) fn endpoint_config(shard: Option) -> Result, Error> { + let mut config = moq_quic::EndpointConfig::default(); config.cid_generator(Arc::new(move || Box::new(Cids { shard }))); config .max_udp_payload_size(SEGMENT as u16) @@ -104,7 +111,7 @@ pub(crate) fn endpoint_config( /// Mints the endpoint's connection ids, steering prefix included. /// -/// noq-proto asks its generator for every id it issues, dials and rotations +/// moq-quic asks its generator for every id it issues, dials and rotations /// alike, so this is the one place the reuseport group's byte has to be /// stamped. #[derive(Debug)] @@ -112,9 +119,9 @@ struct Cids { shard: Option, } -impl moq_noq_proto::ConnectionIdGenerator for Cids { - fn generate_cid(&mut self) -> moq_noq_proto::ConnectionId { - moq_noq_proto::ConnectionId::new(&super::endpoint::cid(self.shard)) +impl moq_quic::ConnectionIdGenerator for Cids { + fn generate_cid(&mut self) -> moq_quic::ConnectionId { + moq_quic::ConnectionId::new(&super::endpoint::cid(self.shard)) } fn cid_len(&self) -> usize { @@ -127,7 +134,7 @@ impl moq_noq_proto::ConnectionIdGenerator for Cids { } /// Dial as `config` says. -pub(crate) fn client_config(config: &client::Config) -> Result { +pub(crate) fn client_config(config: &client::Config) -> Result { let provider = provider(); let builder = rustls::ClientConfig::builder_with_provider(provider.clone()) .with_protocol_versions(&[&rustls::version::TLS13]) @@ -170,21 +177,21 @@ pub(crate) fn client_config(config: &client::Config) -> Result Result { +pub(crate) fn server_config(config: &server::Config) -> Result { let crypto = QuicServerConfig::try_from(config.tls.clone()).map_err(|err| Error::Tls(err.to_string()))?; - let mut server = moq_noq_proto::ServerConfig::with_crypto(Arc::new(crypto)); + let mut server = moq_quic::ServerConfig::with_crypto(Arc::new(crypto)); let transport = transport_config(&config.transport)?; #[cfg(feature = "qlog")] - let transport = with_qlog(transport, &config.transport); + let transport = with_qlog(transport, &config.transport, qlog::Side::Server); server.transport_config(Arc::new(transport)); Ok(server) } @@ -233,53 +240,36 @@ pub(crate) fn identity_server_config( /// Attach the configured qlog sink, if any. /// -/// Noq asks a factory per connection, so each gets a file of its own. +/// moq-quic holds one sink per transport config, which is per endpoint on the +/// accept side and per dial on the connect side; its events carry the +/// connection's qlog `group_id` either way. #[cfg(feature = "qlog")] -fn with_qlog(mut transport: moq_noq_proto::TransportConfig, config: &Transport) -> moq_noq_proto::TransportConfig { +fn with_qlog( + mut transport: moq_quic::TransportConfig, + config: &Transport, + side: qlog::Side, +) -> moq_quic::TransportConfig { let Some(sink) = config.qlog.clone() else { return transport; }; - transport.qlog_factory(Arc::new(Traces { sink })); + let mut qlog = moq_quic::QlogConfig::default(); + qlog.writer(sink.trace(side)); + transport.qlog_stream(qlog.into_stream()); transport } -/// Opens one trace per connection, which is what noq-proto's factory hook is -/// for. -#[cfg(feature = "qlog")] -#[derive(Debug)] -struct Traces { - sink: qlog::Sink, -} - -#[cfg(feature = "qlog")] -impl moq_noq_proto::QlogFactory for Traces { - fn for_connection( - &self, - side: moq_noq_proto::Side, - _remote: std::net::SocketAddr, - initial_dst_cid: moq_noq_proto::ConnectionId, - _now: std::time::Instant, - ) -> Option { - let side = match side { - moq_noq_proto::Side::Client => qlog::Side::Client, - moq_noq_proto::Side::Server => qlog::Side::Server, - }; - Some(moq_noq_proto::QlogConfig::new(self.sink.trace(&initial_dst_cid, side))) - } -} - /// The per-connection knobs both roles share. -fn transport_config(config: &Transport) -> Result { - use moq_noq_proto::VarInt; +fn transport_config(config: &Transport) -> Result { + use moq_quic::VarInt; - let idle = moq_noq_proto::IdleTimeout::try_from(config.idle_timeout) + let idle = moq_quic::IdleTimeout::try_from(config.idle_timeout) .map_err(|_| Error::Quic(format!("idle timeout out of range: {:?}", config.idle_timeout)))?; let streams = VarInt::from_u64(config.max_streams) .map_err(|_| Error::Quic(format!("stream limit out of range: {}", config.max_streams)))?; - let mut transport = moq_noq_proto::TransportConfig::default(); + let mut transport = moq_quic::TransportConfig::default(); transport.max_idle_timeout(Some(idle)); transport.keep_alive_interval(config.keep_alive); transport.max_concurrent_bidi_streams(streams); @@ -301,22 +291,22 @@ fn transport_config(config: &Transport) -> Result Arc::new(moq_noq_proto::congestion::CubicConfig::default()) - as Arc, - Congestion::Delay => Arc::new(moq_noq_proto::congestion::Bbr3Config::default()), + Congestion::Loss => Arc::new(moq_quic::congestion::CubicConfig::default()) + as Arc, + Congestion::Delay => Arc::new(moq_quic::congestion::Bbr3Config::default()), }); Ok(transport) } /// A receive window on the wire is a QUIC varint. One that does not fit cannot /// be applied, so it is an error rather than a smaller window than was asked. -fn varint_window(name: &str, value: u64) -> Result { - moq_noq_proto::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) +fn varint_window(name: &str, value: u64) -> Result { + moq_quic::VarInt::from_u64(value).map_err(|_| Error::Quic(format!("{name} out of range: {value}"))) } /// ALPN protocols on the wire, which is a length-prefixed list of byte @@ -410,10 +400,10 @@ impl rustls::client::danger::ServerCertVerifier for NoVerify { mod tests { use super::{CONNECTION_WINDOW, STREAM_WINDOW, Transport, transport_config}; - /// noq keeps the windows private, but its `Debug` prints them. The leading + /// moq-quic keeps the windows private, but its `Debug` prints them. The leading /// space keeps `receive_window` from matching `stream_receive_window`, and /// the trailing comma keeps a shorter value from matching a longer one. - fn has_window(config: &moq_noq_proto::TransportConfig, name: &str, value: u64) -> bool { + fn has_window(config: &moq_quic::TransportConfig, name: &str, value: u64) -> bool { format!("{config:?}").contains(&format!(" {name}: {value},")) } diff --git a/rs/moq-uring/src/quic/noq/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs similarity index 90% rename from rs/moq-uring/src/quic/noq/stream.rs rename to rs/moq-uring/src/quic/driver/stream.rs index bb9d90110e..e415f57f30 100644 --- a/rs/moq-uring/src/quic/noq/stream.rs +++ b/rs/moq-uring/src/quic/driver/stream.rs @@ -1,6 +1,6 @@ -//! Stream handles: thin, direct calls into the shared noq-proto connection. +//! Stream handles: thin, direct calls into the shared moq-quic connection. //! -//! Single-threaded sans-IO means a write goes straight into noq's send +//! Single-threaded sans-IO means a write goes straight into moq-quic's send //! queue (no staging copy) and a read comes straight out of its reassembly //! buffer; the handles just kick the driver so egress reaches the wire and //! park on the per-stream waiter lists the driver wakes. @@ -8,7 +8,7 @@ use std::task::{Context, Poll}; use bytes::{Buf, Bytes, BytesMut}; -use moq_noq_proto::{StreamId, VarInt}; +use moq_quic::{StreamId, VarInt}; use super::super::Error; use super::{End, Shared}; @@ -51,7 +51,7 @@ impl SendStream { self.fin || self.reset } - /// Queue as much of `buf` as noq will take right now, without parking. + /// Queue as much of `buf` as moq-quic will take right now, without parking. /// Best-effort, for the close path where nobody is left to poll. pub(crate) fn try_write(&mut self, buf: &[u8]) -> usize { if self.fin || self.reset { @@ -105,14 +105,14 @@ impl web_transport_trait::poll::SendStream for SendStream { self.shared.kick(); Poll::Ready(Ok(n)) } - // No capacity right now; the driver wakes us when noq reports + // No capacity right now; the driver wakes us when moq-quic reports // the stream writable. - Err(moq_noq_proto::WriteError::Blocked) => { + Err(moq_quic::WriteError::Blocked) => { self.shared.park_writable(self.id, waiter); Poll::Pending } - Err(moq_noq_proto::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), - Err(moq_noq_proto::WriteError::ClosedStream) => { + Err(moq_quic::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), + Err(moq_quic::WriteError::ClosedStream) => { Poll::Ready(Err(Error::Quic("stream already finished".to_string()))) } } @@ -120,7 +120,7 @@ impl web_transport_trait::poll::SendStream for SendStream { fn set_priority(&mut self, order: i32) { // The trait (like W3C sendOrder) sends HIGHER values first, and so - // does noq. + // does moq-quic. let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_priority(order); } @@ -133,12 +133,12 @@ impl web_transport_trait::poll::SendStream for SendStream { // A STOP_SENDING beat us here. Carry the code like `poll_write` // does, or `moq_net::Error::from_transport` cannot decode a // routine cancellation. - Err(moq_noq_proto::FinishError::Stopped(code)) => { + Err(moq_quic::FinishError::Stopped(code)) => { self.reset = true; return Err(Error::Stop(code.into_inner())); } // Already finished or reset, so the FIN it wanted is out. - Err(moq_noq_proto::FinishError::ClosedStream) => {} + Err(moq_quic::FinishError::ClosedStream) => {} } self.fin = true; self.shared.kick(); @@ -154,7 +154,7 @@ impl web_transport_trait::poll::SendStream for SendStream { if self.reset { return Poll::Ready(Ok(())); } - // noq reports a send stream's end as an event, which the driver + // moq-quic reports a send stream's end as an event, which the driver // records: an acknowledged FIN, or the peer's STOP_SENDING. match self.shared.ended(self.id) { Some(End::Stopped(code)) => Poll::Ready(Err(Error::Stop(code))), @@ -199,7 +199,7 @@ const READ_AHEAD: usize = 64 * 1024; /// How much to ask for per read-ahead chunk. const READ_CHUNK: usize = 8 * 1024; -/// One read out of noq's reassembly buffer. +/// One read out of moq-quic's reassembly buffer. enum Read { /// Bytes, at most as many as were asked for. Chunk(Bytes), @@ -216,12 +216,12 @@ pub struct RecvStream { shared: Shared, id: StreamId, park: kio::Park, - /// Every byte up to the FIN was read out of noq; reads report the end + /// Every byte up to the FIN was read out of moq-quic; reads report the end /// once `backlog` is drained too. finished: bool, /// We stopped the stream; no more reads matter. stopped: bool, - /// Bytes `poll_closed` read ahead, handed to `poll_read` before noq's. + /// Bytes `poll_closed` read ahead, handed to `poll_read` before moq-quic's. backlog: BytesMut, } @@ -261,7 +261,7 @@ impl RecvStream { self.shared.kick(); } - /// Take up to `max` bytes out of noq's reassembly buffer. + /// Take up to `max` bytes out of moq-quic's reassembly buffer. /// /// Reading is what returns the peer's flow control credit, so a read that /// owes it a frame kicks the driver. @@ -279,8 +279,8 @@ impl RecvStream { let read = match chunks.next(max) { Ok(Some(chunk)) => Read::Chunk(chunk.bytes), Ok(None) => Read::Finished, - Err(moq_noq_proto::ReadError::Blocked) => Read::Blocked, - Err(moq_noq_proto::ReadError::Reset(code)) => Read::Reset(code.into_inner()), + Err(moq_quic::ReadError::Blocked) => Read::Blocked, + Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), }; let transmit = chunks.finalize().should_transmit(); drop(conn); @@ -351,7 +351,7 @@ impl web_transport_trait::poll::RecvStream for RecvStream { if self.finished || self.stopped { return Poll::Ready(Ok(())); } - // The FIN sits behind whatever the peer sent before it, and noq only + // The FIN sits behind whatever the peer sent before it, and moq-quic only // reports the stream finished once that is read out. Waiting on // readability alone would park behind bytes nobody is reading, so read // ahead into the backlog `poll_read` serves first: this watch resolves diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index a21d821425..a312ab9584 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -25,7 +25,7 @@ use super::server; -pub use super::noq::Endpoint; +pub use super::driver::Endpoint; /// Every connection id this endpoint issues is this long: long enough to be /// unguessable per socket, and fixed because a short header does not encode diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index 9ed2b81116..f6e84c8959 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -18,7 +18,7 @@ //! [`web`] layer's HTTP/3 CONNECT handshake on top of the same adapter, with //! [`web::Session`] as the one transport type covering both. //! -//! Noq is the sans-IO QUIC stack underneath the worker. +//! moq-quic is the sans-IO QUIC stack underneath the worker. pub mod client; pub mod endpoint; @@ -28,10 +28,10 @@ pub mod qlog; pub mod server; pub mod web; -mod noq; +mod driver; +pub use driver::{Connection, RecvStream, SendStream}; pub use endpoint::Endpoint; -pub use noq::{Connection, RecvStream, SendStream}; /// The QUIC payload size every full datagram in a GSO train uses, and the /// stride GRO coalesces with. @@ -77,7 +77,7 @@ impl Identity { client_auth: server::ClientAuth, ) -> Result, Error> { let alpn = alpn.iter().map(|proto| proto.as_ref().as_bytes().to_vec()).collect(); - noq::identity_server_config(self, alpn, &client_auth) + driver::identity_server_config(self, alpn, &client_auth) } /// The PEM certificate chain being presented, for a caller that publishes @@ -135,8 +135,9 @@ pub struct Transport { pub keep_alive: Option, /// Where to write qlog traces, or `None` (the default) to write none. /// - /// Noq takes one sink per configuration and writes one file per connection, - /// named from that connection's Initial destination connection ID. + /// moq-quic takes one sink per transport config, so this writes one file + /// per endpoint on the accept side and per dial on the connect side, each + /// event tagged with the qlog `group_id` of the connection it belongs to. /// /// Only compiled with the `qlog` feature, so a build without it cannot ask /// for traces the backends would not produce. @@ -162,9 +163,9 @@ impl Default for Transport { /// The congestion control family a connection runs. /// -/// Noq uses CUBIC for loss-based control and BBRv3 for delay-based control. +/// CUBIC is used for loss-based control and BBRv3 for delay-based control. /// -/// The default is [`Loss`](Self::Loss), which is what noq runs +/// The default is [`Loss`](Self::Loss), which is what moq-quic runs /// unasked. An application carrying live media wants [`Delay`](Self::Delay) /// and should say so; the relay does. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/rs/moq-uring/src/quic/qlog.rs b/rs/moq-uring/src/quic/qlog.rs index 0fe2d8250c..e3e412763c 100644 --- a/rs/moq-uring/src/quic/qlog.rs +++ b/rs/moq-uring/src/quic/qlog.rs @@ -8,7 +8,7 @@ //! the worker would stall every connection sharing its core for as long as //! the filesystem takes. //! -//! One file per connection, in the qlog JSON-SEQ format the tokio listener already +//! One file per trace, in the qlog JSON-SEQ format the tokio listener already //! writes, so an existing workflow reads both without knowing which runtime //! produced them. @@ -117,23 +117,14 @@ impl Sink { }) } - /// A writer for one connection's trace, named from its connection id. - pub(crate) fn trace(&self, cid: &[u8], side: Side) -> Box { - self.open(cid, side) - } - - fn open(&self, cid: &[u8], side: Side) -> Box { - use std::fmt::Write as _; - + /// A writer for one trace, which every connection sharing a transport + /// config writes into, each event tagged with its connection's qlog + /// `group_id`. + pub(crate) fn trace(&self, side: Side) -> Box { let inner = self.inner.clone(); + // The trace slot keeps each file unique within the sink. let file = inner.next.fetch_add(1, Ordering::Relaxed); - // The trace slot stays unique within the sink even when a peer reuses - // the same Initial destination connection id. - let cid = cid.iter().fold(String::with_capacity(cid.len() * 2), |mut id, byte| { - let _ = write!(id, "{byte:02x}"); - id - }); - let id = format!("connection{file}-{cid}"); + let id = format!("trace{file}"); let path = inner.dir.join(format!( "moq-{}-{}-{}-{id}-{}.qlog", inner.started, @@ -369,8 +360,8 @@ mod tests { let first = Sink::directory(dir.path()).expect("first sink"); let second = Sink::directory(dir.path()).expect("second sink"); - let mut first_trace = first.trace(b"first", Side::Server); - let mut second_trace = second.trace(b"second", Side::Server); + let mut first_trace = first.trace(Side::Server); + let mut second_trace = second.trace(Side::Server); first_trace.write_all(b"first").expect("write first trace"); second_trace.write_all(b"second").expect("write second trace"); drop(first_trace); @@ -387,14 +378,14 @@ mod tests { } #[test] - fn reused_connection_ids_use_distinct_files() { + fn traces_use_distinct_files() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let mut first = sink.trace(b"same", Side::Server); + let mut first = sink.trace(Side::Server); first.write_all(b"first").expect("write first trace"); drop(first); - let mut second = sink.trace(b"same", Side::Server); + let mut second = sink.trace(Side::Server); second.write_all(b"second").expect("write second trace"); drop(second); drop(sink); @@ -411,7 +402,7 @@ mod tests { fn an_open_trace_accounts_for_staging_and_close() { let dir = tempfile::tempdir().expect("temp dir"); let sink = Sink::directory(dir.path()).expect("sink"); - let trace = sink.trace(b"accounting", Side::Server); + let trace = sink.trace(Side::Server); let accounted = CHUNK + std::mem::size_of::(); for _ in 0..10_000 { @@ -433,7 +424,7 @@ mod tests { let held = MEMORY_MAX - CHUNK + 1; assert!(sink.inner.reserve(held)); - let mut trace = sink.trace(b"dropped", Side::Server); + let mut trace = sink.trace(Side::Server); trace.write_all(b"dropped").expect("write dropped trace"); drop(trace); sink.inner.release(held); diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 915724d195..192592f9b2 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -1012,7 +1012,7 @@ impl std::fmt::Debug for RecvStream { /// Unmap a raw-QUIC stream code that arrived in the HTTP/3 code space. /// -/// Older `web-transport-moq` raw sessions map RESET_STREAM and STOP_SENDING +/// Older moq-tokio raw sessions map RESET_STREAM and STOP_SENDING /// codes as if they were WebTransport. No `u32` application code reaches that /// range, so a code in it can only be the mapped form. Anything else is the /// peer's own code and passes through. diff --git a/rs/moq-uring/tests/echo.rs b/rs/moq-uring/tests/echo.rs index f50825b136..2f2fc623d9 100644 --- a/rs/moq-uring/tests/echo.rs +++ b/rs/moq-uring/tests/echo.rs @@ -1,6 +1,6 @@ -//! A tokio noq peer echoing through the io_uring WebTransport server. +//! A tokio QUIC peer echoing through the io_uring WebTransport server. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -48,7 +48,7 @@ async fn write_finish(send: &mut quic::web::SendStream, mut buf: &[u8]) { } #[test] -fn echo_noq_peer() { +fn echo_tokio_peer() { let Some(mut worker) = worker() else { return }; let handle = worker.handle(); let certs = support::certs().expect("certificates"); @@ -56,7 +56,7 @@ fn echo_noq_peer() { let server = quic::server::Config::new( quic::Identity::open(&certs.cert, &certs.key) .expect("identity") - .server(&[web_transport_moq::ALPN], quic::server::ClientAuth::None) + .server(&[moq_tokio::quic::web::ALPN], quic::server::ClientAuth::None) .expect("tls"), ); let socket = handle @@ -78,11 +78,11 @@ fn echo_noq_peer() { .build() .expect("runtime"); runtime.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = web_transport_moq::proto::ConnectRequest::new( + let request = moq_tokio::quic::web::proto::ConnectRequest::new( url::Url::parse(&format!("https://{addr}/echo")).expect("url"), ); let session = client.connect(request).await.expect("connect"); diff --git a/rs/moq-uring/tests/ecn.rs b/rs/moq-uring/tests/ecn.rs index 5f2c311796..1f89c4e7a7 100644 --- a/rs/moq-uring/tests/ecn.rs +++ b/rs/moq-uring/tests/ecn.rs @@ -2,7 +2,7 @@ //! the peer's `Packet::ecn` reads back, over IPv4 and IPv6, as a lone //! datagram and as a GSO train, with CE surviving too. Without the TOS and //! traffic-class control messages the peer sees no mark, its ACKs carry no -//! ECN counts, and noq disables ECN on the first one. +//! ECN counts, and moq-quic disables ECN on the first one. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. diff --git a/rs/moq-uring/tests/endpoint.rs b/rs/moq-uring/tests/endpoint.rs index 899bb2cd9b..a6e41b28e6 100644 --- a/rs/moq-uring/tests/endpoint.rs +++ b/rs/moq-uring/tests/endpoint.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -200,7 +200,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { // Junk first: it must not be fatal, and whatever answer it draws // must be smaller than it was, so the socket is never an - // amplifier. Noq answers a packet naming no connection with a + // amplifier. moq-quic answers a packet naming no connection with a // stateless reset, which is what tells a // peer holding stale state to give up. let junk = [0u8; 64]; @@ -276,7 +276,7 @@ fn unsupported_version_is_negotiated_only_by_servers() { assert!( versions .iter() - .any(|version| moq_noq_proto::DEFAULT_SUPPORTED_VERSIONS.contains(version)), + .any(|version| moq_quic::DEFAULT_SUPPORTED_VERSIONS.contains(version)), "the supported version is offered: {versions:?}" ); drop((endpoint, dial_only)); @@ -382,7 +382,7 @@ fn an_unanswered_dial_times_out() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The dial ends at noq's handshake idle timeout (10s), not the idle timeout. + // The dial ends at moq-quic's handshake idle timeout (10s), not the idle timeout. let dial = dial_config(hole.local_addr().expect("addr")); let result = worker @@ -419,7 +419,7 @@ fn the_backlog_bounds_pending_handshakes() { let client_sock = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - // The server never answers, so the dial ends at noq's handshake idle + // The server never answers, so the dial ends at moq-quic's handshake idle // timeout (10s). let dial = dial_config(server); diff --git a/rs/moq-uring/tests/identity.rs b/rs/moq-uring/tests/identity.rs index 4a1673e2bc..18bf14da98 100644 --- a/rs/moq-uring/tests/identity.rs +++ b/rs/moq-uring/tests/identity.rs @@ -4,7 +4,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/qlog.rs b/rs/moq-uring/tests/qlog.rs index 9a1ff28ef6..c732b74bb5 100644 --- a/rs/moq-uring/tests/qlog.rs +++ b/rs/moq-uring/tests/qlog.rs @@ -2,14 +2,12 @@ //! tooling can read. //! //! The point of the test is the two halves the relay depends on: the sink -//! actually produces a file for the connection (per connection on noq and -//! per connection on noq, and -//! every record in it is JSON, so a reader is not handed a truncated or +//! actually produces a file for each end of the connection, and every record in it is JSON, so a reader is not handed a truncated or //! interleaved trace. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "qlog", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "qlog", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/session.rs b/rs/moq-uring/tests/session.rs index 22fef4fb5d..5571c8e587 100644 --- a/rs/moq-uring/tests/session.rs +++ b/rs/moq-uring/tests/session.rs @@ -8,7 +8,7 @@ //! Origin drivers run through Tokio's explicit-time adapter while session //! drivers use the worker's clock and timer. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -308,7 +308,7 @@ fn two_lite_sessions_share_the_server_socket() { } /// The client verifies for real, trusting only the certificate the server -/// presents: nothing handshakes unless the configured roots reach noq. +/// presents: nothing handshakes unless the configured roots reach moq-quic. #[test] fn configured_roots_verify_the_server() { let Some(mut worker) = worker() else { return }; diff --git a/rs/moq-uring/tests/teardown.rs b/rs/moq-uring/tests/teardown.rs index ad1383a5e9..26a8993b95 100644 --- a/rs/moq-uring/tests/teardown.rs +++ b/rs/moq-uring/tests/teardown.rs @@ -10,7 +10,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/rs/moq-uring/tests/web.rs b/rs/moq-uring/tests/web.rs index 0cfef78d1c..2fbf2157ba 100644 --- a/rs/moq-uring/tests/web.rs +++ b/rs/moq-uring/tests/web.rs @@ -1,11 +1,11 @@ -//! WebTransport interop: the reference tokio stack (`web-transport-moq`, +//! WebTransport interop: the reference tokio stack (`moq_tokio::quic::web`, //! what browsers interop with) dials the uring server. One test hand-drives //! streams and datagrams through the H3 framing; the other runs a whole //! moq-lite session over it, which is exactly the browser-to-relay path. //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; @@ -53,9 +53,9 @@ fn h3_endpoint(handle: &moq_uring::Handle, certs: &support::Certs) -> quic::Endp } /// The tokio-side client, in its own runtime on its own thread. -fn noq_client( +fn tokio_client( url: String, - body: impl FnOnce(web_transport_moq::Session) -> ClientFuture + Send + 'static, + body: impl FnOnce(moq_tokio::quic::web::Session) -> ClientFuture + Send + 'static, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { // May already be installed by a sibling test; either way one exists. @@ -65,12 +65,12 @@ fn noq_client( .build() .expect("tokio runtime"); rt.block_on(async move { - let client = web_transport_moq::ClientBuilder::new() + let client = moq_tokio::quic::web::ClientBuilder::new() .dangerous() .with_no_certificate_verification() .expect("client"); - let request = - web_transport_moq::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")).with_protocol(PROTO); + let request = moq_tokio::quic::web::proto::ConnectRequest::new(url::Url::parse(&url).expect("url")) + .with_protocol(PROTO); let session = client.connect(request).await.expect("connect"); body(session).await; }); @@ -79,8 +79,8 @@ fn noq_client( type ClientFuture = std::pin::Pin + Send>>; -/// Read a noq-side stream to its end. -async fn read_all(recv: &mut web_transport_moq::RecvStream) -> Vec { +/// Read a tokio-side stream to its end. +async fn read_all(recv: &mut moq_tokio::quic::web::RecvStream) -> Vec { let mut out = Vec::new(); let mut buf = [0u8; 4096]; while let Some(n) = recv.read(&mut buf).await.expect("read") { @@ -126,7 +126,7 @@ fn webtransport_echo_end_to_end() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/echo?token=abc"), |session| { + let client = tokio_client(format!("https://{addr}/echo?token=abc"), |session| { Box::pin(async move { // Bidirectional echo. let (mut send, mut recv) = session.open_bi().await.expect("open_bi"); @@ -239,7 +239,7 @@ fn lite_session_over_webtransport() { let endpoint = h3_endpoint(&handle, &certs); let addr = endpoint.local_addr(); - let client = noq_client(format!("https://{addr}/"), |session| { + let client = tokio_client(format!("https://{addr}/"), |session| { Box::pin(async move { assert_eq!(session.protocol(), Some(PROTO), "negotiated subprotocol"); let (sub_origin, sub_driver) = origin::Producer::new(origin::Config::default()); @@ -339,7 +339,7 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future std::thread::JoinHandle { +fn tokio_client_err(url: String) -> std::thread::JoinHandle { std::thread::spawn(move || { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let rt = tokio::runtime::Builder::new_current_thread() @@ -347,12 +347,12 @@ fn noq_client_err(url: String) -> std::thread::JoinHandle break err, } }; - assert!(matches!(err, web_transport_moq::WriteError::Stopped(0)), "got {err:?}"); + assert!( + matches!(err, moq_tokio::quic::web::WriteError::Stopped(0)), + "got {err:?}" + ); session.close(CLOSE_CODE, CLOSE_REASON.as_bytes()); session.closed().await; diff --git a/rs/moq-uring/tests/workers.rs b/rs/moq-uring/tests/workers.rs index d832d4c0a4..e427de24b4 100644 --- a/rs/moq-uring/tests/workers.rs +++ b/rs/moq-uring/tests/workers.rs @@ -9,7 +9,7 @@ //! //! Unsupported local kernels skip loudly; MOQ_STRICT makes CI refuse them. -#![cfg(all(target_os = "linux", feature = "noq"))] +#![cfg(all(target_os = "linux", feature = "quic"))] #[path = "support.rs"] mod support; diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index 8c6e634b14..3724bfd794 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -128,7 +128,7 @@ def test_media_keeps_dependency_contracts_and_vpx_tests(self): CARGO_STUB = r"""#!/bin/bash printf '%s\n' "$*" >> "$FEATURE_CALLS" -if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features noq' ]]; then +if [[ "$*" == 'check --locked -p moq-tokio --no-default-features --features quic' ]]; then echo 'a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' >&2 exit 1 fi diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh index 8727b4a233..555ef55272 100755 --- a/sh/rs/tokio-features.sh +++ b/sh/rs/tokio-features.sh @@ -19,8 +19,8 @@ cargo clippy --locked -p moq-tokio --all-features -- -D warnings # Only a feature-owner edit needs every isolated permutation on a PR. Shared # dependencies still compile at both extremes; Nightly runs every permutation. if ((all)); then - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,aws-lc-rs -- -D warnings + cargo clippy --locked -p moq-tokio --no-default-features --features quic,ring -- -D warnings for feature in aws-lc-rs ring tcp uds websocket; do cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings @@ -29,12 +29,12 @@ fi # shellcheck disable=SC2016 # the backticks are literal, as rustc prints them want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' -if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 +if output=$(cargo check --locked -p moq-tokio --no-default-features --features quic 2>&1); then + echo "rs: moq-tokio with only quic should reject its missing crypto provider" >&2 exit 1 fi if ! grep -qF "$want" <<<"$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "rs: moq-tokio with only quic should fail on: $want" >&2 echo "$output" | grep -E '^error' >&2 || true exit 1 fi From 866b13d62b58895086916ae116c4d9c958eaf04a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:42 -0700 Subject: [PATCH 04/17] fix(moq-tokio): log a CONNECT stream cut off by the close at debug The capsule reader maps a lost stream to UnexpectedEnd, so every ordinary session close warned "failed to read capsule". A read error while the connection is closed is now debug; one on a live connection still warns. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/web/session.rs | 49 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index ad0fc8c2d6..38e01d6684 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -154,7 +154,7 @@ impl Session { recv: crate::quic::driver::RecvStream, close: Arc, ) { - let close_info = Self::read_capsules(recv).await; + let close_info = Self::read_capsules(&conn, recv).await; let code = close_info.as_ref().map_or(0, |(c, _)| *c); let http3_code: crate::quic::driver::VarInt = web_transport_proto::error_to_http3(code).try_into().unwrap(); @@ -182,7 +182,10 @@ impl Session { // Keep reading capsules from the CONNECT recv stream until it's closed. // Returns Some((code, reason)) if a CloseWebTransportSession capsule was received, // or None if the stream closed without a capsule. - async fn read_capsules(recv: crate::quic::driver::RecvStream) -> Option<(u32, String)> { + async fn read_capsules( + conn: &crate::quic::driver::Connection, + recv: crate::quic::driver::RecvStream, + ) -> Option<(u32, String)> { let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); loop { match reader.read().await { @@ -194,8 +197,13 @@ impl Session { tracing::warn!(%typ, size = payload.len(), "unknown capsule"); } Ok(None) => return None, + // The capsule reader reports a lost stream as `UnexpectedEnd`, so ask the + // connection: a read cut short by its close is the ordinary teardown. Err(e) => { - tracing::warn!(?e, "failed to read capsule"); + match conn.close_reason() { + Some(reason) => tracing::debug!(?e, %reason, "CONNECT stream ended with the connection"), + None => tracing::warn!(?e, "failed to read capsule"), + } return None; } } @@ -1223,6 +1231,41 @@ mod tests { ); } + /// A CONNECT stream cut off by the connection's close is ordinary teardown, not a warning. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_ends_quietly_with_connection() { + let (client, server, _endpoints) = pair().await; + + // A zero-length reserved frame, so the server sees the stream without a capsule. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x21, 0x00]).await.unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + client.close(0u32.into(), b"done"); + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!(!logs_contain("failed to read capsule"), "an ordinary close warned"); + } + + /// A CONNECT stream that ends mid-frame on a live connection still warns. + #[tokio::test] + #[tracing_test::traced_test] + async fn capsule_read_truncated_warns() { + let (client, server, _endpoints) = pair().await; + + // A DATA frame that promises 5 bytes and carries 1. + let (mut send, _recv) = client.open_bi().await.unwrap(); + send.write_all(&[0x00, 0x05, 0x01]).await.unwrap(); + send.finish().unwrap(); + let (_send, recv) = server.accept_bi().await.unwrap(); + + assert!(Session::read_capsules(&server, recv).await.is_none()); + assert!( + logs_contain("failed to read capsule"), + "a truncated capsule stream was not reported" + ); + } + /// A uni stream reset before its header arrived reports the reset, not a foreign session. #[tokio::test] async fn uni_reset_before_header() { From 88a6fae64385c685bacc5af8d6a1bde783375932 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:05:46 -0700 Subject: [PATCH 05/17] docs: point fork and advisory guidance at moq-quic Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 7 +++---- rs/AGENTS.md | 2 +- rs/moq-quic/README.md | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbfcde1df5..d168b341ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,10 +74,9 @@ For non-trivial tasks, file an issue or offer to run `/quest-plan`. # Forks -[moq-dev/noq](https://github.com/moq-dev/noq) publishes `moq-noq*`, the QUIC stack every published MoQ crate builds on; iroh keeps upstream noq. -Its `moq-sync` workflow merges n0-computer/noq weekly as a PR; review it like any other, and `PARENT` names the upstream commit each release includes. -A carried change lists its upstream PR, or the reason it has none, in the fork PR. -For an advisory against noq or Quinn, compare the pinned release's `PARENT` with the fixing upstream commit, then sync, release the fork, and bump the pin here. +`rs/moq-quic` is our hard fork of quinn, the QUIC stack MoQ builds on; iroh keeps upstream noq. +It never merges upstream: fixes are cherry-picked by hand, and each carried change is listed in its README with its upstream PR, or the reason it has none. +For an advisory against Quinn, follow the advisory triage in [`rs/moq-quic/README.md`](rs/moq-quic/README.md). # Releases diff --git a/rs/AGENTS.md b/rs/AGENTS.md index d9bd16d6b4..52ff1bc226 100644 --- a/rs/AGENTS.md +++ b/rs/AGENTS.md @@ -5,7 +5,7 @@ The `/rs` Cargo workspace. Extends the root `AGENTS.md`. One crate per component from the root list, named `moq-` (`hang` is the exception). Keep them modular; a crate does one thing. Beyond that list: - `kio`: "easy async" primitives everything else polls through. -- `moq-tokio`: configures the QUIC backends (Quinn/Quiche/Noq/Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. +- `moq-tokio`: configures the QUIC backends (`moq-quic`, our quinn fork, and Iroh) and the fallback transports for native binaries, on tokio. `moq-sock` holds the socket plumbing it shares with `moq-uring`, the experimental thread-per-core io\_uring runtime. - `moq-cli` builds the `moq` binary and owns the CLI surface for the gateway crates. Binaries never carry a `-cli` suffix. `moq-net`, `moq-mux`, `moq-relay` (including the config conventions every binary shares), and `moq-ffi` have their own `AGENTS.md`. diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index d47b7629b0..b31ec65aae 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -72,4 +72,4 @@ Recheck the BBR3 changes when cherry-picking quinn#2481 updates, or if it merges `cargo audit` cannot match the renamed crate, so security fixes are tracked by hand. Watch quinn-rs/quinn's [security advisories](https://github.com/quinn-rs/quinn/security/advisories) and releases (including the `0.11.x` branch, which sometimes gets a fix `main` does not need). -For each quinn-proto or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. +For each quinn, quinn-proto, or quinn-udp advisory, check whether the vulnerable code exists in this fork, and if it does, port the fix together with its regression test. From 2d5026751bea6cecf9e598ee73a0c9fb94a0f486 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:06:36 -0700 Subject: [PATCH 06/17] docs: format the upgrade note Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index e0e6502d99..2bd0678625 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -28,7 +28,7 @@ These land with the next breaking release, not the 2026-09-23 train. it. The QUIC stack is now MoQ's own `moq-quic` rather than the `moq-noq` crates. In Rust, `moq_tokio::noq::Error` is `moq_tokio::quic::Error`, and `moq_tokio::Error::Noq` is `moq_tokio::Error::Quic`. A qlog directory gets one - file per endpoint instead of one per connection; on io_uring workers, one per + file per endpoint instead of one per connection; on io\_uring workers, one per endpoint and one per dial. - **moq-tokio config durations are optional.** Wrap durations in `Some(...)` for connect `race`, `resolution_delay`, and `timeout`; listen `timeout`; QUIC From c2993ca71eeaed7bc3fa31d28832fddc96fc2a7b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:16:50 -0700 Subject: [PATCH 07/17] quest: finish quest/m1/quic/fork/switch Relay peak RSS on the fanout workloads matches moq-noq on origin/main, which was the last open part of the Goal. Co-Authored-By: Claude Opus 5.5 --- quest/m1/p2p/native-quic.md | 5 +-- quest/m1/p2p/udp-demux.md | 4 +-- quest/m1/quic/fork/README.md | 10 ++---- quest/m1/quic/fork/switch.md | 58 -------------------------------- quest/m1/quic/receive-credit.md | 4 --- quest/m1/time/README.md | 5 ++- quest/m1/time/relay-seam.md | 1 - quest/m1/time/uring.md | 1 - quest/m2/bbr-ack-cleanup.md | 1 - quest/m2/quic-bbr-loss-parity.md | 4 --- quest/m2/quic-sim.md | 3 +- 11 files changed, 8 insertions(+), 88 deletions(-) delete mode 100644 quest/m1/quic/fork/switch.md diff --git a/quest/m1/p2p/native-quic.md b/quest/m1/p2p/native-quic.md index 912de3082f..81626f6fd3 100644 --- a/quest/m1/p2p/native-quic.md +++ b/quest/m1/p2p/native-quic.md @@ -21,9 +21,7 @@ Decided with the maintainer on 2026-10-13: candidates through the module [webrtc](/quest/m1/p2p/webrtc.md) lifts out of `moq-rtc`, with host candidates, `.local` resolution, and the STUN Binding client; don't copy it. -- QUIC is `moq-quic`, fed datagrams from the nominated pair, so this waits - on [the switch](/quest/m1/quic/fork/switch.md) instead of building on - `moq-noq` and porting it. +- QUIC is `moq-quic`, fed datagrams from the nominated pair. - Authentication: each side generates a self-signed certificate and sends its SHA-256 fingerprint in its offer or answer, as DTLS fingerprints ride in SDP. Both sides pin it with the existing `tls.fingerprint` check. MoQ @@ -61,7 +59,6 @@ covers LAN and NAT pairs between two `moq-cli --p2p` nodes. ## Required -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - `moq-quic` replaces `moq-noq` in moq-tokio - [UDP demux](/quest/m1/p2p/udp-demux.md) - the classifier that splits STUN from QUIC - [Native data channel transport](/quest/m1/p2p/webrtc.md) - the shared ICE gathering module - [Signaling and policy](/quest/m1/p2p/signal.md) - the offer schema and roster this extends diff --git a/quest/m1/p2p/udp-demux.md b/quest/m1/p2p/udp-demux.md index f3d97c8776..01bcd15e71 100644 --- a/quest/m1/p2p/udp-demux.md +++ b/quest/m1/p2p/udp-demux.md @@ -32,9 +32,7 @@ one. Bound queues per stack with a per-source drop rather than unbounded growth; report drops as a counter. QUIC: the server takes the virtual socket through whatever the QUIC stack -accepts: noq's `new_with_abstract_socket` today, `moq-quic`'s fed datagrams -after [the switch](/quest/m1/quic/fork/switch.md). Build on whichever has -landed rather than both. QUIC-bit greasing is already off, from +accepts: the driver's `new_with_abstract_socket` on `moq-quic`. QUIC-bit greasing is already off, from [shard steering](/quest/m1/p2p/shard-steering.md). STUN: an ICE connectivity check is a Binding request carrying USERNAME, so diff --git a/quest/m1/quic/fork/README.md b/quest/m1/quic/fork/README.md index 65e1f27219..091f1bafe6 100644 --- a/quest/m1/quic/fork/README.md +++ b/quest/m1/quic/fork/README.md @@ -1,4 +1,4 @@ -# Hard-fork quinn into the monorepo as moq-quic +# [XS] Hard-fork quinn into the monorepo as moq-quic ## Goal @@ -55,12 +55,8 @@ them: This README's own work: delete moq-dev/noq's `moq-sync.yml` and mark its README frozen. Advisory triage is documented in `rs/moq-quic/README.md`. Every carried change in moq-dev/noq's `CHANGELOG-MOQ.md` -is either ported by a child quest or recorded as not applicable in the -[switch](/quest/m1/quic/fork/switch.md) PR. - -## Required - -- [Switch](/quest/m1/quic/fork/switch.md) - quinn's async layer and `web-transport-moq` join moq-tokio, both runtimes run on `moq-quic`, `moq-noq*` is gone, and relay memory matches `moq-noq` +is either ported by a child quest or recorded as not applicable in +[#5664](https://github.com/moq-dev/moq/pull/5664). ## Related diff --git a/quest/m1/quic/fork/switch.md b/quest/m1/quic/fork/switch.md deleted file mode 100644 index 32e547d53e..0000000000 --- a/quest/m1/quic/fork/switch.md +++ /dev/null @@ -1,58 +0,0 @@ -# [L] Switch MoQ onto moq-quic - -## Goal - -quinn's async layer and `web-transport-moq` live in moq-tokio as modules, -moq-tokio and moq-uring run on `moq-quic`, and no workspace crate depends on -`moq-noq*`. Upstream `noq-proto` remains only for the `iroh` feature. Relay -memory on the bulk and fanout workloads matches `moq-noq`. - -## Plan - -Decided 2026-10-06: `moq-quic` (quinn-proto) is the only crate the fork adds. -quinn's async layer is imported into moq-tokio as a module rather than a -`moq-quic-tokio` crate, tokio-only: drop quinn's `Runtime` abstraction and -its smol and async-io implementations instead of carrying them. Its UDP -sockets come from `moq_sock::udp`. -Keep the first commit verbatim from upstream (quinn's `quinn/src`) so a -reviewer can diff it, and extend the cherry-pick recipe in -`rs/moq-quic/README.md` to map `quinn/src/` onto the module. - -Move `web-transport-moq` from moq-dev/noq into moq-tokio as a module too. It -derives from web-transport-quinn; its noq-only parts are `PathId::ZERO` and -`path_stats`. Report bandwidth from quinn's `PathStats::bandwidth_estimate` -instead of the current cwnd/rtt guess; it is in bytes per second and -`web-transport-trait` wants bits. moq-uring uses it only in tests, -through moq-tokio. - -BBR3 is already `moq-quic`'s default, with `Bbr3Config` and `Bbr3` named as -in noq, so the `Delay` family maps over unchanged. Run the benchmark matrix -against `moq-noq` 1.3.x on the same workloads and report any throughput or -latency difference; the BBR3 port could not, with no relay on `moq-quic` yet. - -moq-uring built against both quinn-proto and noq-proto until #3811, behind -about 20 `cfg` lines (stats fields, CID generator, qlog, BBR). Rename -`rs/moq-uring/src/quic/noq` by role and use that history as the map. Keep -moq-uring's `qlog` feature working on quinn's qlog. - -Rename the `noq` cargo features by role (`quic` is the recommendation); -confirm the name with the maintainer in the PR. Update every doc and example -that names noq, and run `just test interop --all`. - -Decided 2026-10-10: `moq-quic` takes `moq_clock::Instant` at its boundary in -place of `std::time::Instant`, so the sim can drive real QUIC later -([QUIC on the sim](/quest/m2/quic-sim.md)) and the io_uring worker's per-turn -sample passes straight through. `moq_clock::Instant` mirrors std's methods, so -alias it in `moq-quic` and keep the swap an import change; that keeps -upstream cherry-picks mechanical. - -In the PR, list each carried change from moq-dev/noq's `CHANGELOG-MOQ.md` as -ported (with its quest) or not applicable (with the reason). - -Lazy stream slots ([quinn#2601](https://github.com/quinn-rs/quinn/pull/2601), -the change noq took as noq#667) arrived with the import at quinn `7616e6b2`, -so only its measurement remains, and that needs the relay on `moq-quic`. -Re-run #3342's bulk and fanout relay memory workloads after the switch and -report them in the PR. On `moq-noq` they measured 75 MiB (bulk) and 31 MiB -(fanout), against 141 and 97 MiB without lazy slots; `moq-quic` should land -near the former. diff --git a/quest/m1/quic/receive-credit.md b/quest/m1/quic/receive-credit.md index c2bd474fbb..9f93925876 100644 --- a/quest/m1/quic/receive-credit.md +++ b/quest/m1/quic/receive-credit.md @@ -47,10 +47,6 @@ with its consumer. Wire: existing QUIC flow-control and cancellation frames; no new MoQ message or negotiation. Browser and iroh receivers do not gain a capability their transports cannot provide. -## Required - -- [Switch MoQ onto moq-quic](/quest/m1/quic/fork/switch.md) - both native adapters use the in-tree transport - ## Related - [Bound message assembly](/quest/m1/quic/receive-memory.md) - retains connection credit until a complete message is handed off diff --git a/quest/m1/time/README.md b/quest/m1/time/README.md index 9b070511dd..b4f507c457 100644 --- a/quest/m1/time/README.md +++ b/quest/m1/time/README.md @@ -85,9 +85,8 @@ reasons, so later sessions don't reopen them: [moq-uring on moq-clock](/quest/m1/time/uring.md). The relay seam [bench-relay](/quest/m2/bench-relay.md) needed is [the relay seam](/quest/m1/time/relay-seam.md). Quests that already asked - for "mocked time" Require the core quest. moq-quic adopts the logical - instant in the [switch](/quest/m1/quic/fork/switch.md), which Requires the - crate. + for "mocked time" Require the core quest. moq-quic takes the logical + instant at its boundary. This milestone verifies the exported clock contract and production consumers needed by the QUIC switch and clustering regressions. Unrelated test/private diff --git a/quest/m1/time/relay-seam.md b/quest/m1/time/relay-seam.md index 2bf68fdcb5..5a938ca285 100644 --- a/quest/m1/time/relay-seam.md +++ b/quest/m1/time/relay-seam.md @@ -33,4 +33,3 @@ Public API: the in-memory session and the relay seam. Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the sim and instant the relay tests run on -- [Switch](/quest/m1/quic/fork/switch.md) - moq-tokio's server types settle first diff --git a/quest/m1/time/uring.md b/quest/m1/time/uring.md index 6151101783..6987317e34 100644 --- a/quest/m1/time/uring.md +++ b/quest/m1/time/uring.md @@ -36,7 +36,6 @@ Wire: none. ## Required - [moq-net on moq-clock](/quest/m1/time/net.md) - the driver's instant becomes moq-clock's -- [Switch](/quest/m1/quic/fork/switch.md) - the QUIC driver this edits moves onto `moq-quic` ## Closes diff --git a/quest/m2/bbr-ack-cleanup.md b/quest/m2/bbr-ack-cleanup.md index 4791c2bb46..00462c87cc 100644 --- a/quest/m2/bbr-ack-cleanup.md +++ b/quest/m2/bbr-ack-cleanup.md @@ -54,6 +54,5 @@ packet-lifetime comments inline; no new user guide is needed. ## Related -- [Switch](/quest/m1/quic/fork/switch.md) - not a blocker: the core and its BBR3 are already in `rs/moq-quic` - [Benchmark reporting](/bench/README.md) - reusable measurement guidance, not a prerequisite for this fix - [Loss sampling](/quest/m2/quic-bbr-loss-parity.md) - preserve packet metadata needed by the separate loss-sample repair; both edit `bbr3/mod.rs`, so sequence them diff --git a/quest/m2/quic-bbr-loss-parity.md b/quest/m2/quic-bbr-loss-parity.md index b936b04d1c..83639152a5 100644 --- a/quest/m2/quic-bbr-loss-parity.md +++ b/quest/m2/quic-bbr-loss-parity.md @@ -39,7 +39,3 @@ traffic yet. ## Required - [BBR ACK cleanup](/quest/m2/bbr-ack-cleanup.md) - edits the same `bbr3/mod.rs` packet bookkeeping and lands first; this keeps the metadata it preserves - -## Related - -- [Switch](/quest/m1/quic/fork/switch.md) - MoQ runs on `moq-quic`; not a blocker, since this is a local `rs/moq-quic` fix diff --git a/quest/m2/quic-sim.md b/quest/m2/quic-sim.md index 9e3c21b829..423cd102c0 100644 --- a/quest/m2/quic-sim.md +++ b/quest/m2/quic-sim.md @@ -11,8 +11,7 @@ real QUIC are tested deterministically with no sockets. Decided 2026-10-10: the in-memory session comes first ([the relay seam](/quest/m1/time/relay-seam.md)); simulated UDP follows. -`moq-quic` already takes `moq_clock::Instant` after the -[switch](/quest/m1/quic/fork/switch.md), so this is a datagram link plus a +`moq-quic` already takes `moq_clock::Instant`, so this is a datagram link plus a driver. Start from moq-shaper's in-memory `UdpSocket` (`rs/moq-shaper/src/mem.rs`). BBR's controller-level `Sim` in `bbr3/mod.rs` stays separate unless this replaces it cleanly. From 3b50dd6d079850ca7d20f386c9a7be25e9933bd0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:46:55 -0700 Subject: [PATCH 08/17] chore(quest): claim quest/m1/quic/shard Co-Authored-By: Claude Opus 5.5 From 0c8cfcc136503ae6489ade267f6ef2cdba343b25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:33:37 -0700 Subject: [PATCH 09/17] feat(quic)!: shard the relay's QUIC endpoint per core by default moq-quic now issues shard-naming connection IDs itself (HashedConnectionIdGenerator::set_shard), replacing the custom generators in moq-tokio and moq-uring, and refuses a sibling shard's IDs so a stray packet is dropped instead of reset. moq_sock::shard::bind forms a steered group in one call, replacing the Group/Member/Claim API both runtimes looped over, and moq_sock::shard::per_core picks the default size. The relay runs one QUIC worker per core on Linux unless runtime.workers says otherwise (0 keeps the shared runtime), and runtime.pin now defaults off. Adds `just bench-shard`, a shard-count by connection-count sweep, nightly. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 2 + Cargo.lock | 1 - bench/README.md | 15 + bench/relay.sh | 9 +- bench/run.sh | 70 ++ doc/bin/relay/config.md | 21 +- doc/bin/relay/index.md | 1 + justfile | 5 + quest/m1/p2p/shard-steering.md | 15 +- quest/m1/quic/README.md | 2 - quest/m1/quic/fork/README.md | 1 - quest/m1/quic/shard.md | 47 -- .../m2/perf/kio-channel-contention/README.md | 4 +- rs/moq-quic/src/cid_generator.rs | 92 +++ rs/moq-relay/Cargo.toml | 2 +- rs/moq-relay/src/config.rs | 12 +- rs/moq-relay/src/relay.rs | 85 ++- rs/moq-relay/src/runtime.rs | 63 +- rs/moq-relay/src/uring.rs | 21 +- rs/moq-sock/Cargo.toml | 2 +- rs/moq-sock/README.md | 16 +- rs/moq-sock/src/lib.rs | 2 +- rs/moq-sock/src/shard.rs | 661 +++++------------- rs/moq-tokio/src/listen.rs | 2 - rs/moq-tokio/src/quic/backend.rs | 54 +- rs/moq-tokio/src/server.rs | 31 - rs/moq-tokio/src/worker/group.rs | 43 +- rs/moq-uring/Cargo.toml | 1 - rs/moq-uring/src/quic/driver/mod.rs | 36 +- rs/moq-uring/src/quic/endpoint.rs | 29 +- rs/moq-uring/src/udp.rs | 8 +- rs/moq-uring/tests/workers.rs | 8 +- 32 files changed, 552 insertions(+), 809 deletions(-) delete mode 100644 quest/m1/quic/shard.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3b6b5606dc..6493ee58dd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -68,6 +68,8 @@ jobs: artifact-path: target/bench/ffi - name: Audio grouping relay benchmark recipe: bench-audio + - name: Relay shard sweep + recipe: bench-shard # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. - name: Relay lock-wait profile recipe: bench-lock fanout-heavy 4 diff --git a/Cargo.lock b/Cargo.lock index a2a305b404..040ac80540 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5010,7 +5010,6 @@ dependencies = [ "moq-sock", "moq-tokio", "nix", - "rand 0.10.3", "rcgen", "rustc-hash 2.1.3", "rustls", diff --git a/bench/README.md b/bench/README.md index 5c8323a2ab..62c5ee1321 100644 --- a/bench/README.md +++ b/bench/README.md @@ -48,6 +48,21 @@ Runtime comparison requires Linux because io\_uring and relay process metrics come from Linux interfaces. The default worker count is the number of online logical CPUs. +Sweep the relay's QUIC shard count against the connection count, for the +room (`video`: every connection publishes and subscribes) and `fanout` (one +publisher, the rest subscribers) workloads: + +```bash +nix develop --command just bench-shard +nix develop --command just bench-shard "0 1 8 24" "64 512" +``` + +Zero shards is the shared runtime with one Tokio thread per logical CPU; any +other count is that many unpinned Tokio workers. The table reports CPU per +connection, so a steering, dispatch, or per-shard cost that grows with either +axis reads as a slope. Linux only, since nothing else steers a reuseport +group. It runs nightly. + Attribute relay worker lock waits to the stacks that blocked, as a share of each worker's CPU: diff --git a/bench/relay.sh b/bench/relay.sh index d14df4e638..6b5a6562e9 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -23,9 +23,16 @@ relay_config() { fi case $runtime in - '' | tokio-shared) + '') runtime_config=() ;; + tokio-shared) + runtime_config=( + '' + '[runtime]' + 'workers = 0' + ) + ;; tokio-workers) runtime_config=( '' diff --git a/bench/run.sh b/bench/run.sh index d1fc701c7d..65bbe4b9d0 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -8,6 +8,7 @@ set -euo pipefail # on the current tree. The current load generator drives every relay so the # workloads stay identical. `--audio` isolates the audio grouping/load-shape matrix. # `--lock` records relay worker off-CPU stacks to attribute std mutex waits. +# `--shard` sweeps the QUIC shard count against the connection count. MODE=${1:-} ROUNDS=${MOQ_BENCH_RUNTIME_ROUNDS:-3} @@ -27,6 +28,17 @@ case $MODE in # Without frame pointers, perf's DWARF unwinder stops at the first relay frame. export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C force-frame-pointers=yes" ;; + --shard) + BASE= + SHARD_COUNTS=${MOQ_BENCH_SHARD_COUNTS:-0 1 4 16} + SHARD_CONNECTIONS=${MOQ_BENCH_SHARD_CONNECTIONS:-16 64 256} + for value in $SHARD_COUNTS $SHARD_CONNECTIONS; do + if [[ ! $value =~ ^[0-9]+$ ]]; then + printf 'shard benchmark counts must be integers, got %q\n' "$value" >&2 + exit 1 + fi + done + ;; --runtime) BASE= RUNTIME_WORKERS=${MOQ_BENCH_RUNTIME_WORKERS:-} @@ -492,6 +504,61 @@ run_runtime_comparison() { print_runtime_comparison } +# Each workload's load axis is its connection count: every room connection +# publishes and subscribes, while fanout adds subscribers to one publisher. A +# cost that grows with the shard count or the load shows up as a slope across +# the shard counts or the connection counts, which one default-versus-single- +# socket run cannot show. +run_shard_sweep() { + if [[ $(uname -s) != Linux ]]; then + printf 'shard sweep needs Linux: only it steers a reuseport group\n' >&2 + return 1 + fi + generate_certificate + + printf 'Building relay and load generator...\n' + CARGO_TARGET_DIR=$CURRENT_TARGET cargo build --locked --release -p moq-relay -p moq-bench + LOAD_BIN=$CURRENT_TARGET/release/moq-bench + HOST_BIN=$CURRENT_TARGET/release/moq-bench-host + + local cpus workload connections shards label + cpus=$(getconf _NPROCESSORS_ONLN) + for workload in video fanout; do + for connections in $SHARD_CONNECTIONS; do + for shards in $SHARD_COUNTS; do + label=shard-$shards-$connections + printf ' %s: %d connections, %d shards\n' "$workload" "$connections" "$shards" + # Zero shards is the shared runtime, given every CPU. + if ((shards == 0)); then + run_workload "$label" "$CURRENT_TARGET/release/moq-relay" "$workload" tokio-shared "$cpus" \ + --connections "$connections" + else + run_workload "$label" "$CURRENT_TARGET/release/moq-relay" "$workload" tokio-workers "$shards" \ + --connections "$connections" + fi + done + done + done + + printf '\nShard sweep (0 shards is the shared runtime on %d threads)\n' "$cpus" + printf '%-8s %11s %7s %10s %10s %8s %7s %8s %9s %9s\n' \ + workload connections shards recv-Mbps recv-fps p99-ms loss-% CPU CPU/conn RSS-MiB + for workload in video fanout; do + for connections in $SHARD_CONNECTIONS; do + for shards in $SHARD_COUNTS; do + jq -r --arg connections "$connections" --arg shards "$shards" '[ + .workload, $connections, $shards, .recv_mbps, .recv_fps, .latency_p99_ms, .loss_pct, + .cpu_cores, (.cpu_cores / ($connections | tonumber)), (.rss_bytes / 1048576) + ] | @tsv' "$RUN/relay/shard-$shards-$connections/$workload/summary.json" | + while IFS=$'\t' read -r name conns count recv fps p99 loss cpu per rss; do + printf '%-8s %11s %7s %10.3f %10.1f %8.1f %7.3f %8.3f %9.5f %9.1f\n' \ + "$name" "$conns" "$count" "$recv" "$fps" "$p99" "$loss" "$cpu" "$per" "$rss" + done + done + done + done +} + run_lock_profile() { if [[ $(uname -s) != Linux ]]; then printf 'lock profiling needs Linux for perf and /proc schedstat\n' >&2 @@ -543,6 +610,9 @@ elif [[ $MODE == --runtime ]]; then elif [[ $MODE == --lock ]]; then run_lock_profile exit 0 +elif [[ $MODE == --shard ]]; then + run_shard_sweep + exit 0 elif [[ -n $BASE ]]; then BASE_COMMIT=$(git rev-parse --verify "$BASE^{commit}") WORKTREE=$RUN/base diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 9bd91a4a5e..d30bfb7fc9 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -109,23 +109,32 @@ window, so very small values can prevent other requests from progressing. ## \[runtime] -By default one work-stealing runtime serves every connection off one UDP -socket. On Linux, QUIC can instead run on pinned single-threaded workers, each +On Linux, QUIC runs on one single-threaded worker per core by default, each with its own socket on the listen address (`SO_REUSEPORT`), so a connection is -handled start to finish by one thread and its packets never cross cores. +handled start to finish by one thread and its packets never cross cores. The +count follows the CPUs the process may use, cgroup quotas included. Elsewhere, +and with `workers = 0`, one work-stealing runtime serves every connection off +one UDP socket. ```toml [runtime] -workers = 8 # Single-threaded QUIC workers. Omit for the shared runtime. -pin = true # Pin each worker to a core. Default. +workers = 8 # Override the per-core count. 0 keeps the shared runtime. +pin = false # Pin each worker to a core. Off by default. io_uring = false # Drive them with io_uring instead of tokio. ``` Packets are steered by connection ID, so a client that migrates stays with its -worker. `workers` needs the `quic` feature. Every worker and the TLS stream +worker. An explicit count needs Linux and the `quic` feature, and is refused +anywhere else rather than ignored. A relay with a QUIC-LB server id +(`listen.lb_id`) or no QUIC listener keeps the shared runtime unless a count is +set, and a count is refused alongside `lb_id`. Every worker and the TLS stream listener serve one shared identity, generated or file-backed, so a certificate rotation reaches them together and `/certificate.sha256` stays accurate. +Pinning is off by default because every relay pins its workers to the same +first cores, so two relays on one host, or one limited to fewer CPUs than the +host has, would crowd onto them. Turn it on for a host the relay has to itself. + `io_uring` additionally needs Linux 6.12+ and the `io-uring` cargo feature; it shares the same reloadable TLS identity, serves moq-lite only, and refuses `mtu_discovery`. A setting the build or host cannot deliver refuses to diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index acd69bfae7..59def5fb7f 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -12,6 +12,7 @@ relay serves video, audio, and data alike. ## Features - **QUIC, WebTransport, and WebSocket** listeners, so browsers and native clients connect to one process. +- **Thread-per-core QUIC** on Linux by default: one worker and socket per core, with packets steered by connection ID. See [`[runtime]`](/bin/relay/config#runtime). - **Path-scoped authentication** with JWTs, mTLS for peers, and anonymous patterns, decided by an auth server or a static grant. See [Authentication](/bin/relay/auth). - **Clustering** across hosts and regions with hop-list routing, per-link costs, LAN discovery, and dynamic peer lists. See [Clustering](/bin/relay/cluster). - **A group cache** with byte and age budgets, so late joiners and the HLS gateway can fetch recent history. diff --git a/justfile b/justfile index 1b926cd1fc..bc4b8aa04f 100644 --- a/justfile +++ b/justfile @@ -60,6 +60,11 @@ bench-audio: bench-runtime $ROUNDS="3" $WORKERS="": MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime +# Sweep the relay's QUIC shard count against room and fanout connection counts; +# 0 shards is the shared runtime. Example: `just bench-shard "0 1 8" "64 512"`. +bench-shard $SHARDS="0 1 4 16" $CONNECTIONS="16 64 256": + MOQ_BENCH_SHARD_COUNTS="$SHARDS" MOQ_BENCH_SHARD_CONNECTIONS="$CONNECTIONS" bench/run.sh --shard + # Attribute relay worker std mutex waits by stack, for each worker count. bench-lock $WORKLOAD="fanout" $WORKERS="1 4 16": MOQ_BENCH_LOCK_WORKLOAD="$WORKLOAD" MOQ_BENCH_LOCK_WORKERS="$WORKERS" bench/run.sh --lock diff --git a/quest/m1/p2p/shard-steering.md b/quest/m1/p2p/shard-steering.md index 3f7d646d4c..93f255d669 100644 --- a/quest/m1/p2p/shard-steering.md +++ b/quest/m1/p2p/shard-steering.md @@ -12,9 +12,8 @@ off so their peers' short headers stay recognizable. ## Plan -The filter (`moq_sock::shard`, which stays in `moq-sock` under -[quic/shard](/quest/m1/quic/shard.md)) reads byte 1 or byte 6 by the -long-header bit and reduces it modulo the group size. A non-QUIC datagram +The filter (`moq_sock::shard`, the one copy both runtimes use) reads byte 1 +or byte 6 by the long-header bit and reduces it modulo the group size. A non-QUIC datagram goes through the same arithmetic: RTP and RTCP are steered by timestamp and SSRC bytes and scatter across shards, SRT data is steered by a sequence byte, and STUN and DTLS, whose byte 1 or 6 barely varies, pile onto one shard. @@ -42,9 +41,9 @@ bytes can: destination connection ID carries a fixed relay magic of N bytes beside the existing shard byte. This is a CID-format change: every CID the relay issues, the initial source CID and every NEW_CONNECTION_ID, embeds the - magic. Its owner is the shard CID generator (moq-tokio's and moq-uring's - today, `moq-quic`'s once [quic/shard](/quest/m1/quic/shard.md) replaces - them). Say whether the CID grows past today's 8 bytes or the magic takes + magic. Its owner is the shard CID generator, + `moq_quic::HashedConnectionIdGenerator::set_shard`, which both runtimes + use. Say whether the CID grows past today's 8 bytes or the magic takes random bytes from it. - Everything else returns an out-of-range index, which the kernel answers with its 4-tuple hash. @@ -75,7 +74,3 @@ flow includes retransmissions (R set) of the same packets. QUIC v1 and v2 long headers and short headers still reach their CID's shard. A client talking to each backend receives transport parameters without `grease_quic_bit`, and every short header it sends reaches its CID's shard. - -## Related - -- [Shard the QUIC endpoint](/quest/m1/quic/shard.md) - makes `moq_sock::shard` the one filter both runtimes use, and moves the CID generator into `moq-quic` diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index 5b00ba7b8e..7376d0eb75 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -79,8 +79,6 @@ cache limits remain separate; bitrate pacing composes with this memory cap. datagrams join their subscription's send group instead of preempting every stream - [Relay peers get wider limits](/quest/m1/quic/peer-limits.md) - MAX_STREAMS and MAX_DATA are raised after SETUP identifies a cluster peer -- [Shard the endpoint](/quest/m1/quic/shard.md) - the library shards a - server endpoint across cores by default, replacing the `--workers` group - [Surface UDP send errors](/quest/m1/quic/send-errors.md) - a client drops a QUIC attempt the moment its address family is unreachable ## Related diff --git a/quest/m1/quic/fork/README.md b/quest/m1/quic/fork/README.md index 091f1bafe6..92706af438 100644 --- a/quest/m1/quic/fork/README.md +++ b/quest/m1/quic/fork/README.md @@ -60,6 +60,5 @@ is either ported by a child quest or recorded as not applicable in ## Related -- [Shard the endpoint](/quest/m1/quic/shard.md) - the first feature the fork makes possible - [Remove moq-uring copies](/quest/m2/perf/uring-copies.md) - cheap copy removals on the new core - [Redesign the QUIC I/O boundary](/quest/m2/quic-io-boundary.md) - the breaking perf rewrite, gated on a profile diff --git a/quest/m1/quic/shard.md b/quest/m1/quic/shard.md deleted file mode 100644 index 8e163959a0..0000000000 --- a/quest/m1/quic/shard.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] Shard the QUIC endpoint across cores - -## Goal - -A `moq-quic` server endpoint shards across cores by default where the -platform can steer (Linux): the library -forms the steered `SO_REUSEPORT` group and issues connection IDs that name -their shard, and each runtime (moq-tokio and moq-uring) only drives the -shards it is handed on threads of its own. The relay's `--workers` flag -becomes an optional count override, and the two copies of the group logic -are gone. - -## Plan - -Decided in the 2026-09-30 plan: the library shards and the runtime drives. -The library does not own threads, so embedders (moq-ffi and the bindings over it) keep their -own threading model. - -- `moq_sock::shard` (the group, claims, and steering filter) stays in - `moq-sock` beside `moq_sock::udp` (quinn-udp), the one copy both runtimes - use; `moq-sock` keeps binding and CPU helpers. -- `moq-quic` issues shard-prefixed connection IDs itself, replacing the - custom CID generators in moq-tokio and moq-uring. -- moq-tokio's `worker/group.rs` shrinks to spawning one pinned - `current_thread` runtime per shard; moq-uring's `udp::Bound` group code is - replaced by the library's. -- A server endpoint defaults to one shard per core only where - `SO_REUSEPORT` steering works, which today is Linux: `moq-sock` returns - `Unsupported` for both load balancing and steering elsewhere. macOS and - Windows keep a genuine single-socket endpoint, and an explicit count above - one is refused there rather than silently ignored. -- Before flipping the default, add a benchmark sweeping shard count against - concurrent session load (publishers and subscribers), so steering, - dispatch, and per-shard overhead show up as slopes. A single - default-versus-one-socket run cannot show a cost that grows with either - axis. - -Update `doc/bin/relay/` for the flag change. - -## Required - -- [Hard fork](/quest/m1/quic/fork/README.md) - the stack has to be in-tree before the library can own sharding - -## Related - -- [UDP demux](/quest/m1/p2p/udp-demux.md) - demuxes each shard's socket; keep the demux over whatever the library hands out -- [Steer only QUIC by connection ID](/quest/m1/p2p/shard-steering.md) - the filter this moves leaves non-QUIC flows to the kernel's 4-tuple hash diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index b68d806397..7c774490cf 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -134,8 +134,8 @@ Decided (maintainer, 2026-10-14, baseline follow-up): Decided (maintainer, 2026-10-14, per-thread follow-up): - Per-worker sharding is no longer ruled out; two spikes decide. Both run on - the thread-per-core workers (`--runtime-workers`, the default once - [Shard](/quest/m1/quic/shard.md) lands) and link workers over an + the thread-per-core workers (`--runtime-workers`, the default on Linux) + and link workers over an in-memory transport (`moq_net::transport::memory`), never real QUIC. The [replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) diff --git a/rs/moq-quic/src/cid_generator.rs b/rs/moq-quic/src/cid_generator.rs index 810f5603e7..05edbd3ee0 100644 --- a/rs/moq-quic/src/cid_generator.rs +++ b/rs/moq-quic/src/cid_generator.rs @@ -100,6 +100,28 @@ impl ConnectionIdGenerator for RandomConnectionIdGenerator { pub struct HashedConnectionIdGenerator { key: u64, lifetime: Option, + shard: Option, +} + +/// One endpoint's slot in a group sharing a port, which its connection IDs name. +#[derive(Debug, Clone, Copy)] +struct Shard { + index: u16, + count: u16, +} + +impl Shard { + /// A first byte congruent to `index` modulo `count`, keeping as much of the byte random as + /// the group size leaves: `256 / count` values. + fn prefix(self) -> u8 { + let count = u32::from(self.count); + let stride = rand::rng().random_range(0..256 / count); + (stride * count + u32::from(self.index)) as u8 + } + + fn owns(self, first: u8) -> bool { + u16::from(first) % self.count == self.index + } } impl HashedConnectionIdGenerator { @@ -116,6 +138,7 @@ impl HashedConnectionIdGenerator { Self { key, lifetime: None, + shard: None, } } @@ -124,6 +147,24 @@ impl HashedConnectionIdGenerator { self.lifetime = Some(d); self } + + /// Name shard `index` of a `count`-endpoint group in the first byte of every CID + /// + /// The byte is congruent to `index` modulo `count`, so a `SO_REUSEPORT` steering filter that + /// reduces a packet's first destination CID byte modulo the group size delivers it to the + /// endpoint that issued the CID. [`validate`](ConnectionIdGenerator::validate) then also + /// refuses another shard's CIDs, so a stray packet is dropped rather than answered with a + /// stateless reset for a connection a sibling owns. + /// + /// Panics unless `index < count <= 256`: one byte cannot name more shards than that. + pub fn set_shard(&mut self, index: u16, count: u16) -> &mut Self { + assert!( + index < count && count <= 256, + "shard {index} of {count} cannot be named by one byte" + ); + self.shard = Some(Shard { index, count }); + self + } } impl Default for HashedConnectionIdGenerator { @@ -136,6 +177,9 @@ impl ConnectionIdGenerator for HashedConnectionIdGenerator { fn generate_cid(&mut self) -> ConnectionId { let mut bytes_arr = [0; NONCE_LEN + SIGNATURE_LEN]; rand::rng().fill_bytes(&mut bytes_arr[..NONCE_LEN]); + if let Some(shard) = self.shard { + bytes_arr[0] = shard.prefix(); + } let mut hasher = rustc_hash::FxHasher::default(); hasher.write_u64(self.key); hasher.write(&bytes_arr[..NONCE_LEN]); @@ -144,6 +188,11 @@ impl ConnectionIdGenerator for HashedConnectionIdGenerator { } fn validate(&self, cid: ConnectionId) -> Result<(), InvalidCid> { + if let Some(shard) = self.shard + && !cid.first().is_some_and(|&first| shard.owns(first)) + { + return Err(InvalidCid); + } let (nonce, signature) = cid.split_at(NONCE_LEN); let mut hasher = rustc_hash::FxHasher::default(); hasher.write_u64(self.key); @@ -177,4 +226,47 @@ mod tests { let cid = generator.generate_cid(); generator.validate(cid).unwrap(); } + + /// The steering filter's only requirement: whatever first byte a shard issues reduces back + /// to that shard, at every group size including the one-value-per-shard ceiling. + #[test] + fn a_shard_cid_names_its_shard() { + for count in [1u16, 2, 3, 7, 8, 64, 255, 256] { + for index in 0..count { + let mut generator = HashedConnectionIdGenerator::new(); + generator.set_shard(index, count); + for _ in 0..32 { + let cid = generator.generate_cid(); + assert_eq!(cid.len(), generator.cid_len()); + assert_eq!(u16::from(cid[0]) % count, index); + generator.validate(cid).unwrap(); + } + } + } + } + + /// A shard refuses a sibling's CIDs even under the same key, so a misrouted packet is dropped + /// rather than reset. + #[test] + fn a_shard_refuses_a_sibling_cid() { + let mut first = HashedConnectionIdGenerator::from_key(7); + first.set_shard(0, 4); + let mut second = HashedConnectionIdGenerator::from_key(7); + second.set_shard(1, 4); + let cid = second.generate_cid(); + second.validate(cid).unwrap(); + assert!(first.validate(cid).is_err()); + } + + #[test] + #[should_panic] + fn a_shard_past_the_group_is_refused() { + HashedConnectionIdGenerator::new().set_shard(4, 4); + } + + #[test] + #[should_panic] + fn a_group_past_one_byte_is_refused() { + HashedConnectionIdGenerator::new().set_shard(0, 257); + } } diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 7791f78594..4dc26a41df 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -34,7 +34,7 @@ quic = ["_quic", "moq-tokio/quic"] # The tokio QUIC listener, and the thread-per-core worker group behind # `runtime.workers`. Not one to enable on its own: it names no backend, so # moq-tokio would have nothing to bind a socket with. -_quic = [] +_quic = ["dep:moq-sock"] # qlog trace capture (`--quic-qlog `). Off by default: it compiles the # qlog machinery into the enabled backends and is only wanted for debugging or # congestion-control testing. Covers the io_uring workers too, so one feature diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 707ffbc379..32484b7e60 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -59,8 +59,8 @@ pub struct Config { #[serde(default)] pub log: moq_tokio::Log, - /// How QUIC work is laid out over threads. One shared runtime unless - /// `runtime.workers` is set. + /// How QUIC work is laid out over threads: one worker per core on Linux + /// unless `runtime.workers` says otherwise. #[usage(flatten)] #[serde(default)] pub runtime: crate::runtime::Config, @@ -1351,12 +1351,12 @@ uid = [1001] }, Case { name: "declared-default-true-false-from-file", - toml: "[runtime]\npin = false\n", + toml: "[web]\nws = false\n", env: &[], cli: &[], - key: "runtime.pin", - check: |config| assert!(!config.runtime.pin, "file false must beat a declared true default"), - source: "pin", + key: "web.ws", + check: |config| assert!(!config.web.ws, "file false must beat a declared true default"), + source: "ws", }, Case { name: "optional-with-default-from-file", diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 8f463f4c3a..0afceab9b6 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -95,14 +95,15 @@ pub struct Relay { /// Live sessions on this node, listed and nudged from the internal listener. sessions: crate::session::Registry, /// The thread-per-core QUIC workers, already bound and waiting to be split - /// and run. `None` unless `runtime.workers` is configured, in which case - /// `server` carries no QUIC listener of its own. + /// and run. `None` when QUIC stays on the shared runtime (`runtime.workers` + /// is `0`, or unset where a listener cannot be sharded); otherwise `server` + /// carries no QUIC listener of its own. #[cfg(feature = "_quic")] workers: Option, /// The io_uring QUIC workers, already bound and waiting for - /// [`serve`](crate::uring::Workers::serve). `None` unless both - /// `runtime.workers` and `runtime.io_uring` are configured, in which case - /// they own the QUIC listen address instead of `workers`. + /// [`serve`](crate::uring::Workers::serve). `None` unless QUIC runs on + /// workers and `runtime.io_uring` is set, in which case they own the QUIC + /// listen address instead of `workers`. #[cfg(all(target_os = "linux", feature = "_uring"))] uring: Option, } @@ -154,9 +155,27 @@ impl Relay { // Bind the QUIC workers first: they own the listen address when configured, // so the server below must not also try to bind it. let io_uring = config.runtime.io_uring(); + + // Sharding is the default only where a group serves exactly the listener + // the shared runtime would have opened: a backend that can drive workers, + // a QUIC listen address (not a stream-only relay), and no QUIC-LB server + // id, whose connection IDs leave the steering filter no byte to read. An + // explicit count skips these and is refused below where it cannot deliver. + #[cfg(any(feature = "_quic", feature = "_uring"))] + let per_core = { + let backend = cfg!(feature = "_quic") || io_uring; + let shardable = backend + && (config.listen.bind.is_some() || !config.listen.has_explicit_bind()) + && config.listen.load_balancer().is_none(); + shardable.then(moq_sock::shard::per_core).flatten() + }; + #[cfg(not(any(feature = "_quic", feature = "_uring")))] + let per_core = None; + let worker = config.runtime.workers(per_core); + anyhow::ensure!( - !io_uring || config.runtime.workers().is_some(), - "runtime.io_uring requires runtime.workers" + !io_uring || worker.is_some(), + "runtime.io_uring needs QUIC workers, which runtime.workers = 0 turns off" ); #[cfg(not(target_os = "linux"))] anyhow::ensure!(!io_uring, "runtime.io_uring is Linux-only"); @@ -170,20 +189,20 @@ impl Relay { // silently serving from the shared runtime instead would look like it worked. #[cfg(not(feature = "_quic"))] anyhow::ensure!( - io_uring || config.runtime.workers().is_none(), + io_uring || worker.is_none(), "runtime.workers needs moq-relay built with the `quic` feature" ); // Workers and the stream server split one relay across listeners, so they share // one loaded identity. A lone server loads its own, only if a listener needs it. - let served = config.runtime.workers().map(|_| config.listen.tls.load()).transpose()?; + let served = worker.map(|_| config.listen.tls.load()).transpose()?; let mut listen = config.listen.clone(); if served.is_some() { listen.tls = Default::default(); } #[cfg(feature = "_quic")] - let workers = match config.runtime.workers() { + let workers = match worker { Some(worker) if !io_uring => { let mut server = moq_tokio::server::Config::default(); server.listen = listen.clone(); @@ -207,7 +226,7 @@ impl Relay { let workers_certificates: Option = None; #[cfg(all(target_os = "linux", feature = "_uring"))] - let uring = match config.runtime.workers() { + let uring = match worker { Some(worker) if io_uring => { let mut server = moq_tokio::server::Config::default(); server.listen = listen.clone(); @@ -825,6 +844,50 @@ async fn serve_listening( #[cfg(test)] mod tests { + /// A QUIC relay with no `[runtime]` section shards across every core it may + /// use, where the kernel can steer a group. + #[cfg(all(target_os = "linux", feature = "_quic"))] + #[tokio::test] + async fn quic_shards_per_core_by_default() { + let mut config = crate::Config::default(); + config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + config.listen.tls.generate = vec!["localhost".into()]; + let relay = super::Relay::load(config).await.unwrap(); + let workers = relay.workers.as_ref().expect("an unset count shards"); + assert_eq!(Some(workers.len()), moq_sock::shard::per_core().map(usize::from)); + assert_eq!(relay.addr(), Some(workers.local_addr())); + } + + /// Each of these keeps QUIC on the shared runtime when no count is set: the + /// explicit opt-out, a stream-only relay (sharding would open a QUIC port + /// nobody asked for), and QUIC-LB, whose connection IDs leave the steering + /// filter nothing to read. + #[cfg(all(target_os = "linux", feature = "_quic"))] + #[tokio::test] + async fn quic_stays_shared_where_sharding_does_not_fit() { + let optout = { + let mut config = crate::Config::default(); + config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + config.listen.tls.generate = vec!["localhost".into()]; + config.runtime.workers = Some(0); + config + }; + let streams = { + let mut config = crate::Config::default(); + config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + config + }; + let quic_lb: crate::Config = toml::from_str( + "[listen]\nbind = \"127.0.0.1:0\"\ntls.generate = [\"localhost\"]\nload_balancer = { id = \"0a0b\", nonce = 4 }\n", + ) + .unwrap(); + + for (name, config) in [("workers = 0", optout), ("stream-only", streams), ("QUIC-LB", quic_lb)] { + let relay = super::Relay::load(config).await.unwrap(); + assert!(relay.workers.is_none(), "{name} sharded"); + } + } + #[tokio::test] async fn stun_listener_runs_with_the_relay_and_stops_with_it() { use stun_types::message::{BINDING, Message, MessageWriteVec}; diff --git a/rs/moq-relay/src/runtime.rs b/rs/moq-relay/src/runtime.rs index d9c83cb27d..25fe0b460e 100644 --- a/rs/moq-relay/src/runtime.rs +++ b/rs/moq-relay/src/runtime.rs @@ -1,42 +1,39 @@ -//! Flags for serving QUIC from pinned per-core workers. +//! Flags for serving QUIC from per-core workers. //! //! The workers themselves are [`moq_tokio::worker`]; this is the operator-facing -//! half, which is the only part specific to the relay. Unset (the default) keeps -//! QUIC on the shared runtime with everything else. +//! half, which is the only part specific to the relay. Unset (the default) shards +//! QUIC across one worker per core where the kernel can steer a group (Linux) and +//! keeps it on the shared runtime elsewhere. use serde::{Deserialize, Serialize}; /// How the relay lays its QUIC work out over threads. -#[derive(usage::Args, Clone, Debug, Deserialize, Serialize)] +#[derive(usage::Args, Clone, Debug, Default, Deserialize, Serialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] pub struct Config { - /// Serve QUIC from this many single-threaded workers instead of the shared - /// runtime, each pinned to a core with its own socket on the listen address. + /// Serve QUIC from this many single-threaded workers, each with its own + /// socket on the listen address, instead of the shared runtime. /// /// A connection is handled start to finish by one worker, which trades the /// shared runtime's load balancing for no cross-thread traffic per packet. - /// Linux-only; every worker shares the relay's served identity and reloads. - /// Unset (the default) keeps QUIC on the shared runtime. + /// Unset (the default) runs one worker per core on Linux and the shared + /// runtime elsewhere; `0` keeps the shared runtime. Any other count is + /// Linux-only. Every worker shares the relay's served identity and reloads. #[usage(long = "runtime-workers", env = "MOQ_RUNTIME_WORKERS", setting = "runtime.workers")] pub workers: Option, - /// Pin each worker to a CPU core, defaulting to on. + /// Pin each worker to a CPU core, defaulting to off. /// /// The mode's measured win comes from each worker owning a socket and a /// runtime, not from pinning: on a single-socket machine, pinned and unpinned /// benchmark inside run-to-run noise of each other. Pinning stops the - /// scheduler migrating a busy worker, which should matter on a multi-socket - /// or NUMA machine, and costs nothing elsewhere. Turn it off when sharing - /// the machine with something that manages CPU placement itself. - #[usage( - long = "runtime-pin", - env = "MOQ_RUNTIME_PIN", - setting = "runtime.pin", - default = "true", - bool_value - )] + /// scheduler migrating a busy worker, which may matter on a multi-socket or + /// NUMA machine with the cores to itself. It is off by default because every + /// process pins its workers to the same first cores: two relays, or a relay + /// limited to fewer CPUs than the host has, would crowd onto those cores. + #[usage(long = "runtime-pin", env = "MOQ_RUNTIME_PIN", setting = "runtime.pin", bool_value)] pub pin: bool, /// Drive the QUIC workers with io_uring instead of tokio. @@ -45,8 +42,9 @@ pub struct Config { /// with `UDP_GRO`, `UDP_SEGMENT` send), userspace timers, and a local task /// set; sessions are moq-lite only (native raw QUIC and browser /// WebTransport alike), and everything else (HTTP, WebSocket, cluster - /// dials, stats) stays on the shared runtime. Requires `runtime-workers`, - /// Linux 6.12+, and refuses to start anywhere it cannot deliver. + /// dials, stats) stays on the shared runtime. Requires QUIC workers (any + /// `runtime-workers` but `0`), Linux 6.12+, and refuses to start anywhere it + /// cannot deliver. #[usage( long = "runtime-io-uring", env = "MOQ_RUNTIME_IO_URING", @@ -60,10 +58,15 @@ impl Config { /// The worker group this asks for, or `None` to keep QUIC on the shared /// runtime. /// - /// Zero workers reads as unset rather than as an error: it is the natural way - /// to switch the mode off from a config file that already has the section. - pub fn workers(&self) -> Option { - let count = self.workers.filter(|count| *count > 0)?; + /// `per_core` is the count an unset `workers` falls back to: one per core + /// where the kernel can steer a group, or `None` where the listener cannot be + /// sharded at all. `0` is the explicit way back to the shared runtime. + pub(crate) fn workers(&self, per_core: Option) -> Option { + let count = match self.workers { + Some(0) => return None, + Some(count) => count, + None => per_core?, + }; Some(moq_tokio::worker::Config::new(count).with_pin(self.pin)) } @@ -72,13 +75,3 @@ impl Config { self.io_uring } } - -impl Default for Config { - fn default() -> Self { - Self { - workers: None, - pin: true, - io_uring: false, - } - } -} diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 5df9462027..c0f9b57a60 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -169,23 +169,10 @@ impl Workers { // The group owns the reuseport invariants: it locks the port against a // concurrently-constructing same-UID group (the bind probe covers one // that already holds it), refuses a size the steering filter could not - // address, and hands out members in the order the kernel numbers them - // by. - let mut forming = moq_sock::shard::Group::acquire(requested, config.count) - .with_context(|| format!("failed to take the reuseport group on {requested}"))?; - let count = forming.count(); - - let mut claims = Vec::with_capacity(count as usize); - while let Some(member) = forming.member() { - let shard = member.shard(); - let claim = member - .bind() - .with_context(|| format!("failed to bind worker {} on {}", shard.index(), forming.addr()))?; - claims.push(claim); - } - let mut group = forming - .complete(claims) - .context("failed to complete the reuseport group")?; + // address, and binds members in the order the kernel numbers them by. + let mut group = moq_sock::shard::bind(requested, config.count) + .with_context(|| format!("failed to bind the reuseport group on {requested}"))?; + let count = group.count(); let mut members = Vec::with_capacity(count as usize); while let Some(member) = group.member().context("failed to clone a reuseport member")? { members.push(member); diff --git a/rs/moq-sock/Cargo.toml b/rs/moq-sock/Cargo.toml index 919c9334f2..e1c63232f5 100644 --- a/rs/moq-sock/Cargo.toml +++ b/rs/moq-sock/Cargo.toml @@ -19,7 +19,6 @@ fast-apple-datapath = [] [dependencies] core_affinity = "0.8" libc = { workspace = true } -rand = "0.10" socket2 = { version = "0.6", features = ["all"] } thiserror = "2" tracing = "0.1" @@ -31,4 +30,5 @@ windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_System_ cfg_aliases = "0.2" [dev-dependencies] +rand = "0.10" tracing-test = "0.2" diff --git a/rs/moq-sock/README.md b/rs/moq-sock/README.md index d5a4e69743..800c803774 100644 --- a/rs/moq-sock/README.md +++ b/rs/moq-sock/README.md @@ -7,14 +7,14 @@ runtimes (`moq-tokio` and `moq-uring`). `SO_REUSEPORT` for worker-per-core groups (Linux-only, refused loudly elsewhere). - **`shard`**: forming and steering a reuseport group by QUIC connection id. - A `shard::Group` holds the port (probing and locking against a second - same-UID group), fixes the member count, and hands out one `Member` per slot - in index order. Binding a member yields an opaque claim, and completing the - group with every claim attaches a classic-BPF filter before any socket is - released for serving. The completed group retains every member socket, so - dropping one serving handle cannot renumber the survivors. A `Shard` names - the slot a member ended up in, and `cid_prefix` is the byte its issued ids - lead with. + `shard::bind` holds the port (probing and locking against a second same-UID + group), binds every member in index order, and attaches a classic-BPF filter + before any socket is released for serving. The bound group retains every + member socket, so dropping one serving handle cannot renumber the survivors. + A `Shard` names the slot a member holds, which the endpoint serving it names + in every connection id it issues (`moq_quic::HashedConnectionIdGenerator::set_shard`). + `shard::per_core` is the default group size: one per core on Linux, and no + group elsewhere. - **`cpu`**: pinning worker threads to cores. - **`udp`**: sending and receiving with GSO, GRO, and ECN, a hard fork of [quinn-udp](https://github.com/quinn-rs/quinn). All credit for it goes to diff --git a/rs/moq-sock/src/lib.rs b/rs/moq-sock/src/lib.rs index a0392b4f30..fa3f6641f4 100644 --- a/rs/moq-sock/src/lib.rs +++ b/rs/moq-sock/src/lib.rs @@ -2,7 +2,7 @@ //! runtimes. //! //! [`bind`] opens dual-stack UDP/TCP sockets with sane buffers; -//! [`shard::Group`] forms and steers an `SO_REUSEPORT` group by QUIC connection +//! [`shard::bind`] forms and steers an `SO_REUSEPORT` group by QUIC connection //! id, so a worker-per-core listener keeps each connection on the socket that //! owns it; [`cpu`] pins those workers; [`udp`] (forked from quinn-udp) sends //! and receives with GSO, GRO, and ECN. Both `moq-tokio` and `moq-uring` build diff --git a/rs/moq-sock/src/shard.rs b/rs/moq-sock/src/shard.rs index adec7f7ecd..d1ec1ab4a7 100644 --- a/rs/moq-sock/src/shard.rs +++ b/rs/moq-sock/src/shard.rs @@ -12,37 +12,33 @@ //! //! # The group //! -//! [`Group`] is the whole entry point. It takes the port, fixes the member -//! count, and hands out one [`Member`] per slot in index order. Binding yields -//! an opaque [`Claim`], so no socket can be served while the group is partial. -//! [`Group::complete`] accepts every claim, attaches the steering filter, and -//! returns a [`Bound`] group that hands out sockets while retaining every -//! underlying member for its own lifetime. +//! [`bind`] is the whole entry point. It takes the port, binds every member in +//! index order, attaches the steering filter, and returns a [`Bound`] group +//! that hands out one serving [`Socket`] per member while retaining every +//! underlying socket for its own lifetime. [`per_core`] is the size a server +//! should ask for by default. //! //! ```no_run //! # fn example(addr: std::net::SocketAddr) -> Result<(), Box> { -//! let mut group = moq_sock::shard::Group::acquire(addr, 4)?; -//! let mut claims = Vec::new(); -//! while let Some(member) = group.member() { -//! claims.push(member.bind()?); -//! } -//! let mut group = group.complete(claims)?; +//! let mut group = moq_sock::shard::bind(addr, 4)?; //! let mut members = Vec::new(); //! while let Some(member) = group.member()? { //! members.push((member.shard(), member.into_inner())); //! } -//! // Serve each socket, issuing connection IDs led by `cid_prefix(shard)`. +//! // Serve each socket with an endpoint whose connection IDs name its shard. //! # Ok(()) //! # } //! ``` //! //! # The encoding //! -//! [`cid_prefix`] makes the first byte of every connection ID this member issues -//! congruent to its index modulo the group size, keeping the remaining -//! `256 / count` values of that byte (and every later byte) random. The kernel -//! selects `socks[A % count]` for a returned accumulator `A`, so feeding it that -//! byte lands on the owner. +//! The filter reads the first byte of a packet's destination connection ID and +//! reduces it modulo the group size. The endpoint serving a member has to issue +//! connection IDs whose first byte is congruent to the member's +//! [`Shard::index`] modulo [`Shard::count`], which is what +//! `moq_quic::HashedConnectionIdGenerator::set_shard` does. The kernel selects +//! `socks[A % count]` for a returned accumulator `A`, so feeding it that byte +//! lands on the owner. //! //! A client's first packets carry a connection ID the *client* invented, which //! encodes nothing. Those hash to an arbitrary member, and that is fine: whoever @@ -57,24 +53,23 @@ //! (the kernel pulls the header off first) and uses the return value as the //! index. That is the whole rule in seven instructions, with no `CAP_BPF`, no //! BPF toolchain in the build, and no map to keep alive across restarts. The -//! cost is that the kernel selects by *position*, so the group has to be built -//! once, in order, and never resized, which is what [`Group`] is for. +//! cost is that the kernel selects by *position*, so the group has to be bound +//! once, in order, and never resized, which is why [`bind`] does all of it. use std::io; use std::net::{SocketAddr, UdpSocket}; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; /// One member's slot in a group of sockets sharing a port via `SO_REUSEPORT`. /// /// Every member binds the same address and the kernel spreads inbound datagrams /// across the group, so N servers on N threads can serve one port without a /// shared socket between them. The kernel identifies a member by its *position* -/// in the group, which is why only [`Group::member`] mints one: a shard is -/// meaningful only inside a group that was bound once, in index order, and is -/// never resized. +/// in the group, which is why only [`bind`] mints one: a shard is meaningful +/// only inside a group that was bound once, in index order, and is never +/// resized. /// -/// Carry it wherever the member issues connection IDs, which [`cid_prefix`] -/// leads with its index. +/// Carry it wherever the member issues connection IDs, which have to name it +/// (see [the encoding](self#the-encoding)). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Shard { index: u16, @@ -82,17 +77,6 @@ pub struct Shard { } impl Shard { - /// Slot `index` of a group of `count` sockets, or `None` if that slot cannot - /// exist: a `count` of zero or past [`MAX_SHARDS`], or an `index` at or past - /// the end. - /// - /// The upper bound is what makes every shard safe to steer: [`cid_prefix`] - /// spends `256 / count` values of a byte, so a larger group would leave it - /// none to spend. - fn new(index: u16, count: u16) -> Option { - (count <= MAX_SHARDS && index < count).then_some(Self { index, count }) - } - /// This member's position in the group, from zero. pub fn index(self) -> u16 { self.index @@ -133,200 +117,93 @@ pub enum Error { /// No port could be resolved for an ephemeral request. #[error("failed to resolve an ephemeral port")] Resolve(#[source] io::Error), + + /// A member could not bind, including [`io::ErrorKind::Unsupported`] + /// anywhere but Linux and [`io::ErrorKind::AddrInUse`] when the address is + /// already held. + #[error("failed to bind reuseport member {index}")] + Bind { + /// The member that failed. + index: u16, + /// What the bind returned. + #[source] + source: io::Error, + }, + + /// The steering filter could not be attached. + #[error("failed to steer the reuseport group")] + Steer(#[source] io::Error), } -/// A `SO_REUSEPORT` group being formed. +/// One shard per core this process may run on, or `None` where the kernel +/// cannot steer a group, which is everywhere but Linux. /// -/// Everything a steered group has to get right lives here rather than in its +/// Counts the cores the process is allowed (its affinity mask and cgroup +/// quota), not the machine's, so a container limited to two CPUs runs two +/// shards. Capped at [`MAX_SHARDS`]. +pub fn per_core() -> Option { + if !cfg!(target_os = "linux") { + return None; + } + let cores = std::thread::available_parallelism().map_or(1, |cores| cores.get()); + Some(cores.min(usize::from(MAX_SHARDS)) as u16) +} + +/// Bind a steered group of `count` sockets sharing `addr`. +/// +/// Everything a steered group has to get right happens here rather than in its /// caller, because none of it is visible in a socket afterwards: /// /// - The port is locked before the first member binds and stays locked until the -/// group and every member it handed out are gone, so a second same-UID group -/// cannot interleave into this one. Hold the group for as long as its sockets -/// are served. +/// returned group and every socket it handed out are gone, so a second +/// same-UID group cannot interleave into this one. Hold the group for as long +/// as its sockets are served. /// - The member count is checked once, against what the steering filter can /// address, and never changes. -/// - [`member`](Self::member) hands out slots in index order, and a [`Member`] -/// is the only thing that can bind into the group. The kernel numbers the -/// group by bind order, so a member that binds out of turn is refused rather -/// than silently taking a sibling's slot. -/// - [`complete`](Self::complete) accepts only every claim from this group and -/// attaches the filter before returning a [`Bound`] group. A claim exposes no -/// socket, so a partial group cannot serve. -/// - [`Bound`] retains every underlying socket. A serving handle may be dropped -/// without removing its member and renumbering the survivors. +/// - Members bind in index order, which is the order the kernel numbers them +/// by, and the filter is attached before any socket is released. /// -/// Members may be bound wherever their sockets are served, a worker's own thread -/// included, as long as each one binds before the next takes its turn. -#[derive(Debug)] -pub struct Group { - count: u16, - - /// The next slot to hand out. Not the same as the next slot to *bind*: a - /// member may be bound on a thread of its own, which is what [`State`] - /// tracks. - next: u16, - - state: Arc, -} - -impl Group { - /// Take the port and fix the group at `count` members. - /// - /// A `count` of zero is a group of one: a lone member is a valid group, and - /// the callers that size a group from a worker count would otherwise each - /// clamp it themselves. - /// - /// An ephemeral port (`0`) is resolved here, with a plain bind that is - /// dropped again, and the group then takes that port like a named one. - /// Never a reuseport bind of port `0`: Linux counts a port held only by - /// same-UID reuseport sockets as free for one, so the kernel could hand - /// back a port another group holds and this one would silently join it. A - /// plain bind conflicts with every holder. Another socket taking the port - /// before the first member binds fails that bind with - /// [`io::ErrorKind::AddrInUse`]. - pub fn acquire(addr: SocketAddr, count: u16) -> Result { - let count = count.max(1); - if count > MAX_SHARDS { - return Err(Error::Count { count, max: MAX_SHARDS }); - } - - let addr = match addr.port() { - 0 => crate::bind::udp(crate::bind::Udp::new(addr)) - .and_then(|socket| socket.local_addr()) - .map_err(Error::Resolve)?, - _ => addr, - }; - - let port = addr.port(); - let lock = Lock::acquire(port).map_err(|_| Error::Overlap { port })?; - - Ok(Self { - count, - next: 0, - state: Arc::new(State::new(addr, lock)), - }) - } - - /// How many sockets share the port. - pub fn count(&self) -> u16 { - self.count - } - - /// The address the group holds, with an ephemeral port already resolved. - pub fn addr(&self) -> SocketAddr { - self.state.addr - } - - /// The next slot to bind, or `None` once every slot has been handed out. - /// - /// Bind each member before taking the next: they join the group in the order - /// they bind, and that order is the only thing the kernel knows them by. - pub fn member(&mut self) -> Option { - let shard = Shard::new(self.next, self.count)?; - self.next += 1; - Some(Member { - shard, - state: self.state.clone(), - }) - } - - /// Complete the group and make its sockets available for serving. - /// - /// Every member must have bound successfully, and the claims must belong to - /// this group in slot order. Failure closes the partial group as a whole. - pub fn complete(self, claims: Vec) -> io::Result { - if claims.len() != usize::from(self.count) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "reuseport group needs {} bound members; {} were provided", - self.count, - claims.len() - ), - )); - } - - for (index, claim) in claims.iter().enumerate() { - if !Arc::ptr_eq(&self.state, &claim.state) || usize::from(claim.shard.index()) != index { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "reuseport claims must belong to this group in slot order", - )); - } - } - - let sockets: Vec<_> = claims.into_iter().map(|claim| (claim.shard, claim.socket)).collect(); - let (_, last) = sockets.last().expect("a group always has at least one member"); - attach(last, self.count)?; - - Ok(Bound { - sockets, - next: 0, - state: self.state, - }) - } -} - -/// One member's claim on a slot in a [`Group`], which binding spends. +/// A `count` of zero is a group of one: a lone member is a valid group. /// -/// Send it wherever the socket is served, a worker's own thread included. It -/// carries the group's address and its share of the group's port lock, so a -/// member still waiting to bind cannot be raced by a second group even if its -/// group is dropped first. -#[derive(Debug)] -pub struct Member { - shard: Shard, - state: Arc, -} - -impl Member { - /// This member's slot, which its connection IDs have to encode - /// ([`cid_prefix`]). - pub fn shard(&self) -> Shard { - self.shard - } - - /// Bind this member into the group and return its opaque claim. - /// - /// Fails when a sibling has not bound yet: the kernel numbers a reuseport - /// group by bind order, so a member joining out of turn would take another's - /// slot and steer its traffic, with no error to show for it. - /// - /// The claim exposes no socket. Pass every claim to [`Group::complete`], - /// which attaches the filter before releasing serving handles. - pub fn bind(self) -> io::Result { - let mut bound = self.state.bound(); - if *bound != self.shard.index() { - return Err(io::Error::other(format!( - "reuseport member {} cannot bind while {} of {} are in: the kernel numbers a group by bind order", - self.shard.index(), - *bound, - self.shard.count(), - ))); - } +/// An ephemeral port (`0`) is resolved first, with a plain bind that is +/// dropped again, and the group then takes that port like a named one. +/// Never a reuseport bind of port `0`: Linux counts a port held only by +/// same-UID reuseport sockets as free for one, so the kernel could hand +/// back a port another group holds and this one would silently join it. A +/// plain bind conflicts with every holder. Another socket taking the port +/// before the first member binds fails that bind with +/// [`io::ErrorKind::AddrInUse`]. +pub fn bind(addr: SocketAddr, count: u16) -> Result { + let count = count.max(1); + if count > MAX_SHARDS { + return Err(Error::Count { count, max: MAX_SHARDS }); + } + + let addr = match addr.port() { + 0 => crate::bind::udp(crate::bind::Udp::new(addr)) + .and_then(|socket| socket.local_addr()) + .map_err(Error::Resolve)?, + _ => addr, + }; - let socket = bind(self.state.addr, self.shard)?; - *bound += 1; - drop(bound); + let port = addr.port(); + let lock = Lock::acquire(port).map_err(|_| Error::Overlap { port })?; - Ok(Claim { - shard: self.shard, - socket, - state: self.state, - }) + let mut sockets = Vec::with_capacity(usize::from(count)); + for index in 0..count { + let socket = member(addr, index).map_err(|source| Error::Bind { index, source })?; + sockets.push((Shard { index, count }, socket)); } -} -/// One successfully bound member of a forming [`Group`]. -/// -/// Opaque so a socket cannot be served before every member has joined and the -/// steering filter is attached. Pass all claims to [`Group::complete`]. -#[derive(Debug)] -pub struct Claim { - shard: Shard, - socket: UdpSocket, - state: Arc, + let (_, last) = sockets.last().expect("a group always has at least one member"); + attach(last, count).map_err(Error::Steer)?; + + Ok(Bound { + sockets, + next: 0, + addr, + _lock: lock, + }) } /// A complete steered group that retains every member socket. @@ -334,14 +211,16 @@ pub struct Claim { /// [`member`](Self::member) returns duplicate handles to the retained sockets. /// Dropping one handle stops serving that slot without removing it from the /// kernel group or renumbering any survivor. Dropping this group closes every -/// retained member together. +/// retained member together and releases the port. #[derive(Debug)] pub struct Bound { sockets: Vec<(Shard, UdpSocket)>, next: usize, + addr: SocketAddr, - /// Holds the port lock for exactly as long as the complete socket group. - state: Arc, + /// Held for exactly as long as the complete socket group, and released by + /// the kernel with the process. `None` on a host with no lock directory. + _lock: Option, } impl Bound { @@ -350,9 +229,9 @@ impl Bound { self.sockets.len().try_into().expect("group count fits u16") } - /// The address every member holds. + /// The address every member holds, with an ephemeral port already resolved. pub fn addr(&self) -> SocketAddr { - self.state.addr + self.addr } /// The next socket to serve, or `None` once every member was handed out. @@ -388,50 +267,11 @@ impl Socket { } } -/// What the members of a forming group share: the address and port lock it -/// holds, and how far the group has been bound. +/// Bind member `index` of the group at `addr`. /// -/// Shared rather than owned by the [`Group`] because a member is bound wherever -/// its socket is served, which is usually not where the group lives. The lock -/// rides along for the same reason: a member outstanding after its group is -/// dropped can still bind, so the port has to stay held until the last of them -/// is gone. -#[derive(Debug)] -struct State { - addr: SocketAddr, - - /// How many members have bound, which is also the only index allowed to bind - /// next. - bound: Mutex, - - /// Held until the group and its members are all dropped, and released by the - /// kernel with the process. `None` on a host with no lock directory. - _lock: Option, -} - -impl State { - fn new(addr: SocketAddr, lock: Option) -> Self { - Self { - addr, - bound: Mutex::new(0), - _lock: lock, - } - } - - /// The bound count, whatever a panicking member left behind: a failed bind - /// is reported by the count it did not advance, so there is no torn state a - /// poisoned lock would be protecting. - fn bound(&self) -> MutexGuard<'_, u16> { - self.bound.lock().unwrap_or_else(PoisonError::into_inner) - } -} - -/// Bind one member's socket into the group at `addr`. -/// -/// Private because the order is the whole invariant: reachable only through a -/// [`Member`], which a [`Group`] hands out in index order and only while it -/// holds the port. -fn bind(addr: SocketAddr, shard: Shard) -> io::Result { +/// Private because the order is the whole invariant: only [`bind`] calls this, +/// once per index in order, while it holds the port. +fn member(addr: SocketAddr, index: u16) -> io::Result { // The first member probes with a plain bind before the group forms. // `SO_REUSEPORT` groups by address and UID, so a member would otherwise // *join* any group a same-UID process already has on the address, as two @@ -445,9 +285,8 @@ fn bind(addr: SocketAddr, shard: Shard) -> io::Result { // *constructing* concurrently could each probe while the other holds // nothing yet, which is what [`Lock`] excludes; the probe's job is the // holder the lock cannot see, one that predates it or never took it, or - // one that took an ephemeral group's port after [`Group::acquire`] - // resolved it. - if shard.index() == 0 { + // one that took an ephemeral group's port after [`bind`] resolved it. + if index == 0 { drop(crate::bind::udp(crate::bind::Udp::new(addr))?); } @@ -649,29 +488,9 @@ fn prepare(dir: std::path::PathBuf) -> Option { /// The largest group the steering filter can address. /// /// It selects with one byte of the connection ID, so a member past 255 could -/// never be named: [`cid_prefix`] would have no stride to spend and the filter's -/// `byte % count` could never return that index. +/// never be named: the filter's `byte % count` could never return that index. pub const MAX_SHARDS: u16 = 256; -/// The first byte of a connection ID issued by `shard`. -/// -/// `count` values of the byte are reserved per member, so this keeps -/// `256 / count` of its randomness: 32 values across 8 members, and the other -/// bytes of the ID stay fully random either way. Connection IDs are not secrets, -/// but they are unlinkable only while they look random, which is why this spends -/// as little of that byte as it can. -pub fn cid_prefix(shard: Shard) -> u8 { - use rand::RngExt; - - let count = u32::from(shard.count()); - // The number of whole strides of `count` that fit in a byte. At least one, - // because a group is never larger than `MAX_SHARDS`. - let strides = 256 / count; - let stride = rand::rng().random_range(0..strides); - - (stride * count + u32::from(shard.index())) as u8 -} - /// Attach the steering filter to a socket in a reuseport group of `count`. /// /// Applies to the group, so once is enough, and the last member to bind is the @@ -768,152 +587,71 @@ fn program(count: u16) -> [libc::sock_filter; 7] { mod tests { use super::*; - /// A slot has to exist in the group it names, and the group has to be one - /// the steering filter can address: `cid_prefix` divides by the count, so - /// an oversized group would panic on a zero-width stride. - #[test] - fn shard_slots_are_bounded() { - assert_eq!(Shard::new(0, 1).map(|shard| shard.count()), Some(1)); - assert_eq!(Shard::new(3, 4).map(|shard| shard.index()), Some(3)); - assert!(Shard::new(4, 4).is_none()); - assert!(Shard::new(0, 0).is_none()); - assert!(Shard::new(0, MAX_SHARDS).is_some()); - assert!(Shard::new(0, MAX_SHARDS + 1).is_none()); - } - - /// The group is the only source of slots, and it hands out exactly the ones - /// it was sized for, in order. - #[test] - fn a_group_hands_out_every_slot_in_order() { - const COUNT: u16 = 4; - - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), COUNT).unwrap(); - assert_eq!(group.count(), COUNT); - - for index in 0..COUNT { - let member = group.member().expect("a slot per member"); - assert_eq!(member.shard().index(), index); - assert_eq!(member.shard().count(), COUNT); - } - assert!(group.member().is_none(), "the group cannot be resized"); - } - - /// A group of zero is a group of one: every caller sizes a group from a - /// worker count, and a lone member is a valid group. - #[test] - fn an_empty_group_holds_one_member() { - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), 0).unwrap(); - assert_eq!(group.count(), 1); - assert_eq!(group.member().map(|member| member.shard().count()), Some(1)); + /// A first byte that steers to `shard`, randomized within its stride the way + /// an endpoint's connection IDs are, so nothing but the reduction can be + /// steering. + fn prefix(shard: Shard) -> u8 { + use rand::RngExt; + let count = u32::from(shard.count()); + let stride = rand::rng().random_range(0..256 / count); + (stride * count + u32::from(shard.index())) as u8 } /// The steering filter names a member with one byte, so a group past 256 has - /// members it could never reach. Refused where the group is sized, rather - /// than when the first connection ID has no stride left to spend. + /// members it could never reach. Refused before anything binds. #[test] fn an_unaddressable_group_is_refused() { let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); - assert!(Group::acquire(addr, MAX_SHARDS).is_ok()); assert!(matches!( - Group::acquire(addr, MAX_SHARDS + 1), + bind(addr, MAX_SHARDS + 1), Err(Error::Count { count: 257, max: 256 }) )); } - /// The encoding's only real requirement: whatever byte a member issues has to - /// reduce back to that member. + /// A group can only be steered on Linux, so a default elsewhere is a single + /// socket rather than a group. #[test] - fn a_prefix_reduces_to_its_own_shard() { - // `MAX_SHARDS` is the interesting end: one stride per member, so the prefix - // is the index itself and there is no randomness left to spend. - for count in [1u16, 2, 3, 4, 7, 8, 16, 64, 255, MAX_SHARDS] { - for index in 0..count { - let shard = Shard::new(index, count).unwrap(); - // Sampled, since the prefix is randomized within the member's stride. - for _ in 0..64 { - let prefix = cid_prefix(shard); - assert_eq!( - u16::from(prefix) % count, - index, - "prefix {prefix} of shard {index}/{count} steers elsewhere" - ); - } - } - } + fn per_core_is_linux_only() { + let count = per_core(); + assert_eq!(count.is_some(), cfg!(target_os = "linux")); + assert!(count.is_none_or(|count| (1..=MAX_SHARDS).contains(&count))); } - /// Every member of a group has to be reachable, or the ones that aren't sit - /// idle while their share of the traffic goes to a sibling. + /// Elsewhere the kernel cannot steer, so a group is refused at its first + /// member rather than served by a socket that silently balances nothing. #[test] - fn prefixes_cover_every_shard() { - const COUNT: u16 = 8; - let mut seen = std::collections::HashSet::new(); - for index in 0..COUNT { - let shard = Shard::new(index, COUNT).unwrap(); - for _ in 0..256 { - seen.insert(u16::from(cid_prefix(shard)) % COUNT); - } - } - assert_eq!(seen.len(), usize::from(COUNT)); + #[cfg(not(target_os = "linux"))] + fn a_group_is_refused_off_linux() { + let err = bind("127.0.0.1:0".parse().unwrap(), 2).expect_err("steered a group off Linux"); + assert!(matches!(err, Error::Bind { index: 0, ref source } if source.kind() == io::ErrorKind::Unsupported)); } - /// An ephemeral request gives the first member a port of the kernel's - /// choosing, and the group is only balanced over a port every member holds. + /// The group hands out exactly the slots it was sized for, in order, every + /// one on the port the first member resolved. #[test] #[cfg(target_os = "linux")] - fn a_group_shares_one_ephemeral_port() { + fn a_group_hands_out_every_slot_on_one_port() { const COUNT: u16 = 3; - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), COUNT).unwrap(); - let claims: Vec = (0..COUNT) - .map(|_| group.member().expect("a slot per member").bind().expect("bind member")) - .collect(); - let mut group = group.complete(claims).expect("complete group"); - let sockets: Vec = (0..COUNT) - .map(|_| group.member().unwrap().expect("a socket per member").into_inner()) - .collect(); - + let mut group = bind("127.0.0.1:0".parse().unwrap(), COUNT).unwrap(); + assert_eq!(group.count(), COUNT); let addr = group.addr(); assert_ne!(addr.port(), 0, "the group holds the port its first member bound"); - for socket in &sockets { - assert_eq!( - socket.local_addr().unwrap(), - addr, - "every member holds the group's port" - ); - } - } - - /// The kernel numbers a group by bind order, so a member that binds out of - /// turn would take a sibling's slot and steer its traffic. Refused instead, - /// which is the one part of the ordering rule a caller can still get wrong - /// once its members are bound on threads of their own. - #[test] - #[cfg(target_os = "linux")] - fn binding_out_of_order_is_refused() { - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), 2).unwrap(); - let first = group.member().expect("first slot"); - let second = group.member().expect("second slot"); - second.bind().expect_err("the second member cannot bind first"); - - // Refused, not deferred: nothing joined the group, so the slot the - // kernel would have handed the second member is still the first's. - first.bind().expect("bind the first member"); - assert_ne!(group.addr().port(), 0); + for index in 0..COUNT { + let member = group.member().unwrap().expect("a socket per member"); + assert_eq!(member.shard(), Shard { index, count: COUNT }); + assert_eq!(member.into_inner().local_addr().unwrap(), addr); + } + assert!(group.member().unwrap().is_none(), "the group cannot be resized"); } - /// A forming group exposes claims, not sockets, and refuses completion until - /// every declared member is bound. + /// A group of zero is a group of one: a lone member is a valid group. #[test] #[cfg(target_os = "linux")] - fn a_partial_group_cannot_serve() { - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), 2).unwrap(); - let claim = group.member().unwrap().bind().expect("bind first member"); - - group - .complete(vec![claim]) - .expect_err("a partial group must not complete"); + fn an_empty_group_holds_one_member() { + let group = bind("127.0.0.1:0".parse().unwrap(), 0).unwrap(); + assert_eq!(group.count(), 1); } /// The whole point, end to end against a real kernel: a packet carrying a @@ -927,12 +665,7 @@ mod tests { fn a_connection_id_reaches_its_own_member() { const COUNT: u16 = 4; - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), COUNT).unwrap(); - let mut claims = Vec::new(); - while let Some(member) = group.member() { - claims.push(member.bind().expect("bind group member")); - } - let mut group = group.complete(claims).expect("complete group"); + let mut group = bind("127.0.0.1:0".parse().unwrap(), COUNT).unwrap(); let mut sockets = Vec::new(); let mut shards = Vec::new(); while let Some(member) = group.member().expect("clone retained socket") { @@ -950,7 +683,7 @@ mod tests { } for (index, shard) in shards.into_iter().enumerate() { - let prefix = cid_prefix(shard); + let prefix = prefix(shard); // A short header: form bit clear, connection ID straight after. let short = [0x40, prefix, 1, 2, 3, 4, 5, 6, 7, 8]; @@ -980,12 +713,7 @@ mod tests { #[test] #[cfg(target_os = "linux")] fn dropping_a_serving_handle_does_not_resize_the_group() { - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), 2).unwrap(); - let mut claims = Vec::new(); - while let Some(member) = group.member() { - claims.push(member.bind().expect("bind group member")); - } - let mut group = group.complete(claims).expect("complete group"); + let mut group = bind("127.0.0.1:0".parse().unwrap(), 2).unwrap(); let first = group.member().unwrap().expect("first serving socket"); let second = group.member().unwrap().expect("second serving socket"); let shard = second.shard(); @@ -995,7 +723,7 @@ mod tests { .unwrap(); drop(first); - let packet = [0x40, cid_prefix(shard), 1, 2, 3, 4, 5, 6, 7, 8]; + let packet = [0x40, prefix(shard), 1, 2, 3, 4, 5, 6, 7, 8]; let sender = crate::bind::udp(crate::bind::Udp::new("127.0.0.1:0".parse().unwrap())).unwrap(); sender.send_to(&packet, group.addr()).unwrap(); @@ -1003,88 +731,47 @@ mod tests { assert_eq!(socket.recv_from(&mut buf).unwrap().0, packet.len()); } - /// A second group on a named port must lose it while the first is alive: two - /// groups constructing at once would each pass the bind probe before either - /// held the port, then interleave into one group whose filter steers each - /// process's traffic to the other's sockets. + /// A second group on a held port must lose it while the first is alive, for + /// a named port and a resolved ephemeral one alike: two groups constructing + /// at once would each pass the bind probe before either held the port, then + /// interleave into one group whose filter steers each process's traffic to + /// the other's sockets. #[test] #[cfg(target_os = "linux")] fn a_second_group_cannot_take_the_port() { // A port the kernel just handed out and nothing holds any more, which is // as close to a reserved port as a test gets. - let addr = { + let named = { let probe = crate::bind::udp(crate::bind::Udp::new("127.0.0.1:0".parse().unwrap())).unwrap(); probe.local_addr().unwrap() }; - let mut first = Group::acquire(addr, 1).unwrap(); - let claim = first.member().unwrap().bind().expect("bind the first group"); - let first = first.complete(vec![claim]).expect("complete the first group"); - - assert!( - matches!(Group::acquire(addr, 1), Err(Error::Overlap { .. })), - "a second group took a held port" - ); - - // The lock dies with the group, so the port is takeable again. - drop(first); - Group::acquire(addr, 1).expect("the released port must be takeable again"); - } - - /// An ephemeral request is resolved to a concrete port and locked before any - /// member binds. A reuseport bind of port `0` could land on a port another - /// same-UID group holds, and join it. - #[test] - #[cfg(target_os = "linux")] - fn an_ephemeral_group_takes_its_port_up_front() { - let group = Group::acquire("127.0.0.1:0".parse().unwrap(), 1).unwrap(); - let addr = group.addr(); - assert_ne!(addr.port(), 0, "the port is resolved before any member binds"); + for addr in [named, "127.0.0.1:0".parse().unwrap()] { + let first = bind(addr, 1).expect("bind the first group"); + let addr = first.addr(); - assert!( - matches!(Group::acquire(addr, 1), Err(Error::Overlap { .. })), - "a second group took an ephemeral group's port" - ); - } - - /// A same-UID reuseport socket that takes an ephemeral group's port between - /// its resolution and the first member's bind must fail that bind, not be - /// joined by it. - #[test] - #[cfg(target_os = "linux")] - fn an_ephemeral_port_taken_before_binding_is_refused() { - let mut group = Group::acquire("127.0.0.1:0".parse().unwrap(), 1).unwrap(); - let _intruder = crate::bind::udp(crate::bind::Udp::new(group.addr()).with_reuse_port(true)).unwrap(); + assert!( + matches!(bind(addr, 1), Err(Error::Overlap { .. })), + "a second group took a held port" + ); - let err = group.member().unwrap().bind().expect_err("joined the intruder's group"); - assert_eq!(err.kind(), io::ErrorKind::AddrInUse); + // The lock dies with the group, so the port is takeable again. + drop(first); + bind(addr, 1).expect("the released port must be takeable again"); + } } - /// A member outlives the group that handed it out and can still bind, so the - /// exclusion has to outlive the group too. Releasing the port at the group's - /// drop would let a second group probe and interleave into this one while its - /// first member had yet to join. + /// A same-UID reuseport socket already on the address, one the lock cannot + /// see, must fail the first member's bind rather than be joined by it. #[test] #[cfg(target_os = "linux")] - fn an_outstanding_member_keeps_the_port() { - let addr = { - let probe = crate::bind::udp(crate::bind::Udp::new("127.0.0.1:0".parse().unwrap())).unwrap(); - probe.local_addr().unwrap() - }; - - let mut group = Group::acquire(addr, 1).unwrap(); - let member = group.member().expect("the only slot"); - drop(group); - - assert!( - matches!(Group::acquire(addr, 1), Err(Error::Overlap { .. })), - "a second group took a port an unbound member still holds" - ); - - let claim = member.bind().expect("bind the outstanding member"); + fn a_held_address_is_refused_not_joined() { + let intruder = + crate::bind::udp(crate::bind::Udp::new("127.0.0.1:0".parse().unwrap()).with_reuse_port(true)).unwrap(); + let addr = intruder.local_addr().unwrap(); - drop(claim); - Group::acquire(addr, 1).expect("the released port must be takeable again"); + let err = bind(addr, 1).expect_err("joined the intruder's group"); + assert!(matches!(err, Error::Bind { index: 0, ref source } if source.kind() == io::ErrorKind::AddrInUse)); } /// The jumps are offsets from the *following* instruction, so an off-by-one diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 16959fd75c..0acf266745 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -292,8 +292,6 @@ impl Default for Config { } } -#[cfg(feature = "quic")] -pub(crate) use moq_sock::shard::Shard; /// One server's socket in a complete `SO_REUSEPORT` group. /// /// Crate-private on purpose: [`crate::worker::Workers`] is the only thing that diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index b0b5b08414..53ab291a6b 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -597,15 +597,11 @@ impl Socket { }) } - /// Close the socket, unless a [`crate::server::SocketRetainer`] still holds it. + /// Close the socket. A reuseport member stays in its group regardless: the + /// group retains a handle of its own. fn release(&self) { self.io.write().unwrap().take(); } - - /// A handle keeping the socket open after it is released, or `None` once it is. - fn retain(&self) -> Option> { - self.io.read().unwrap().clone() - } } impl driver::AsyncUdpSocket for Socket { @@ -773,7 +769,11 @@ impl Server { count = shard.count(), "encoding the shard in connection IDs" ); - endpoint_config.cid_generator(Arc::new(move || Box::new(ShardIdGenerator::new(shard)))); + endpoint_config.cid_generator(Arc::new(move || { + let mut cids = moq_quic::HashedConnectionIdGenerator::new(); + cids.set_shard(shard.index(), shard.count()); + Box::new(cids) + })); } else if let Some(load_balancer) = load_balancer { let server_id = load_balancer.id; let nonce_len = load_balancer.nonce; @@ -840,11 +840,6 @@ impl Server { self.quic.wait_all_draining().await; self.socket.release(); } - - /// A handle keeping the socket open after this server is gone. - pub fn retain(&self) -> Option> { - self.socket.retain() - } } // ── QuicRequest ───────────────────────────────────────────────────── @@ -947,41 +942,6 @@ pub(crate) async fn accept( } } -// ── ShardIdGenerator ──────────────────────────────────────────────── - -/// Connection IDs whose first byte names the reuseport member that owns them, -/// so the group's steering filter can route later packets back to it. -struct ShardIdGenerator { - shard: crate::listen::Shard, -} - -impl ShardIdGenerator { - /// Only the first byte is spoken for; the rest stays random. - const LEN: usize = 8; - - fn new(shard: crate::listen::Shard) -> Self { - Self { shard } - } -} - -impl driver::ConnectionIdGenerator for ShardIdGenerator { - fn generate_cid(&mut self) -> driver::ConnectionId { - use rand::RngExt; - let mut cid = Vec::with_capacity(Self::LEN); - cid.push(moq_sock::shard::cid_prefix(self.shard)); - cid.extend(rand::rng().random_iter::().take(Self::LEN - 1)); - driver::ConnectionId::new(cid.as_slice()) - } - - fn cid_len(&self) -> usize { - Self::LEN - } - - fn cid_lifetime(&self) -> Option { - None - } -} - // ── ServerIdGenerator ─────────────────────────────────────────────── struct ServerIdGenerator { diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 754d40ff57..b9b82a7bee 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -238,24 +238,6 @@ pub struct Server { websocket: Option, } -/// A worker member's QUIC socket, kept in the reuseport group after the serving -/// [`Server`] is gone. -/// -/// Dropping or closing a serving server closes its socket, which renumbers the -/// survivors. The worker group holds one of these per member until serving has -/// stopped, so a dropped or finished member leaves the steering intact. The -/// fields are never read: holding the sockets is what keeps them open. -/// -/// Only compiled with a QUIC backend, matching the worker group that is its -/// only caller. -#[derive(Clone, Debug)] -#[allow(dead_code)] -#[cfg(feature = "quic")] -pub(crate) struct SocketRetainer { - #[cfg(feature = "quic")] - quic: Option>, -} - impl Server { /// Build a server from its config, binding the QUIC socket up front. /// @@ -518,19 +500,6 @@ impl Server { crate::tls::Certificates::empty() } - /// Clone this server's QUIC endpoint, keeping its socket in the reuseport - /// group after this server is gone. - /// - /// Crate-private: only the worker group retains sockets this way. A worker - /// member serves the same QUIC backend as the tokio listener. - #[cfg(feature = "quic")] - pub(crate) fn retain(&self) -> SocketRetainer { - SocketRetainer { - #[cfg(feature = "quic")] - quic: self.quic.as_ref().and_then(|server| server.retain()), - } - } - #[cfg(not(any( feature = "quic", feature = "iroh", diff --git a/rs/moq-tokio/src/worker/group.rs b/rs/moq-tokio/src/worker/group.rs index c98642d752..ff93a159aa 100644 --- a/rs/moq-tokio/src/worker/group.rs +++ b/rs/moq-tokio/src/worker/group.rs @@ -6,7 +6,7 @@ use std::sync::{ }; use super::Config; -use crate::{Error, Result, Server, abort::AbortOnDrop, listen::Socket as ShardSocket, server::SocketRetainer}; +use crate::{Error, Result, Server, abort::AbortOnDrop, listen::Socket as ShardSocket}; /// A bound group of QUIC workers sharing one port. /// @@ -91,20 +91,16 @@ impl Workers { // The group owns everything a reuseport group has to get right: it takes // the port before the first member binds and holds it until the group is // dropped, refuses a size the steering filter could not address, and - // hands out one member per slot in the order the kernel numbers them by. - let mut forming = moq_sock::shard::Group::acquire(requested, config.count).map_err(|err| match err { + // binds one member per slot in the order the kernel numbers them by. + let mut group = moq_sock::shard::bind(requested, config.count).map_err(|err| match err { moq_sock::shard::Error::Count { count, max } => Error::WorkerCount { count, max }, - moq_sock::shard::Error::Resolve(err) => crate::quic::Error::BindSocket(err).into(), - // The port lock is the only other way to lose the address, and it is - // held by exactly one thing: another group of this UID. - _ => Error::WorkerOverlap { addr: requested }, + moq_sock::shard::Error::Overlap { .. } => Error::WorkerOverlap { addr: requested }, + moq_sock::shard::Error::Resolve(err) + | moq_sock::shard::Error::Bind { source: err, .. } + | moq_sock::shard::Error::Steer(err) => crate::quic::Error::BindSocket(err).into(), + err => crate::quic::Error::BindSocket(std::io::Error::other(err)).into(), })?; - let count = forming.count(); - let mut claims = Vec::with_capacity(count as usize); - while let Some(member) = forming.member() { - claims.push(member.bind().map_err(crate::quic::Error::BindSocket)?); - } - let mut group = forming.complete(claims).map_err(crate::quic::Error::BindSocket)?; + let count = group.count(); let shared = Arc::new(Shared::default()); @@ -178,21 +174,6 @@ impl Workers { None => Arc::new(Shared::default()), }; - // Retain every socket before handing any server out: dropping a server - // afterwards closes its endpoint, but the retainer keeps the socket in - // the group, so the kernel never renumbers a survivor. - let mut retainers = Vec::with_capacity(self.workers.len()); - for worker in &self.workers { - retainers.push( - worker - .server - .as_ref() - .expect("a bound worker holds its server before the split") - .retain(), - ); - } - *shared.retainers.lock().unwrap() = retainers; - Group { workers: self.workers, certificates: self.certificates, @@ -426,16 +407,14 @@ fn join(threads: Vec<(u16, std::thread::JoinHandle<()>)>) { } } -/// What the workers share: the shutdown signal and the retained sockets. +/// What the workers share: the shutdown signal. /// /// One per group, held by the group and borrowed by every spawner. Cancelling -/// it stops every worker thread; the retainers keep every socket in the -/// reuseport group until the group is gone. +/// it stops every worker thread. #[derive(Debug, Default)] struct Shared { shutdown: AtomicBool, notify: tokio::sync::Notify, - retainers: std::sync::Mutex>, } impl Shared { diff --git a/rs/moq-uring/Cargo.toml b/rs/moq-uring/Cargo.toml index a0e1f2513e..83595fdf5e 100644 --- a/rs/moq-uring/Cargo.toml +++ b/rs/moq-uring/Cargo.toml @@ -32,7 +32,6 @@ moq-clock = { workspace = true, optional = true } moq-net = { workspace = true } moq-quic = { workspace = true, features = ["bloom", "rustls-aws-lc-rs"], optional = true } moq-sock = { workspace = true } -rand = "0.10" rustc-hash = { workspace = true } rustls = { version = "0.23", optional = true } # OS-native server-cert verifier for the rustls backends. Version matched to diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index b1dc8fc86d..28d9986463 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -27,7 +27,7 @@ use rustls::pki_types::{CertificateDer, PrivateKeyDer}; #[cfg(feature = "qlog")] use super::qlog; -use super::{Congestion, Error, Identity, SEGMENT, Transport, client, endpoint::CID_LEN, server}; +use super::{Congestion, Error, Identity, SEGMENT, Transport, client, server}; use crate::udp; /// Per-stream flow control credit. @@ -102,37 +102,21 @@ fn provider() -> Arc { /// largest datagram we tell peers we can receive. pub(crate) fn endpoint_config(shard: Option) -> Result, Error> { let mut config = moq_quic::EndpointConfig::default(); - config.cid_generator(Arc::new(move || Box::new(Cids { shard }))); + // moq-quic asks its generator for every id it issues, dials and rotations + // alike, so naming the shard here steers all of them. + if let Some(shard) = shard { + config.cid_generator(Arc::new(move || { + let mut cids = moq_quic::HashedConnectionIdGenerator::new(); + cids.set_shard(shard.index(), shard.count()); + Box::new(cids) + })); + } config .max_udp_payload_size(SEGMENT as u16) .map_err(|err| Error::Quic(err.to_string()))?; Ok(Arc::new(config)) } -/// Mints the endpoint's connection ids, steering prefix included. -/// -/// moq-quic asks its generator for every id it issues, dials and rotations -/// alike, so this is the one place the reuseport group's byte has to be -/// stamped. -#[derive(Debug)] -struct Cids { - shard: Option, -} - -impl moq_quic::ConnectionIdGenerator for Cids { - fn generate_cid(&mut self) -> moq_quic::ConnectionId { - moq_quic::ConnectionId::new(&super::endpoint::cid(self.shard)) - } - - fn cid_len(&self) -> usize { - CID_LEN - } - - fn cid_lifetime(&self) -> Option { - None - } -} - /// Dial as `config` says. pub(crate) fn client_config(config: &client::Config) -> Result { let provider = provider(); diff --git a/rs/moq-uring/src/quic/endpoint.rs b/rs/moq-uring/src/quic/endpoint.rs index a312ab9584..12d3b95c08 100644 --- a/rs/moq-uring/src/quic/endpoint.rs +++ b/rs/moq-uring/src/quic/endpoint.rs @@ -7,18 +7,18 @@ //! cluster dials at once. A single demux task receives; each connection keeps //! a driver task of its own for timers and egress. //! -//! Every connection id this endpoint issues is [`CID_LEN`] bytes, which is -//! what lets a short header (which does not encode the id's length) be -//! parsed at all. Ids rotate as peers consume them (`NEW_CONNECTION_ID`), so -//! a migrating client stays routable; an Initial for an unsupported version -//! gets a version negotiation packet back. +//! Every connection id this endpoint issues is moq-quic's 8-byte hashed id, +//! fixed-length because a short header does not encode the id's length. Ids +//! rotate as peers consume them (`NEW_CONNECTION_ID`), so a migrating client +//! stays routable; an Initial for an unsupported version gets a version +//! negotiation packet back. //! //! The socket is the endpoint's identity. Its worker is where the demux and //! every connection driver run, whichever thread built the endpoint. A socket //! adopted as a member of a steered `SO_REUSEPORT` group (one worker per core //! on one port; see [`udp::Bound`](crate::udp::Bound)) brings its slot along, -//! and every id the endpoint issues then carries the -//! [`moq_sock::shard::cid_prefix`] steering byte, so the kernel keeps +//! and every id the endpoint issues then names that shard +//! (`moq_quic::HashedConnectionIdGenerator::set_shard`), so the kernel keeps //! delivering a connection's packets to the worker that owns it. Dials //! through the endpoint carry it too, which is what steers a cluster peer's //! responses back to the dialing worker. @@ -27,11 +27,6 @@ use super::server; pub use super::driver::Endpoint; -/// Every connection id this endpoint issues is this long: long enough to be -/// unguessable per socket, and fixed because a short header does not encode -/// the id's length, so parsing one assumes it. -pub const CID_LEN: usize = 8; - /// What an endpoint serves, beyond dialing. #[derive(Clone, Debug)] #[non_exhaustive] @@ -72,13 +67,3 @@ impl Config { self } } - -/// A fresh [`CID_LEN`]-byte connection id, leading with the steering prefix -/// when the endpoint's socket is a member of a reuseport group. -pub(crate) fn cid(shard: Option) -> [u8; CID_LEN] { - let mut cid: [u8; CID_LEN] = rand::random(); - if let Some(shard) = shard { - cid[0] = moq_sock::shard::cid_prefix(shard); - } - cid -} diff --git a/rs/moq-uring/src/udp.rs b/rs/moq-uring/src/udp.rs index e05da8ea5a..52b9bb9c83 100644 --- a/rs/moq-uring/src/udp.rs +++ b/rs/moq-uring/src/udp.rs @@ -383,10 +383,10 @@ impl TxSlot { pub enum Bound { /// A socket on its own, bound by the caller. Lone(UdpSocket), - /// One member of a completed steered `SO_REUSEPORT` group. Every - /// connection id an endpoint on it issues then leads with the member's - /// [`cid_prefix`](moq_sock::shard::cid_prefix), so the group's filter - /// keeps delivering a connection's packets to this socket. + /// One member of a steered `SO_REUSEPORT` group. Every connection id an + /// endpoint on it issues then names the member's + /// [`Shard`](moq_sock::shard::Shard), so the group's filter keeps + /// delivering a connection's packets to this socket. Member(moq_sock::shard::Socket), } diff --git a/rs/moq-uring/tests/workers.rs b/rs/moq-uring/tests/workers.rs index e427de24b4..8d3ee8f4e5 100644 --- a/rs/moq-uring/tests/workers.rs +++ b/rs/moq-uring/tests/workers.rs @@ -19,7 +19,6 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::task::Poll; -use moq_sock::shard::Group; use moq_uring::{Config, Error, Worker, quic, udp}; fn worker() -> Option { @@ -79,12 +78,7 @@ fn a_steered_group_serves_a_shared_port() { // kernel steers by, and binding before any thread spawns is what // guarantees it. The group is held for as long as the sockets are served, // since it is what holds the port. - let mut group = Group::acquire("127.0.0.1:0".parse().expect("addr"), WORKERS).expect("group"); - let mut claims = Vec::new(); - while let Some(member) = group.member() { - claims.push(member.bind().expect("bind group member")); - } - let mut group = group.complete(claims).expect("complete group"); + let mut group = moq_sock::shard::bind("127.0.0.1:0".parse().expect("addr"), WORKERS).expect("group"); let mut members = Vec::new(); while let Some(member) = group.member().expect("clone retained socket") { members.push(member); From e111abf07ff42f489961501f0642cfd510642d5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:50:23 -0700 Subject: [PATCH 10/17] test(moq-tokio): gate the timed helper with its only caller Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/client.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 66166b129b..70c9e0a905 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -787,6 +787,7 @@ mod tests { use super::*; /// A millisecond track: the frames written here carry timestamps. + #[cfg(all(feature = "websocket", feature = "quic"))] fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } From 188b08c30dd4d3bae1ab33dfb8f71ba8a60c572b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 22:46:44 -0700 Subject: [PATCH 11/17] fix(tokio): mark quic::web and quic::driver error enums non_exhaustive Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/connection.rs | 1 + rs/moq-tokio/src/quic/driver/recv_stream.rs | 4 ++++ rs/moq-tokio/src/quic/driver/send_stream.rs | 2 ++ rs/moq-tokio/src/quic/web/connect.rs | 1 + rs/moq-tokio/src/quic/web/error.rs | 8 ++++++++ rs/moq-tokio/src/quic/web/settings.rs | 1 + 6 files changed, 17 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 605163a673..e86cecef41 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -1422,6 +1422,7 @@ fn wake_all_notify(wakers: &mut FxHashMap>) { /// Errors that can arise when sending a datagram #[derive(Debug, Error, Clone, Eq, PartialEq)] +#[non_exhaustive] pub enum SendDatagramError { /// The peer does not support receiving datagram frames #[error("datagrams not supported by peer")] diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs index 027349ae12..2bf91f6231 100644 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ b/rs/moq-tokio/src/quic/driver/recv_stream.rs @@ -506,6 +506,7 @@ impl ReadToEndBuffer { /// Errors from [`RecvStream::read_to_end`] #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadToEndError { /// An error occurred during reading #[error("read error: {0}")] @@ -558,6 +559,7 @@ impl Drop for RecvStream { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadError { /// The peer abandoned transmitting data on this stream /// @@ -618,6 +620,7 @@ impl From for io::Error { /// Errors that arise while waiting for a stream to be reset #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ResetError { /// The connection was lost #[error("connection lost")] @@ -691,6 +694,7 @@ impl Future for ReadExact<'_> { /// Errors that arise from reading from a stream. #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ReadExactError { /// The stream finished before all bytes were read #[error("stream finished early ({0} bytes read)")] diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs index e51af7cf25..812f2d4c12 100644 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ b/rs/moq-tokio/src/quic/driver/send_stream.rs @@ -358,6 +358,7 @@ impl Drop for SendStream { /// Errors that arise from writing to a stream #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum WriteError { /// The peer is no longer accepting data on this stream /// @@ -409,6 +410,7 @@ impl From for io::Error { /// Errors that arise while monitoring for a send stream stop from the peer #[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum StoppedError { /// The connection was lost #[error("connection lost")] diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 471f5e416c..d49228d5f4 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -5,6 +5,7 @@ use web_transport_proto::{ConnectRequest, ConnectResponse, VarInt}; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ConnectError { #[error("quic stream was closed early")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 5697bf684e..57d9eb1916 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -6,6 +6,7 @@ use crate::quic::web::{ConnectError, SettingsError}; /// An error returned when connecting to a WebTransport endpoint. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ClientError { #[error("unexpected end of stream")] UnexpectedEnd, @@ -39,6 +40,7 @@ pub enum ClientError { /// An errors returned by [`crate::quic::web::Session`], split based on if they are underlying QUIC errors or /// WebTransport errors. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum SessionError { #[error("connection error: {0}")] ConnectionError(crate::quic::driver::ConnectionError), @@ -140,6 +142,7 @@ impl CloseReason { /// An error that can occur when reading/writing the WebTransport stream header. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WebTransportError { #[error("closed: code={0} reason={1}")] Closed(u32, String), @@ -156,6 +159,7 @@ pub enum WebTransportError { /// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum WriteError { #[error("STOP_SENDING: {0}")] Stopped(u32), @@ -188,6 +192,7 @@ impl From for WriteError { /// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadError { #[error("session error: {0}")] SessionError(#[from] SessionError), @@ -223,6 +228,7 @@ impl From for ReadError { /// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadExactError { #[error("finished early")] FinishedEarly(usize), @@ -242,6 +248,7 @@ impl From for ReadExactError { /// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. #[derive(Clone, Error, Debug)] +#[non_exhaustive] pub enum ReadToEndError { #[error("too long")] TooLong, @@ -272,6 +279,7 @@ impl From for ClosedStream { /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum ServerError { #[error("unexpected end of stream")] UnexpectedEnd, diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index fdee95f23c..034c379ce7 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -3,6 +3,7 @@ use futures::try_join; use thiserror::Error; #[derive(Error, Debug, Clone)] +#[non_exhaustive] pub enum SettingsError { #[error("quic stream was closed early")] UnexpectedEnd, From 5d21b4fe829af8857f114dcebb36767c7d288579 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 23:03:33 -0700 Subject: [PATCH 12/17] fix(tokio): count in-flight accepts in wait_all_draining Endpoint::close now refuses attempts still queued for accept, and wait_all_draining waits for accepts partway through Incoming::accept, so a server shutdown cannot release the socket before those peers hear back. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 60 +++++++++++++++++--- rs/moq-tokio/src/quic/driver/tests.rs | 70 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 8 deletions(-) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2bae9031dc..9093082720 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -292,7 +292,7 @@ impl Endpoint { /// [`Connection::close()`]: crate::quic::driver::Connection::close pub fn close(&self, error_code: VarInt, reason: &[u8]) { let reason = Bytes::copy_from_slice(reason); - let mut endpoint = self.inner.state.lock().unwrap(); + let endpoint = &mut *self.inner.state.lock().unwrap(); endpoint.recv_state.connections.close = Some((error_code, reason.clone())); for sender in endpoint.recv_state.connections.senders.values() { // Ignoring errors from dropped connections @@ -301,7 +301,19 @@ impl Endpoint { reason: reason.clone(), }); } + // Refuse queued attempts now, the same as ones arriving after the close, so the peer + // hears back before a caller of `wait_all_draining` releases the socket. + let mut response_buffer = Vec::new(); + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.stats.refused_handshakes += 1; + response_buffer.clear(); + let transmit = endpoint.inner.refuse(incoming, &mut response_buffer); + respond(transmit, &response_buffer, &mut endpoint.sender); + } self.inner.shared.incoming.notify_waiters(); + if endpoint.all_draining() { + self.inner.shared.all_draining.notify_waiters(); + } } /// Wait for all connections on the endpoint to be cleanly shut down @@ -331,20 +343,22 @@ impl Endpoint { /// Wait until no connection on the endpoint is still active /// /// A connection stops being active once its peer's close arrives, or once its closing - /// period ends if the peer never answers. Unlike [`wait_idle()`], this does not sit out - /// the closing period for a peer that already has the close, nor wait for the + /// period ends if the peer never answers. An attempt queued for [`accept()`] or partway + /// through [`Incoming::accept`] still counts as active. Unlike [`wait_idle()`], this does + /// not sit out the closing period for a peer that already has the close, nor wait for the /// [`Connection`](crate::quic::driver::Connection) handles to drop. /// /// Does not proactively close existing connections or cause incoming connections to be /// rejected. Consider calling [`close()`] if that is desired. /// + /// [`accept()`]: Endpoint::accept /// [`close()`]: Endpoint::close /// [`wait_idle()`]: Endpoint::wait_idle pub async fn wait_all_draining(&self) { loop { { let endpoint = &mut *self.inner.state.lock().unwrap(); - if endpoint.recv_state.connections.all_draining() { + if endpoint.all_draining() { break; } // Construct future while lock is held to avoid race @@ -353,6 +367,22 @@ impl Endpoint { .await; } } + + /// Wait until a connection attempt is queued for [`accept()`](Endpoint::accept) + #[cfg(test)] + pub(crate) async fn wait_queued(&self) { + loop { + { + let endpoint = self.inner.state.lock().unwrap(); + if !endpoint.recv_state.incoming.is_empty() { + break; + } + // Construct future while lock is held to avoid race + self.inner.shared.incoming.notified() + } + .await; + } + } } /// Statistics on [Endpoint] activity @@ -419,13 +449,18 @@ impl Future for EndpointDriver { impl Drop for EndpointDriver { fn drop(&mut self) { - let mut endpoint = self.0.state.lock().unwrap(); + let endpoint = &mut *self.0.state.lock().unwrap(); endpoint.driver_lost = true; self.0.shared.incoming.notify_waiters(); // Drop all outgoing channels, signaling the termination of the endpoint to the associated // connections. endpoint.recv_state.connections.senders.clear(); endpoint.recv_state.connections.draining.clear(); + // `accept` yields nothing once the driver is lost, so queued attempts would hold off + // `wait_all_draining` forever. + for incoming in endpoint.recv_state.incoming.drain(..) { + endpoint.inner.ignore(incoming); + } self.0.shared.all_draining.notify_waiters(); // Clearing the senders is what makes the endpoint idle, but a waiter already parked on // `idle` only re-reads that after being notified, and with the driver gone nothing else @@ -500,6 +535,10 @@ impl EndpointInner { if state.is_idle() { self.shared.idle.notify_waiters(); } + // The pending accept held off `wait_all_draining`, and a failed one sends no event. + if state.all_draining() { + self.shared.all_draining.notify_waiters(); + } result } @@ -609,9 +648,7 @@ impl State { shared.idle.notify_waiters(); } } - if (event.is_draining() || event.is_drained()) - && self.recv_state.connections.all_draining() - { + if (event.is_draining() || event.is_drained()) && self.all_draining() { shared.all_draining.notify_waiters(); } let Some(event) = self.inner.handle_event(ch, event) else { @@ -633,6 +670,13 @@ impl State { fn is_idle(&self) -> bool { self.recv_state.connections.is_empty() && self.pending_accepts == 0 } + + /// Whether no connection is still active, nor any attempt left that could become one + fn all_draining(&self) -> bool { + self.recv_state.connections.all_draining() + && self.pending_accepts == 0 + && self.recv_state.incoming.is_empty() + } } impl Drop for State { diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 0b284f1770..5d50d7a004 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1285,6 +1285,76 @@ async fn split_accept_close_during_pending_accept() { assert_eq!(server.open_connections(), 0); } +/// An accept still in progress when the endpoint closes will send a close of its own, so +/// `wait_all_draining` must not report the endpoint drained until that accept finishes. +#[tokio::test] +async fn split_accept_close_holds_off_all_draining() { + let _guard = subscribe(); + let BlockedHandshake { + client, + server, + blocker, + release_guard: _release_guard, + accept, + connect, + } = blocked_handshake(b"test", b"test"); + + wait_for_blocked_handshake(blocker.clone()).await; + server.close(0u32.into(), b"closing"); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, _) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + + blocker.release(); + // The waiter parked above resolves only if the accept's close is accounted for and announced. + timeout(Duration::from_secs(5), draining).await.unwrap(); + assert!(await_connection(connect).await.is_err()); + + let _ = await_connection(accept).await; + wait_idle(&server).await; + wait_idle(&client).await; +} + +/// `close` refuses attempts still queued for `accept`, so the peer hears back before +/// `wait_all_draining` lets the caller release the socket. +#[tokio::test] +async fn close_refuses_queued_incoming() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + server.close(0u32.into(), b"closing"); + timeout(Duration::from_secs(5), server.wait_all_draining()) + .await + .unwrap(); + assert!(server.accept().await.is_none()); + + // Without the refusal, the attempt sits in the queue and the client waits out its timeout. + let error = timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .unwrap_err(); + assert!(matches!( + error, + crate::quic::driver::ConnectionError::ConnectionClosed(close) + if close.error_code == moq_quic::TransportErrorCode::CONNECTION_REFUSED + )); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From 21cac2b27435c4b2084018c98b0a3cb5be467cc6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 00:06:45 -0700 Subject: [PATCH 13/17] fix(tokio): wake draining waiters when accept empties the queue Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic/driver/endpoint.rs | 5 ++++ rs/moq-tokio/src/quic/driver/tests.rs | 38 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 9093082720..2a214d5466 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -821,6 +821,11 @@ impl Future for Accept<'_> { return Poll::Ready(None); } if let Some(incoming) = endpoint.recv_state.incoming.pop_front() { + // A handed-out attempt no longer holds off `wait_all_draining`, and refusing, + // ignoring, or dropping it sends no event that would wake the waiter later. + if endpoint.all_draining() { + this.endpoint.inner.shared.all_draining.notify_waiters(); + } // Release the mutex lock on endpoint so cloning it doesn't deadlock drop(endpoint); let incoming = Incoming::new(incoming, this.endpoint.inner.clone()); diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index 5d50d7a004..aae7054d83 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -1355,6 +1355,44 @@ async fn close_refuses_queued_incoming() { )); } +/// Taking the last queued attempt and refusing it sends no event, so `accept` itself must wake a +/// parked `wait_all_draining`. +#[tokio::test] +async fn refused_incoming_wakes_all_draining_waiter() { + let _guard = subscribe(); + let factory = EndpointFactory::new(); + let server = factory.endpoint(); + let client = factory.endpoint(); + + let connect = client + .connect(server.local_addr().unwrap(), "localhost") + .unwrap(); + timeout(Duration::from_secs(5), server.wait_queued()) + .await + .unwrap(); + + let mut draining = pin!(server.wait_all_draining()); + let (waker, wake_counter) = new_count_waker(); + assert!( + draining + .as_mut() + .poll(&mut Context::from_waker(&waker)) + .is_pending() + ); + assert_eq!(wake_counter.wakes(), 0); + + server.accept().await.unwrap().refuse(); + assert!(wake_counter.wakes() > 0); + timeout(Duration::from_secs(5), draining).await.unwrap(); + + assert!( + timeout(Duration::from_secs(5), connect) + .await + .unwrap() + .is_err() + ); +} + /// The endpoint driver may exit while an accept runs without the lock. The connection registered /// afterwards must still be told that the driver is gone, so that its `Connecting` fails promptly /// and `wait_idle` does not wait on a connection nobody drives. From c1f4cfd3ef7ad1c6ef7136e1d323ec272708148f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 07:47:22 -0700 Subject: [PATCH 14/17] fix(tokio): surface non-WouldBlock receive errors, cap moq_quic logs The server socket retried every failed recv, so a persistent error spun the endpoint task while tokio kept readiness asserted. Only WouldBlock retries now, matching the driver's tokio runtime. Also cap moq_quic at info like noq was, fix the driver doc example, and point the moq-uring README at the just recipe. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/log.rs | 1 + rs/moq-tokio/src/quic/backend.rs | 8 ++++++-- rs/moq-tokio/src/quic/driver/endpoint.rs | 4 ++-- rs/moq-uring/README.md | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/rs/moq-tokio/src/log.rs b/rs/moq-tokio/src/log.rs index 5d8a5e4f19..6b4833f9d1 100644 --- a/rs/moq-tokio/src/log.rs +++ b/rs/moq-tokio/src/log.rs @@ -54,6 +54,7 @@ impl Log { .with_default_directive(self.level().into()) // Default to our -q/-v args .from_env_lossy() // Allow overriding with RUST_LOG .add_directive("h2=warn".parse()?) + .add_directive("moq_quic=info".parse()?) .add_directive("moq_tokio::quic::driver=info".parse()?) .add_directive("tungstenite=info".parse()?) .add_directive("rustls=info".parse()?) diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 666040d324..e743af6ea7 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -627,8 +627,12 @@ impl driver::AsyncUdpSocket for Socket { // Only the endpoint driver receives, so the one readiness waker is enough. loop { ready!(io.poll_recv_ready(cx))?; - if let Ok(res) = io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { - return Poll::Ready(Ok(res)); + match io.try_io(tokio::io::Interest::READABLE, || self.state.recv(io.into(), bufs, meta)) { + Ok(res) => return Poll::Ready(Ok(res)), + // `try_io` clears readiness only for `WouldBlock`; retrying any other error + // would spin, since `poll_recv_ready` completes at once and the recv fails again. + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {} + Err(err) => return Poll::Ready(Err(err)), } } } diff --git a/rs/moq-tokio/src/quic/driver/endpoint.rs b/rs/moq-tokio/src/quic/driver/endpoint.rs index 2a214d5466..eeaaea5e3d 100644 --- a/rs/moq-tokio/src/quic/driver/endpoint.rs +++ b/rs/moq-tokio/src/quic/driver/endpoint.rs @@ -59,8 +59,8 @@ impl Endpoint { /// the address `[::]:0` is a reasonable default to maximize the ability to connect to other /// address. For example: /// - /// ``` - /// quinn::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); + /// ```no_run + /// moq_tokio::quic::driver::Endpoint::client((std::net::Ipv6Addr::UNSPECIFIED, 0).into()); /// ``` /// /// Some environments may not allow creation of dual-stack sockets, in which case an IPv6 diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index db5a5a8aec..ec84787ea0 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -82,7 +82,7 @@ timers, and UDP socket can be built without QUIC. Building without default features leaves the `quic` module out entirely. ```bash -cargo test -p moq-uring --features quic +just rs uring ``` `moq-relay` enables it with `--features io-uring`. From ccaae36a7a9093d9ce746e858b44b533deaa032b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:21:15 -0700 Subject: [PATCH 15/17] fix(tokio): send a stopped worker's closes before dropping its runtime A worker broke out of its loop and dropped its runtime, and the endpoint driver with it, so any close still queued (a drain's abort, a dropped listener) never reached the wire and the peer waited out its idle timeout. The shared server avoids this by awaiting Listener::close. Each worker now closes its endpoint and waits for every connection to send its close first. Sharding by default put every test relay on workers, which is how moq-cli's fails_when_the_session_ends started timing out in CI. Co-Authored-By: Claude Opus 5.5 --- rs/moq-tokio/src/quic.rs | 2 +- rs/moq-tokio/src/quic/backend.rs | 22 ++++++++++++--- rs/moq-tokio/src/server.rs | 6 +++++ rs/moq-tokio/src/worker/group.rs | 8 ++++++ rs/moq-tokio/tests/worker.rs | 46 ++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 4b28ac64ef..f1e9d9fb73 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -19,7 +19,7 @@ mod backend; #[cfg(feature = "quic")] pub use backend::Error; #[cfg(feature = "quic")] -pub(crate) use backend::{Client, Options, Server, accept}; +pub(crate) use backend::{Client, Closer, Options, Server, accept}; // The async QUIC layer driving `moq-quic` on tokio, forked from quinn (see its module docs). // Public only so the workspace's tests can dial a bare endpoint; not a stable API. diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 9b31e69e52..90bf427190 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -836,11 +836,27 @@ impl Server { /// Close every connection, wait until each has sent its close to the peer, /// then release the socket. pub async fn shutdown(self) { - self.close(); + self.closer().close().await; + self.socket.release(); + } + + /// A handle that closes this server's connections from wherever its owner + /// went, for a worker that has to flush them before its runtime is dropped. + pub fn closer(&self) -> Closer { + Closer(self.quic.clone()) + } +} + +/// Closes a [`Server`]'s connections without owning it; see [`Server::closer`]. +pub(crate) struct Closer(driver::Endpoint); + +impl Closer { + /// Close every connection and wait until each has sent its close to the peer. + pub async fn close(self) { + self.0.close(driver::VarInt::from_u32(0), b"server shutdown"); // Not `wait_idle`, which also sits out each connection's 3 PTO closing // period: that only repeats the close to a peer that already has it. - self.quic.wait_all_draining().await; - self.socket.release(); + self.0.wait_all_draining().await; } } diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index 0d63c01ae2..44bcb1893e 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -788,6 +788,12 @@ impl Server { Err(Error::NoBackend("no QUIC listener configured")) } + /// Closes the QUIC connections once this server has been handed away. + #[cfg(feature = "quic")] + pub(crate) fn quic_closer(&self) -> Option { + self.quic.as_ref().map(crate::quic::Server::closer) + } + /// The address the WebSocket listener from [`Config::websocket`] bound to, if one was set. #[cfg(feature = "websocket")] pub fn websocket_local_addr(&self) -> Option { diff --git a/rs/moq-tokio/src/worker/group.rs b/rs/moq-tokio/src/worker/group.rs index ff93a159aa..5e646aa3a0 100644 --- a/rs/moq-tokio/src/worker/group.rs +++ b/rs/moq-tokio/src/worker/group.rs @@ -753,6 +753,7 @@ fn run( return; } }; + let closer = server.quic_closer(); if ready .send(Ok(Ready { @@ -778,6 +779,13 @@ fn run( Some(spawn) = spawn.recv() => spawn(), } } + + // Dropping the runtime drops the endpoint driver with it, so every + // connection's close has to be sent first. Otherwise a peer only learns + // the session is gone at its idle timeout. + if let Some(closer) = closer { + closer.close().await; + } }); tracing::debug!(index, "QUIC worker stopped"); } diff --git a/rs/moq-tokio/tests/worker.rs b/rs/moq-tokio/tests/worker.rs index 22385c0f89..ba5df60c6f 100644 --- a/rs/moq-tokio/tests/worker.rs +++ b/rs/moq-tokio/tests/worker.rs @@ -139,6 +139,52 @@ async fn dropping_the_workers_releases_the_port() { assert!(owned.is_disjoint(&open_sockets()), "workers left the port bound"); } +/// Stopping a group closes its sessions on the wire. The endpoint driver dies +/// with the worker's runtime, so a close still queued at that point never +/// leaves, and the peer only notices at its idle timeout. +#[tokio::test] +async fn stopping_the_group_closes_its_sessions() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let dir = tempfile::tempdir().unwrap(); + let (cert, key) = certificate(dir.path()); + let workers = bind_workers(listen_config(&cert, &key, 0), Default::default(), config(WORKERS)).unwrap(); + let addr = workers.local_addr(); + let fingerprints = workers.certificates().fingerprints(); + let mut group = workers.split(); + for member in group.members() { + member.serve(|server| async move { + let mut listener = server.listen().await.unwrap(); + while let Some(request) = listener.accept().await { + tokio::spawn(async move { + let (_session, _driver) = request.ok().await.unwrap(); + std::future::pending::<()>().await; + }); + } + }); + } + + let mut connect = moq_tokio::connect::Config::default(); + connect.tls.fingerprint = fingerprints; + connect.once = Some(true); + #[cfg(feature = "websocket")] + { + connect.websocket.enabled = Some(false); + } + // Far past the wait below, so only a close on the wire can end the session. + let mut quic = moq_tokio::quic::Config::default(); + quic.idle_timeout = Some(std::time::Duration::from_secs(600)); + quic.keep_alive = None; + let client = connect.init(quic).unwrap(); + let url: url::Url = format!("https://{addr}/").parse().unwrap(); + let session = client.connect(url).established().await.expect("connect"); + + group.shutdown().await; + tokio::time::timeout(std::time::Duration::from_secs(5), session.closed()) + .await + .expect("the session must see the close") + .expect_err("a stopped server ends the session"); +} + /// The future factory runs on the worker, so the future may hold local state /// across an await without making that state thread-safe, and spawn more of it /// onto the same task set. From 218cbec70b3629ef7a84ed079e46061375accefa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 10:57:22 -0700 Subject: [PATCH 16/17] feat(relay): keep the shared runtime by default, with sharding opt-in The maintainer kept sharding opt-in until a saturating sweep justifies the flip, tracked in quest/m2/perf/shard-default.md. An unset runtime.workers is the shared runtime again, and moq_sock::shard::per_core goes with the default that used it. runtime.pin and moq_tokio::worker::Config::new now default to unpinned: every group pins worker i to core i, so two groups on a host, or one limited to fewer CPUs than the host has, crowd onto the first cores. Co-Authored-By: Claude Opus 5.5 --- bench/relay.sh | 9 +- doc/bin/relay/config.md | 17 ++-- doc/bin/relay/index.md | 1 - quest/m2/perf/README.md | 1 + .../m2/perf/kio-channel-contention/README.md | 4 +- quest/m2/perf/shard-default.md | 37 ++++++++ rs/moq-relay/Cargo.toml | 2 +- rs/moq-relay/src/config.rs | 4 +- rs/moq-relay/src/relay.rs | 85 +++---------------- rs/moq-relay/src/runtime.rs | 32 +++---- rs/moq-sock/README.md | 2 - rs/moq-sock/src/shard.rs | 26 +----- rs/moq-tokio/src/worker/mod.rs | 20 ++--- 13 files changed, 84 insertions(+), 156 deletions(-) create mode 100644 quest/m2/perf/shard-default.md diff --git a/bench/relay.sh b/bench/relay.sh index 8a12deb237..49aef125b2 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -23,16 +23,9 @@ relay_config() { fi case $runtime in - '') + '' | tokio-shared) runtime_config=() ;; - tokio-shared) - runtime_config=( - '' - '[runtime]' - 'workers = 0' - ) - ;; tokio-workers) runtime_config=( '' diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index d30bfb7fc9..d6835ceca7 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -109,25 +109,20 @@ window, so very small values can prevent other requests from progressing. ## \[runtime] -On Linux, QUIC runs on one single-threaded worker per core by default, each -with its own socket on the listen address (`SO_REUSEPORT`), so a connection is -handled start to finish by one thread and its packets never cross cores. The -count follows the CPUs the process may use, cgroup quotas included. Elsewhere, -and with `workers = 0`, one work-stealing runtime serves every connection off -one UDP socket. +By default one work-stealing runtime serves every connection off one UDP +socket. On Linux, QUIC can instead run on single-threaded workers, each with +its own socket on the listen address (`SO_REUSEPORT`), so a connection is +handled start to finish by one thread and its packets never cross cores. ```toml [runtime] -workers = 8 # Override the per-core count. 0 keeps the shared runtime. +workers = 8 # Single-threaded QUIC workers. Omit for the shared runtime. pin = false # Pin each worker to a core. Off by default. io_uring = false # Drive them with io_uring instead of tokio. ``` Packets are steered by connection ID, so a client that migrates stays with its -worker. An explicit count needs Linux and the `quic` feature, and is refused -anywhere else rather than ignored. A relay with a QUIC-LB server id -(`listen.lb_id`) or no QUIC listener keeps the shared runtime unless a count is -set, and a count is refused alongside `lb_id`. Every worker and the TLS stream +worker. `workers` needs the `quic` feature. Every worker and the TLS stream listener serve one shared identity, generated or file-backed, so a certificate rotation reaches them together and `/certificate.sha256` stays accurate. diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 59def5fb7f..acd69bfae7 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -12,7 +12,6 @@ relay serves video, audio, and data alike. ## Features - **QUIC, WebTransport, and WebSocket** listeners, so browsers and native clients connect to one process. -- **Thread-per-core QUIC** on Linux by default: one worker and socket per core, with packets steered by connection ID. See [`[runtime]`](/bin/relay/config#runtime). - **Path-scoped authentication** with JWTs, mTLS for peers, and anonymous patterns, decided by an auth server or a static grant. See [Authentication](/bin/relay/auth). - **Clustering** across hosts and regions with hop-list routing, per-link costs, LAN discovery, and dynamic peer lists. See [Clustering](/bin/relay/cluster). - **A group cache** with byte and age budgets, so late joiners and the HLS gateway can fetch recent history. diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 1f5b159029..a48decf398 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -79,6 +79,7 @@ optimization. - [Demand aggregate](/quest/m2/perf/demand-aggregate.md) - a track subscribe, leave, or preference update no longer walks every reader of the track - [Ingest batch](/quest/m2/perf/ingest-batch.md) - relay ingest pays one lock, wake, and clock read per burst of whole frames instead of per frame - [Remove moq-uring copies](/quest/m2/perf/uring-copies.md) - egress, stream send and receive, and datagram receive stop copying where the QUIC core already allows it, with before/after measurements on `moq-quic` +- [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux ## Related diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index bf01d930b2..c94e961734 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -134,8 +134,8 @@ Decided (maintainer, 2026-10-14, baseline follow-up): Decided (maintainer, 2026-10-14, per-thread follow-up): - Per-worker sharding is no longer ruled out; two spikes decide. Both run on - the thread-per-core workers (`--runtime-workers`, the default on Linux) - and link workers over an + the thread-per-core workers (`--runtime-workers`, the default once + [Shard by default](/quest/m2/perf/shard-default.md) lands) and link workers over an in-memory transport (`moq_net::transport::memory`), never real QUIC. The [replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) diff --git a/quest/m2/perf/shard-default.md b/quest/m2/perf/shard-default.md new file mode 100644 index 0000000000..1cfb53a352 --- /dev/null +++ b/quest/m2/perf/shard-default.md @@ -0,0 +1,37 @@ +# [M] Shard the relay by default + +## Goal + +A saturating benchmark shows whether thread-per-core QUIC beats the shared +runtime, and the relay's default follows the measurement. If sharding wins, +an unset `runtime.workers` runs one worker per core on Linux, and +`workers = 0` is the explicit way back to the shared runtime. Off Linux, the +shared runtime stays the default. + +## Plan + +Sharding is opt-in: `runtime.workers` is unset by default, and +`runtime.pin` defaults off. The maintainer decided on #5688 to keep it that +way until a measurement justifies the flip. + +- Extend `just bench-shard` to load that saturates the shared runtime, for + example 1024+ connections or more than one load host. At 16 to 256 + connections, #5688's sweep never used one full core, so it showed no + regression from sharding and no win either. The 16-shard groups spent about + 15 to 25% more CPU than the shared runtime at low load, from idle per-core + runtimes. +- Flip the default only if sharding raises the throughput ceiling or cuts + CPU per connection at saturation. A measured no-win deletes this quest. +- Shard only where a group serves the same listener the shared runtime would + open. A stream-only relay, or one with a QUIC-LB `lb_id`, stays shared + unless a count is set. A count sized from `available_parallelism`, capped at + `moq_sock::shard::MAX_SHARDS`, follows cgroup quotas. +- With the default flipped, `runtime.io_uring` no longer needs an explicit + count. + +Update `doc/bin/relay/` for the default change. + +## Related + +- [kio channel contention](/quest/m2/perf/kio-channel-contention/README.md) - its per-thread spikes run on the workers +- [UDP demux](/quest/m1/p2p/udp-demux.md) - demuxes each shard's socket diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 4dc26a41df..7791f78594 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -34,7 +34,7 @@ quic = ["_quic", "moq-tokio/quic"] # The tokio QUIC listener, and the thread-per-core worker group behind # `runtime.workers`. Not one to enable on its own: it names no backend, so # moq-tokio would have nothing to bind a socket with. -_quic = ["dep:moq-sock"] +_quic = [] # qlog trace capture (`--quic-qlog `). Off by default: it compiles the # qlog machinery into the enabled backends and is only wanted for debugging or # congestion-control testing. Covers the io_uring workers too, so one feature diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 32484b7e60..e48a066995 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -59,8 +59,8 @@ pub struct Config { #[serde(default)] pub log: moq_tokio::Log, - /// How QUIC work is laid out over threads: one worker per core on Linux - /// unless `runtime.workers` says otherwise. + /// How QUIC work is laid out over threads. One shared runtime unless + /// `runtime.workers` is set. #[usage(flatten)] #[serde(default)] pub runtime: crate::runtime::Config, diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index c433d42ed4..71f5d571e6 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -95,15 +95,14 @@ pub struct Relay { /// Live sessions on this node, listed and nudged from the internal listener. sessions: crate::session::Registry, /// The thread-per-core QUIC workers, already bound and waiting to be split - /// and run. `None` when QUIC stays on the shared runtime (`runtime.workers` - /// is `0`, or unset where a listener cannot be sharded); otherwise `server` - /// carries no QUIC listener of its own. + /// and run. `None` unless `runtime.workers` is configured, in which case + /// `server` carries no QUIC listener of its own. #[cfg(feature = "_quic")] workers: Option, /// The io_uring QUIC workers, already bound and waiting for - /// [`serve`](crate::uring::Workers::serve). `None` unless QUIC runs on - /// workers and `runtime.io_uring` is set, in which case they own the QUIC - /// listen address instead of `workers`. + /// [`serve`](crate::uring::Workers::serve). `None` unless both + /// `runtime.workers` and `runtime.io_uring` are configured, in which case + /// they own the QUIC listen address instead of `workers`. #[cfg(all(target_os = "linux", feature = "_uring"))] uring: Option, } @@ -160,27 +159,9 @@ impl Relay { // Bind the QUIC workers first: they own the listen address when configured, // so the server below must not also try to bind it. let io_uring = config.runtime.io_uring(); - - // Sharding is the default only where a group serves exactly the listener - // the shared runtime would have opened: a backend that can drive workers, - // a QUIC listen address (not a stream-only relay), and no QUIC-LB server - // id, whose connection IDs leave the steering filter no byte to read. An - // explicit count skips these and is refused below where it cannot deliver. - #[cfg(any(feature = "_quic", feature = "_uring"))] - let per_core = { - let backend = cfg!(feature = "_quic") || io_uring; - let shardable = backend - && (config.listen.bind.is_some() || !config.listen.has_explicit_bind()) - && config.listen.load_balancer().is_none(); - shardable.then(moq_sock::shard::per_core).flatten() - }; - #[cfg(not(any(feature = "_quic", feature = "_uring")))] - let per_core = None; - let worker = config.runtime.workers(per_core); - anyhow::ensure!( - !io_uring || worker.is_some(), - "runtime.io_uring needs QUIC workers, which runtime.workers = 0 turns off" + !io_uring || config.runtime.workers().is_some(), + "runtime.io_uring requires runtime.workers" ); #[cfg(not(target_os = "linux"))] anyhow::ensure!(!io_uring, "runtime.io_uring is Linux-only"); @@ -194,20 +175,20 @@ impl Relay { // silently serving from the shared runtime instead would look like it worked. #[cfg(not(feature = "_quic"))] anyhow::ensure!( - io_uring || worker.is_none(), + io_uring || config.runtime.workers().is_none(), "runtime.workers needs moq-relay built with the `quic` feature" ); // Workers and the stream server split one relay across listeners, so they share // one loaded identity. A lone server loads its own, only if a listener needs it. - let served = worker.map(|_| config.listen.tls.load()).transpose()?; + let served = config.runtime.workers().map(|_| config.listen.tls.load()).transpose()?; let mut listen = config.listen.clone(); if served.is_some() { listen.tls = Default::default(); } #[cfg(feature = "_quic")] - let workers = match worker { + let workers = match config.runtime.workers() { Some(worker) if !io_uring => { let mut server = moq_tokio::server::Config::default(); server.listen = listen.clone(); @@ -231,7 +212,7 @@ impl Relay { let workers_certificates: Option = None; #[cfg(all(target_os = "linux", feature = "_uring"))] - let uring = match worker { + let uring = match config.runtime.workers() { Some(worker) if io_uring => { let mut server = moq_tokio::server::Config::default(); server.listen = listen.clone(); @@ -854,50 +835,6 @@ async fn serve_listening( #[cfg(test)] mod tests { - /// A QUIC relay with no `[runtime]` section shards across every core it may - /// use, where the kernel can steer a group. - #[cfg(all(target_os = "linux", feature = "_quic"))] - #[tokio::test] - async fn quic_shards_per_core_by_default() { - let mut config = crate::Config::default(); - config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); - config.listen.tls.generate = vec!["localhost".into()]; - let relay = super::Relay::load(config).await.unwrap(); - let workers = relay.workers.as_ref().expect("an unset count shards"); - assert_eq!(Some(workers.len()), moq_sock::shard::per_core().map(usize::from)); - assert_eq!(relay.addr(), Some(workers.local_addr())); - } - - /// Each of these keeps QUIC on the shared runtime when no count is set: the - /// explicit opt-out, a stream-only relay (sharding would open a QUIC port - /// nobody asked for), and QUIC-LB, whose connection IDs leave the steering - /// filter nothing to read. - #[cfg(all(target_os = "linux", feature = "_quic"))] - #[tokio::test] - async fn quic_stays_shared_where_sharding_does_not_fit() { - let optout = { - let mut config = crate::Config::default(); - config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); - config.listen.tls.generate = vec!["localhost".into()]; - config.runtime.workers = Some(0); - config - }; - let streams = { - let mut config = crate::Config::default(); - config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); - config - }; - let quic_lb: crate::Config = toml::from_str( - "[listen]\nbind = \"127.0.0.1:0\"\ntls.generate = [\"localhost\"]\nload_balancer = { id = \"0a0b\", nonce = 4 }\n", - ) - .unwrap(); - - for (name, config) in [("workers = 0", optout), ("stream-only", streams), ("QUIC-LB", quic_lb)] { - let relay = super::Relay::load(config).await.unwrap(); - assert!(relay.workers.is_none(), "{name} sharded"); - } - } - #[tokio::test] async fn stun_listener_runs_with_the_relay_and_stops_with_it() { use stun_types::message::{BINDING, Message, MessageWriteVec}; diff --git a/rs/moq-relay/src/runtime.rs b/rs/moq-relay/src/runtime.rs index 25fe0b460e..625e7d89ae 100644 --- a/rs/moq-relay/src/runtime.rs +++ b/rs/moq-relay/src/runtime.rs @@ -1,9 +1,8 @@ //! Flags for serving QUIC from per-core workers. //! //! The workers themselves are [`moq_tokio::worker`]; this is the operator-facing -//! half, which is the only part specific to the relay. Unset (the default) shards -//! QUIC across one worker per core where the kernel can steer a group (Linux) and -//! keeps it on the shared runtime elsewhere. +//! half, which is the only part specific to the relay. Unset (the default) keeps +//! QUIC on the shared runtime with everything else. use serde::{Deserialize, Serialize}; @@ -13,14 +12,13 @@ use serde::{Deserialize, Serialize}; #[serde(default, deny_unknown_fields)] #[non_exhaustive] pub struct Config { - /// Serve QUIC from this many single-threaded workers, each with its own - /// socket on the listen address, instead of the shared runtime. + /// Serve QUIC from this many single-threaded workers instead of the shared + /// runtime, each with its own socket on the listen address. /// /// A connection is handled start to finish by one worker, which trades the /// shared runtime's load balancing for no cross-thread traffic per packet. - /// Unset (the default) runs one worker per core on Linux and the shared - /// runtime elsewhere; `0` keeps the shared runtime. Any other count is - /// Linux-only. Every worker shares the relay's served identity and reloads. + /// Linux-only; every worker shares the relay's served identity and reloads. + /// Unset (the default) keeps QUIC on the shared runtime. #[usage(long = "runtime-workers", env = "MOQ_RUNTIME_WORKERS", setting = "runtime.workers")] pub workers: Option, @@ -42,9 +40,8 @@ pub struct Config { /// with `UDP_GRO`, `UDP_SEGMENT` send), userspace timers, and a local task /// set; sessions are moq-lite only (native raw QUIC and browser /// WebTransport alike), and everything else (HTTP, WebSocket, cluster - /// dials, stats) stays on the shared runtime. Requires QUIC workers (any - /// `runtime-workers` but `0`), Linux 6.12+, and refuses to start anywhere it - /// cannot deliver. + /// dials, stats) stays on the shared runtime. Requires `runtime-workers`, + /// Linux 6.12+, and refuses to start anywhere it cannot deliver. #[usage( long = "runtime-io-uring", env = "MOQ_RUNTIME_IO_URING", @@ -58,15 +55,10 @@ impl Config { /// The worker group this asks for, or `None` to keep QUIC on the shared /// runtime. /// - /// `per_core` is the count an unset `workers` falls back to: one per core - /// where the kernel can steer a group, or `None` where the listener cannot be - /// sharded at all. `0` is the explicit way back to the shared runtime. - pub(crate) fn workers(&self, per_core: Option) -> Option { - let count = match self.workers { - Some(0) => return None, - Some(count) => count, - None => per_core?, - }; + /// Zero workers reads as unset rather than as an error: it is the natural way + /// to switch the mode off from a config file that already has the section. + pub fn workers(&self) -> Option { + let count = self.workers.filter(|count| *count > 0)?; Some(moq_tokio::worker::Config::new(count).with_pin(self.pin)) } diff --git a/rs/moq-sock/README.md b/rs/moq-sock/README.md index f23b1f02bd..0c1d8614d9 100644 --- a/rs/moq-sock/README.md +++ b/rs/moq-sock/README.md @@ -15,8 +15,6 @@ runtimes (`moq-tokio` and `moq-uring`). in every connection id it issues, alongside a marker no SRT data header carries (`moq_quic::HashedConnectionIdGenerator::set_shard`). Anything else on the port, including other protocols, keeps the kernel's 4-tuple hash. - `shard::per_core` is the default group size: one per core on Linux, and no - group elsewhere. - **`cpu`**: pinning worker threads to cores. - **`udp`**: sending and receiving with GSO, GRO, and ECN, a hard fork of [quinn-udp](https://github.com/quinn-rs/quinn). All credit for it goes to diff --git a/rs/moq-sock/src/shard.rs b/rs/moq-sock/src/shard.rs index 0a0a4dcb56..2b00f7c038 100644 --- a/rs/moq-sock/src/shard.rs +++ b/rs/moq-sock/src/shard.rs @@ -15,8 +15,7 @@ //! [`bind`] is the whole entry point. It takes the port, binds every member in //! index order, attaches the steering filter, and returns a [`Bound`] group //! that hands out one serving [`Socket`] per member while retaining every -//! underlying socket for its own lifetime. [`per_core`] is the size a server -//! should ask for by default. +//! underlying socket for its own lifetime. //! //! ```no_run //! # fn example(addr: std::net::SocketAddr) -> Result<(), Box> { @@ -137,20 +136,6 @@ pub enum Error { Steer(#[source] io::Error), } -/// One shard per core this process may run on, or `None` where the kernel -/// cannot steer a group, which is everywhere but Linux. -/// -/// Counts the cores the process is allowed (its affinity mask and cgroup -/// quota), not the machine's, so a container limited to two CPUs runs two -/// shards. Capped at [`MAX_SHARDS`]. -pub fn per_core() -> Option { - if !cfg!(target_os = "linux") { - return None; - } - let cores = std::thread::available_parallelism().map_or(1, |cores| cores.get()); - Some(cores.min(usize::from(MAX_SHARDS)) as u16) -} - /// Bind a steered group of `count` sockets sharing `addr`. /// /// Everything a steered group has to get right happens here rather than in its @@ -639,15 +624,6 @@ mod tests { )); } - /// A group can only be steered on Linux, so a default elsewhere is a single - /// socket rather than a group. - #[test] - fn per_core_is_linux_only() { - let count = per_core(); - assert_eq!(count.is_some(), cfg!(target_os = "linux")); - assert!(count.is_none_or(|count| (1..=MAX_SHARDS).contains(&count))); - } - /// Elsewhere the kernel cannot steer, so a group is refused at its first /// member rather than served by a socket that silently balances nothing. #[test] diff --git a/rs/moq-tokio/src/worker/mod.rs b/rs/moq-tokio/src/worker/mod.rs index 1cb9991806..b85962f8ae 100644 --- a/rs/moq-tokio/src/worker/mod.rs +++ b/rs/moq-tokio/src/worker/mod.rs @@ -2,9 +2,9 @@ //! //! A `Server` on a work-stealing runtime serves every connection off one UDP //! socket: every packet can cross threads, and every wakeup is a candidate -//! context switch. `Workers` is the opposite shape. Each -//! member is a thread of its own, pinned to a core, running a `current_thread` -//! runtime and owning one socket in a `SO_REUSEPORT` group. A connection lands +//! context switch. `Workers` is the opposite shape. Each member is a thread of +//! its own, optionally pinned to a core, running a `current_thread` runtime and +//! owning one socket in a `SO_REUSEPORT` group. A connection lands //! on one worker and stays there, so the locks its driver and its session take //! are uncontended and nothing is stolen. //! @@ -33,22 +33,22 @@ pub struct Config { /// How many workers to run, each with a thread and a socket of its own. pub count: u16, - /// Pin each worker to a CPU core. + /// Pin each worker to a CPU core, defaulting to off. /// /// The mode's measured win comes from each worker owning a socket and a /// runtime, not from pinning: on a single-socket machine, pinned and unpinned /// benchmark inside run-to-run noise of each other. Pinning stops the - /// scheduler migrating a busy worker, which should matter on a multi-socket - /// or NUMA machine, and costs nothing elsewhere, so it defaults on. Turn it - /// off when sharing the machine with something that manages CPU placement - /// itself. + /// scheduler migrating a busy worker, which may matter on a multi-socket or + /// NUMA machine with the cores to itself. Every group pins worker *i* to the + /// *i*th core, so two groups on a host, or one limited to fewer CPUs than the + /// host has, would crowd onto the first cores. pub pin: bool, } impl Config { - /// `count` workers, pinned. + /// `count` workers, unpinned. pub fn new(count: u16) -> Self { - Self { count, pin: true } + Self { count, pin: false } } /// Whether to pin each worker to a core. From d34978cc79e063e553589e208350ce72e389d883 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:41:12 -0700 Subject: [PATCH 17/17] fix(quic): issue sharded CIDs from the random generator HashedConnectionIdGenerator::set_shard overwrote the first of its three nonce bytes with the shard prefix, so a 256-way shard had only 65,536 CIDs. A client picks the Initial's destination CID and can steer every connection to one shard, and once that shard's CIDs are taken, Endpoint::new_cid spins forever. moq-uring's unsharded endpoints had also dropped from 8 random bytes to the hashed generator's 3. set_shard moves to RandomConnectionIdGenerator, which keeps every bit but the prefix byte and the two marker bits random (about 54 per shard), the same encoding #5638's moq_sock::shard::cid issued. Its validate refuses a CID without the shard's prefix or the marker, so a stray packet is still dropped rather than reset. HashedConnectionIdGenerator returns to upstream, and moq-uring issues random CIDs sharded or not. The relay config doc no longer says a worker's packets never cross cores: they stay on one thread, and runtime.pin keeps it on one core. Found by Codex and CodeRabbit review on #5688. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 3 +- rs/moq-quic/src/cid_generator.rs | 226 +++++++++++++++++----------- rs/moq-sock/README.md | 2 +- rs/moq-sock/src/shard.rs | 2 +- rs/moq-tokio/src/quic/backend.rs | 2 +- rs/moq-uring/src/quic/driver/mod.rs | 15 +- rs/moq-uring/src/quic/endpoint.rs | 6 +- 7 files changed, 157 insertions(+), 99 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index d6835ceca7..5faf7df0a1 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -112,7 +112,8 @@ window, so very small values can prevent other requests from progressing. By default one work-stealing runtime serves every connection off one UDP socket. On Linux, QUIC can instead run on single-threaded workers, each with its own socket on the listen address (`SO_REUSEPORT`), so a connection is -handled start to finish by one thread and its packets never cross cores. +handled start to finish by one worker thread and its packets never cross +threads. `pin = true` also keeps that thread on one core. ```toml [runtime] diff --git a/rs/moq-quic/src/cid_generator.rs b/rs/moq-quic/src/cid_generator.rs index ace212ceb7..85215eb070 100644 --- a/rs/moq-quic/src/cid_generator.rs +++ b/rs/moq-quic/src/cid_generator.rs @@ -39,11 +39,12 @@ pub struct InvalidCid; /// Generates purely random connection IDs of a specified length /// /// Random CIDs can be smaller than those produced by [`HashedConnectionIdGenerator`], but cannot be -/// usefully [`validate`](ConnectionIdGenerator::validate)d. +/// usefully [`validate`](ConnectionIdGenerator::validate)d unless they name a shard. #[derive(Debug, Clone, Copy)] pub struct RandomConnectionIdGenerator { cid_len: usize, lifetime: Option, + shard: Option, } impl Default for RandomConnectionIdGenerator { @@ -51,10 +52,41 @@ impl Default for RandomConnectionIdGenerator { Self { cid_len: 8, lifetime: None, + shard: None, } } } +/// One endpoint's slot in a group sharing a port, which its connection IDs name. +#[derive(Debug, Clone, Copy)] +struct Shard { + index: u16, + count: u16, +} + +impl Shard { + /// A first byte congruent to `index` modulo `count`, keeping as much of the byte random as + /// the group size leaves: `256 / count` values. + fn prefix(self) -> u8 { + let count = u32::from(self.count); + let stride = rand::rng().random_range(0..256 / count); + (stride * count + u32::from(self.index)) as u8 + } + + /// Whether `cid` carries this shard's prefix and the marker. + fn owns(self, cid: &[u8]) -> bool { + cid.len() > SHARD_MARKER_BYTE + && u16::from(cid[0]) % self.count == self.index + && cid[SHARD_MARKER_BYTE] & SHARD_MARKER == SHARD_MARKER + } +} + +/// The byte of a sharded CID that carries [`SHARD_MARKER`]. +const SHARD_MARKER_BYTE: usize = 3; +/// Both bits overlay the reserved key-encryption pattern of an SRT data header; see +/// [`RandomConnectionIdGenerator::set_shard`]. +const SHARD_MARKER: u8 = 0x18; + impl RandomConnectionIdGenerator { /// Initialize Random CID generator with a fixed CID length /// @@ -72,16 +104,55 @@ impl RandomConnectionIdGenerator { self.lifetime = Some(d); self } + + /// Name shard `index` of a `count`-endpoint group in every CID + /// + /// The first byte is congruent to `index` modulo `count`, so a `SO_REUSEPORT` steering filter + /// that reduces a packet's first destination CID byte modulo the group size delivers it to the + /// endpoint that issued the CID. The fourth byte also sets both bits of `0x18`, which overlay + /// the reserved key-encryption pattern of an SRT data header, so the filter can tell a short + /// header from another protocol sharing the port. Every other bit stays random: a client + /// chooses the Initial's destination CID, so it can steer every connection to one shard, and + /// that shard alone has to have IDs to spare. [`validate`](ConnectionIdGenerator::validate) + /// then refuses a CID without this shard's prefix and the marker, so a stray packet is dropped + /// rather than answered with a stateless reset for a connection a sibling owns. + /// + /// Panics unless `index < count <= 256`, since one byte cannot name more shards, or if the + /// CIDs are too short to carry the marker. + pub fn set_shard(&mut self, index: u16, count: u16) -> &mut Self { + assert!( + index < count && count <= 256, + "shard {index} of {count} cannot be named by one byte" + ); + assert!( + self.cid_len > SHARD_MARKER_BYTE, + "a {}-byte CID cannot carry the shard marker", + self.cid_len + ); + self.shard = Some(Shard { index, count }); + self + } } impl ConnectionIdGenerator for RandomConnectionIdGenerator { fn generate_cid(&mut self) -> ConnectionId { let mut bytes_arr = [0; MAX_CID_SIZE]; rand::rng().fill_bytes(&mut bytes_arr[..self.cid_len]); + if let Some(shard) = self.shard { + bytes_arr[0] = shard.prefix(); + bytes_arr[SHARD_MARKER_BYTE] |= SHARD_MARKER; + } ConnectionId::new(&bytes_arr[..self.cid_len]) } + fn validate(&self, cid: ConnectionId) -> Result<(), InvalidCid> { + match self.shard { + Some(shard) if !shard.owns(&cid) => Err(InvalidCid), + _ => Ok(()), + } + } + /// Provide the length of dst_cid in short header packet fn cid_len(&self) -> usize { self.cid_len @@ -100,28 +171,6 @@ impl ConnectionIdGenerator for RandomConnectionIdGenerator { pub struct HashedConnectionIdGenerator { key: u64, lifetime: Option, - shard: Option, -} - -/// One endpoint's slot in a group sharing a port, which its connection IDs name. -#[derive(Debug, Clone, Copy)] -struct Shard { - index: u16, - count: u16, -} - -impl Shard { - /// A first byte congruent to `index` modulo `count`, keeping as much of the byte random as - /// the group size leaves: `256 / count` values. - fn prefix(self) -> u8 { - let count = u32::from(self.count); - let stride = rand::rng().random_range(0..256 / count); - (stride * count + u32::from(self.index)) as u8 - } - - fn owns(self, first: u8) -> bool { - u16::from(first) % self.count == self.index - } } impl HashedConnectionIdGenerator { @@ -138,7 +187,6 @@ impl HashedConnectionIdGenerator { Self { key, lifetime: None, - shard: None, } } @@ -147,39 +195,6 @@ impl HashedConnectionIdGenerator { self.lifetime = Some(d); self } - - /// Name shard `index` of a `count`-endpoint group in the first byte of every CID - /// - /// The byte is congruent to `index` modulo `count`, so a `SO_REUSEPORT` steering filter that - /// reduces a packet's first destination CID byte modulo the group size delivers it to the - /// endpoint that issued the CID. The fourth byte also sets both bits of `0x18`, which overlay - /// the reserved key-encryption pattern of an SRT data header, so the filter can tell a short - /// header from another protocol sharing the port. [`validate`](ConnectionIdGenerator::validate) - /// then also refuses another shard's CIDs, so a stray packet is dropped rather than answered - /// with a stateless reset for a connection a sibling owns. - /// - /// Panics unless `index < count <= 256`: one byte cannot name more shards than that. - pub fn set_shard(&mut self, index: u16, count: u16) -> &mut Self { - assert!( - index < count && count <= 256, - "shard {index} of {count} cannot be named by one byte" - ); - self.shard = Some(Shard { index, count }); - self - } - - /// The keyed hash of `nonce`, carrying the shard marker in its first byte when sharded. - fn signature(&self, nonce: &[u8]) -> [u8; SIGNATURE_LEN] { - let mut hasher = rustc_hash::FxHasher::default(); - hasher.write_u64(self.key); - hasher.write(nonce); - let mut signature = [0; SIGNATURE_LEN]; - signature.copy_from_slice(&hasher.finish().to_le_bytes()[..SIGNATURE_LEN]); - if self.shard.is_some() { - signature[0] |= SHARD_MARKER; - } - signature - } } impl Default for HashedConnectionIdGenerator { @@ -192,22 +207,20 @@ impl ConnectionIdGenerator for HashedConnectionIdGenerator { fn generate_cid(&mut self) -> ConnectionId { let mut bytes_arr = [0; NONCE_LEN + SIGNATURE_LEN]; rand::rng().fill_bytes(&mut bytes_arr[..NONCE_LEN]); - if let Some(shard) = self.shard { - bytes_arr[0] = shard.prefix(); - } - let signature = self.signature(&bytes_arr[..NONCE_LEN]); - bytes_arr[NONCE_LEN..].copy_from_slice(&signature); + let mut hasher = rustc_hash::FxHasher::default(); + hasher.write_u64(self.key); + hasher.write(&bytes_arr[..NONCE_LEN]); + bytes_arr[NONCE_LEN..].copy_from_slice(&hasher.finish().to_le_bytes()[..SIGNATURE_LEN]); ConnectionId::new(&bytes_arr) } fn validate(&self, cid: ConnectionId) -> Result<(), InvalidCid> { - if let Some(shard) = self.shard - && !cid.first().is_some_and(|&first| shard.owns(first)) - { - return Err(InvalidCid); - } let (nonce, signature) = cid.split_at(NONCE_LEN); - match self.signature(nonce) == signature { + let mut hasher = rustc_hash::FxHasher::default(); + hasher.write_u64(self.key); + hasher.write(nonce); + let expected = hasher.finish().to_le_bytes(); + match expected[..SIGNATURE_LEN] == signature[..] { true => Ok(()), false => Err(InvalidCid), } @@ -224,9 +237,6 @@ impl ConnectionIdGenerator for HashedConnectionIdGenerator { const NONCE_LEN: usize = 3; // Good for more than 16 million connections const SIGNATURE_LEN: usize = 8 - NONCE_LEN; // 8-byte total CID length -/// Set in a sharded CID's fourth byte, the signature's first; see -/// [`HashedConnectionIdGenerator::set_shard`]. -const SHARD_MARKER: u8 = 0x18; #[cfg(test)] mod tests { @@ -239,14 +249,31 @@ mod tests { generator.validate(cid).unwrap(); } + fn sharded(index: u16, count: u16) -> RandomConnectionIdGenerator { + let mut generator = RandomConnectionIdGenerator::default(); + generator.set_shard(index, count); + generator + } + + /// Bits that vary across `cids`: a fixed bit is entropy the generator lost. + fn varying_bits(cids: &[ConnectionId]) -> u32 { + let (mut ones, mut zeros) = ([0u8; 8], [0u8; 8]); + for cid in cids { + for (i, byte) in cid.iter().enumerate() { + ones[i] |= byte; + zeros[i] |= !byte; + } + } + (0..8).map(|i| (ones[i] & zeros[i]).count_ones()).sum() + } + /// The steering filter's only requirement: whatever first byte a shard issues reduces back /// to that shard, at every group size including the one-value-per-shard ceiling. #[test] fn a_shard_cid_names_its_shard() { for count in [1u16, 2, 3, 7, 8, 64, 255, 256] { for index in 0..count { - let mut generator = HashedConnectionIdGenerator::new(); - generator.set_shard(index, count); + let mut generator = sharded(index, count); for _ in 0..32 { let cid = generator.generate_cid(); assert_eq!(cid.len(), generator.cid_len()); @@ -258,28 +285,57 @@ mod tests { } } - /// A shard refuses a sibling's CIDs even under the same key, so a misrouted packet is dropped - /// rather than reset. + /// A client picks the Initial's destination CID, so it can steer every connection to one + /// shard, and that shard alone has to keep a wide space: only the prefix byte and the two + /// marker bits are spent on the encoding. When the prefix took a byte of a 3-byte nonce, a + /// 256-way shard had 65,536 CIDs, and the endpoint spins looking for a free one once they are + /// taken. + #[test] + fn a_full_size_shard_keeps_its_entropy() { + let mut generator = sharded(0, 256); + let cids: Vec<_> = (0..1024).map(|_| generator.generate_cid()).collect(); + assert_eq!(varying_bits(&cids), 64 - 8 - 2); + } + + /// The unsharded path spends nothing on the encoding. + #[test] + fn an_unsharded_cid_keeps_its_entropy() { + let mut generator = RandomConnectionIdGenerator::default(); + let cids: Vec<_> = (0..1024).map(|_| generator.generate_cid()).collect(); + assert_eq!(varying_bits(&cids), 64); + } + + /// A shard refuses a sibling's CIDs and CIDs without the marker, so a misrouted or foreign + /// packet is dropped rather than reset. #[test] - fn a_shard_refuses_a_sibling_cid() { - let mut first = HashedConnectionIdGenerator::from_key(7); - first.set_shard(0, 4); - let mut second = HashedConnectionIdGenerator::from_key(7); - second.set_shard(1, 4); - let cid = second.generate_cid(); - second.validate(cid).unwrap(); - assert!(first.validate(cid).is_err()); + fn a_shard_refuses_what_it_did_not_issue() { + let first = sharded(0, 4); + let cid = sharded(1, 4).generate_cid(); + assert!(first.validate(cid).is_err(), "took a sibling's CID"); + + let mut unmarked = sharded(0, 4).generate_cid(); + unmarked[3] &= !0x18; + assert!( + first.validate(unmarked).is_err(), + "took a CID without the marker" + ); } #[test] #[should_panic] fn a_shard_past_the_group_is_refused() { - HashedConnectionIdGenerator::new().set_shard(4, 4); + RandomConnectionIdGenerator::default().set_shard(4, 4); } #[test] #[should_panic] fn a_group_past_one_byte_is_refused() { - HashedConnectionIdGenerator::new().set_shard(0, 257); + RandomConnectionIdGenerator::default().set_shard(0, 257); + } + + #[test] + #[should_panic] + fn a_cid_too_short_for_the_marker_is_refused() { + RandomConnectionIdGenerator::new(3).set_shard(0, 2); } } diff --git a/rs/moq-sock/README.md b/rs/moq-sock/README.md index 0c1d8614d9..7cab2a2101 100644 --- a/rs/moq-sock/README.md +++ b/rs/moq-sock/README.md @@ -13,7 +13,7 @@ runtimes (`moq-tokio` and `moq-uring`). member socket, so dropping one serving handle cannot renumber the survivors. A `Shard` names the slot a member holds, which the endpoint serving it names in every connection id it issues, alongside a marker no SRT data header - carries (`moq_quic::HashedConnectionIdGenerator::set_shard`). Anything else + carries (`moq_quic::RandomConnectionIdGenerator::set_shard`). Anything else on the port, including other protocols, keeps the kernel's 4-tuple hash. - **`cpu`**: pinning worker threads to cores. - **`udp`**: sending and receiving with GSO, GRO, and ECN, a hard fork of diff --git a/rs/moq-sock/src/shard.rs b/rs/moq-sock/src/shard.rs index 2b00f7c038..af6d349da7 100644 --- a/rs/moq-sock/src/shard.rs +++ b/rs/moq-sock/src/shard.rs @@ -37,7 +37,7 @@ //! [`Shard::index`] modulo [`Shard::count`], and whose fourth byte has both //! bits of `0x18` set, a pattern no valid SRT data header carries, so a short //! header from another protocol on the port keeps the 4-tuple hash. -//! `moq_quic::HashedConnectionIdGenerator::set_shard` issues exactly those. +//! `moq_quic::RandomConnectionIdGenerator::set_shard` issues exactly those. //! The kernel selects `socks[A % count]` for a returned accumulator `A`, so //! feeding it that byte lands on the owner. //! diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 1d1c5777a6..1dbb82af28 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -776,7 +776,7 @@ impl Server { "encoding the shard in connection IDs" ); endpoint_config.cid_generator(Arc::new(move || { - let mut cids = moq_quic::HashedConnectionIdGenerator::new(); + let mut cids = moq_quic::RandomConnectionIdGenerator::default(); cids.set_shard(shard.index(), shard.count()); Box::new(cids) })); diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index b056f68bc1..9a8abcf634 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -105,14 +105,15 @@ pub(crate) fn endpoint_config(shard: Option) -> Result