diff --git a/demo/relay/leaf0.toml b/demo/relay/leaf0.toml index 5aab70afd91..e18d51e2741 100644 --- a/demo/relay/leaf0.toml +++ b/demo/relay/leaf0.toml @@ -16,7 +16,7 @@ tls.key = ["leaf0.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # tls.root = ["ca.pem"] @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/"] public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/demo/relay/leaf1.toml b/demo/relay/leaf1.toml index 5e80c9b1421..b888ff38256 100644 --- a/demo/relay/leaf1.toml +++ b/demo/relay/leaf1.toml @@ -16,7 +16,7 @@ tls.key = ["leaf1.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # tls.root = ["ca.pem"] @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/", "https://localhost:4444/"] public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/demo/relay/prod.toml b/demo/relay/prod.toml index 1529ac7550d..fe4ca26be31 100644 --- a/demo/relay/prod.toml +++ b/demo/relay/prod.toml @@ -30,7 +30,7 @@ key = "key.pem" # validates over web.https, which must be reachable on TCP port 443. # [acme] # domains = ["relay.example.com"] -# dir = "/var/lib/moq-relay/acme" +# cache = "/var/lib/moq-relay/acme" # See docs/auth.md for more information. [auth] @@ -39,7 +39,7 @@ key = "key.pem" # moq auth serve --listen 127.0.0.1:4440 --key root.jwk \ # --public-subscribe 'anon/**' --public-subscribe 'demo/**' \ # --public-publish 'anon/**' --public-publish 'demo/viewer/**' \ -# --mtls-publish '**' --mtls-subscribe '**' --mtls-peer +# --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer # # For multiple keys with rotation, give it `--key-dir /path/to/keys/` instead: # keys are named {kid}.jwk and resolved by the kid in the JWT header. diff --git a/demo/relay/root.toml b/demo/relay/root.toml index 0043426f588..9fb262e472d 100644 --- a/demo/relay/root.toml +++ b/demo/relay/root.toml @@ -17,7 +17,7 @@ tls.key = ["root.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # `just ca` generates the CA. # tls.root = ["ca.pem"] @@ -32,6 +32,6 @@ listen = "[::]:4443" public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 2b4b242aaa9..d3b3b27388e 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -381,7 +381,7 @@ MoQ process and `moq-relay` on the LAN, with no other configuration. Without `--cluster-lan-secret`, anyone who can reach the listener joins, so leave it unset only on networks you trust. `--cluster-connect` is a MoQ side on its own, so it needs no `--connect`. WAN cluster dials (`--cluster-connect`, -`--cluster-connect-api`) share `--connect-bind` with `--connect`, and each binds +`--cluster-api-url`) share `--connect-bind` with `--connect`, and each binds it, while LAN dials take an ephemeral port. So a fixed `--connect-bind` port with both `--connect` and a WAN cluster peer is refused at startup: use port 0, or drop `--connect` (unset `MOQ_CONNECT` if it supplies the URL). diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index fd4299c6630..89e0582a382 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -166,7 +166,7 @@ cannot verify a presented token; both refuse it as unsupported. A moq-transport same way on any draft, but its grant authorizes only that request and ends with it; it never joins the union. A `PUBLISH_NAMESPACE` grant must cover the whole namespace: `room/**`, not just `room`. A session verifies at most -`verifications` tokens at once (`--auth-verifications`, default 128), counting +128 tokens at once, counting checks still running past the admission timeout. Later tokens wait their turn rather than being refused. Each HTTP request is admitted on its own, so an HTTP/2 connection carries at most as many requests at once, and the client @@ -267,14 +267,14 @@ verifies client certificates, so a relay without `listen.bind` refuses `listen.tls.root` too. Cluster peers are admitted the same way, so a mesh runs -`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer` (or a +`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` (or a server granting its cluster CA everything with `peer: true`); see -[Clustering](/bin/relay/cluster). `--mtls-peer` marks every certificate as +[Clustering](/bin/relay/cluster). `--mtls-link peer` marks every certificate as another relay; leave it off when certificates identify clients. Without `peer`, a certificate is a client's: it routes like one and sees [hidden](/concept/moq-lite#hidden-broadcasts) routes only by asking. -`--mtls-upstream`, which needs `--mtls-peer`, also marks them -[upstream](/bin/relay/cluster#upstream-links). Changing either flag ends each +`--mtls-link upstream` also marks them +[upstream](/bin/relay/cluster#upstream-links). Changing the link role ends each live mTLS session at its next re-check (the `--revalidate` cadence or a `moq auth revalidate` push), so the mesh redials once. @@ -300,7 +300,7 @@ map future transport names to `unknown`, which your policy should handle explici moq auth serve --listen 127.0.0.1:4440 \ --key-dir /etc/moq/keys \ --public-subscribe 'anon/**' --public-publish 'anon/**' \ - --mtls-publish '**' --mtls-subscribe '**' --mtls-peer \ + --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer \ --tier edge --expires 1d --revalidate 1m --limit-remote 64 ``` @@ -350,7 +350,7 @@ patterns: `--auth-public 'PREFIX/**'`. | `--auth-public-publish` / `--auth-public-subscribe` | `--public-publish` / `--public-subscribe`, as patterns | | `--auth-public-api URL` | your own server answering the contract | | `--auth-mtls-tier LABEL` | `--tier LABEL` (one tier per server) | -| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-peer` | +| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` | | `--auth-api` (token or proxy mode), `Cache-Control` | `--auth-url` pointed at any server answering the contract; `revalidate` and `expires` in the grant | | `--auth-domain` | your server reads `server_name` and decides | @@ -375,8 +375,7 @@ session at the grant's `expires`, and a connection whose admission is still unanswered at its `listen.timeout` handshake deadline is closed; a token presented in band is refused after ten seconds. `Admission::attached()` marks a token presented [in band](#in-band-renewal) for a session already admitted under the same id, -so count sessions only on the others. Gateway accept loops can call `Cluster::admit` for the same -scoped session as a native connection. See +so count sessions only on the others. See [docs.rs/moq-relay](https://docs.rs/moq-relay) and [moq-auth](/lib/rs/moq-auth). diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index b01de091450..5ec969ea4af 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -67,7 +67,7 @@ that is another source, so it restarts. ## Topology List the peers each relay dials. That's the whole topology: a relay dials only -peers from `connect`, [`connect_api`](#dynamic-peer-lists), or +peers from `connect`, [`api.url`](#dynamic-peer-lists), or [LAN discovery](#lan-discovery), never a URL learned from an announcement. ```toml @@ -79,7 +79,7 @@ connect = ["https://us-east.example.com/"] A chain (`eu-west <- us-east <- us-west`) dedupes fetches through the middle; a full mesh trades that for one fewer hop. Mix shapes as your traffic demands. -For a full mesh, list every other relay, or serve the list from `connect_api`. +For a full mesh, list every other relay, or serve the list from `api.url`. A session carries both directions, so one dial per pair is enough; listing a pair on both sides opens a redundant second session. @@ -124,7 +124,7 @@ connect = [ The mark belongs to the link, whichever side dialed: a dialed peer is upstream when its `connect` entry says so, and a peer that dials in when its [grant](/bin/relay/auth#the-contract) sets `"upstream": true` beside -`"peer": true`. `moq auth serve --mtls-peer --mtls-upstream` grants that to +`"peer": true`. `moq auth serve --mtls-link upstream` grants that to every certificate, so use it only where cores are the only relays dialing in with mTLS: on a hub that leaf relays dial into, it marks every leaf upstream and the hub stops forwarding between them. A relay that predates the mark treats every link as @@ -221,7 +221,7 @@ different `app` never discover each other. ## Dynamic peer lists -Point `connect_api` at an HTTP(S) endpoint or local file returning a JSON +Point `api.url` at an HTTP(S) endpoint or local file returning a JSON array of peers: bare URL strings and/or the same objects `connect` accepts. The relay re-checks it (honoring `Cache-Control`, or watching the file) and reconciles: new peers are dialed, missing ones dropped, @@ -233,15 +233,17 @@ changed URLs redialed. A bad fetch keeps the last good list. ```toml [cluster] -connect_api = "https://api.example.com/cluster/peers" node = "https://us-west.example.com/" + +[cluster.api] +url = "https://api.example.com/cluster/peers" ``` HTTPS peer-list fetches present the same client certificate as cluster dials, but authenticate the API with system roots and the hostname in its URL. Mesh roots, fingerprints, insecure mode, and the mesh hostname override do not -apply to the API. For a private API, set `cluster.connect_api_tls_root` to PEM -paths (or repeat `--cluster-connect-api-tls-root`); these replace system trust +apply to the API. For a private API, set `cluster.api.tls.root` to PEM +paths (or repeat `--cluster-api-tls-root`); these replace system trust for API fetches only. Malformed roots fail startup, and files reload for new connections when rotated. @@ -274,9 +276,9 @@ Peers dial with **mTLS** (recommended: `listen.tls.root` on the listener, URL, `token` on a peer object, or a shared `cluster.token` file for every listed peer). The accepting relay admits a peer like any client, so a mesh needs an auth server that grants the cluster CA, such as -`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`. An +`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`. An accepted peer counts as a cluster peer only when its grant sets `peer: true`, -as `--mtls-peer` does; otherwise what it announces counts as ingest here, like +as `--mtls-link peer` does; otherwise what it announces counts as ingest here, like a client's, and it lists [hidden](/concept/moq-lite#hidden-broadcasts) routes only by asking, which a relay below moq-lite-07-wip, or on moq-transport without the hidden extension, cannot. Dials retry forever with jittered backoff diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 1b8c57f5e1f..90ee5c0d407 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -99,7 +99,7 @@ Raise the receive windows when a fat, long path idles below the link rate. The 128 MiB `receive_window` default carries several Gbps at a 100 ms RTT and bounds how much unread data and incomplete messages one peer can make the relay buffer. It has to fit the largest message, just over 64 MiB, so a smaller -value, here or in `[peer.quic]`, is refused at startup. When incomplete +value, here or in `[limits.peer.quic]`, is refused at startup. When incomplete messages fill the window, the relay cancels the least urgent stream: the lowest subscription priority, then the group its subscription would deliver last. Over iroh the window bounds only @@ -107,38 +107,38 @@ unread data, and over WebSocket or TCP nothing bounds incomplete messages. `stream_receive_window` only paces one stream: its credit returns as bytes are read, whether or not their message is complete. -## \[session] and \[peer] +## \[limits.client] and \[limits.peer] Session limits apply separately to each session on every transport. The relay -uses `[peer]` for an admitted grant marked `peer` or `upstream`, including +uses `[limits.peer]` for an admitted grant marked `peer` or `upstream`, including verified LAN membership, and for links it dials through the cluster. A client certificate, route hop, or cluster-looking URL alone does not select peer limits. ```toml -[session] +[limits.client] announces = 1000 # Simultaneous broadcasts or namespaces. requests = 10000 # Other simultaneous moq-transport requests. -[peer] +[limits.peer] announces = 100000 requests = 20000 -[peer.quic] +[limits.peer.quic] max_streams = 100000 # Concurrent QUIC streams per direction. receive_window = 671088640 # Connection-wide receive window (640 MiB). send_window = 671088640 # Unacknowledged outgoing data (640 MiB). ``` -`[peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`; +`[limits.peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`; any other `[quic]` key is refused. They replace the `[quic]` values for a peer's QUIC connection once its grant is known, before the SETUP reply. Links the relay dials get them from the handshake on. A link carrying a whole cluster is then not throttled by `[quic]` limits sized for one viewer, for example -`--peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep +`--limits-peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep their transport's own limits. These are the defaults. Each subscription and FETCH holds a stream on moq-lite -and on moq-transport draft 17 and later, so `[quic]` or `[peer.quic]` +and on moq-transport draft 17 and later, so `[quic]` or `[limits.peer.quic]` `max_streams` bounds them there, and a peer past it waits rather than fails. On drafts 14 to 16, `announces` and `requests` together size the request window instead, which every request counts against, so very small values can @@ -204,13 +204,14 @@ See [HTTP endpoints](/bin/relay/http). ## \[acme] The relay provisions and renews a publicly trusted certificate itself, for QUIC -and HTTPS alike (Unix only, where the key can be kept owner-only): +and HTTPS alike (Unix only, where the key can be kept owner-only). This needs +the default `acme` Cargo feature; builds without it refuse configured issuance: ```toml [acme] domains = ["relay.example.com"] # Enables ACME. No wildcards. contact = ["mailto:ops@example.com"] # Optional account contact. -dir = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only. +cache = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only. # directory = "https://acme-staging-v02.api.letsencrypt.org/directory" # Default: Let's Encrypt production. # timeout = "5m" # How long the first issuance may take before startup fails. ``` @@ -220,11 +221,11 @@ domain, so `web.https.listen` is required and must be reachable there, directly or forwarded. `[acme]` cannot be combined with `listen.tls.cert`, `key`, or `generate`, or with `web.https.cert` and `key`. -With no usable certificate in `dir` (missing, expired, missing a domain, or +With no usable certificate in `cache` (missing, expired, missing a domain, or from another directory), startup answers only the challenge until one is issued; QUIC, the other routes, and readiness wait for it, and startup fails with the ACME error once `timeout` runs out. It also fails after 3 failed -validations within an hour, counting earlier starts recorded in `dir`, so a +validations within an hour, counting earlier starts recorded in `cache`, so a restart loop stops short of Let's Encrypt's limit of 5 per hostname per hour. Renewal runs in the background well before expiry and reaches every listener on its next handshake. A failed renewal keeps the current certificate serving and retries. @@ -267,8 +268,6 @@ url = "http://127.0.0.1:4440/" # An auth server asked once per session eve # public = "anon/**" # Or a static anonymous grant rooted at /, publish and subscribe alike. # public_subscribe = ["anon/**", "demo/**"] # Or split them; patterns, `foo/**` for a subtree. # public_publish = ["anon/**"] -verifications = 128 # In-band tokens one session, or requests one HTTP/2 connection, - # has verifying at once; later ones wait. ``` See [Authentication](/bin/relay/auth). @@ -279,12 +278,14 @@ See [Authentication](/bin/relay/auth). [cluster] connect = ["https://us-east.example.com/?cost=10"] # Peers to dial. ?cost prices the link, or use {url, cost, egress, token, upstream} objects. node = "https://us-west.example.com/" # This relay's own URL. -# connect_api_tls_root = ["api-ca.pem"] # Private API roots, independent from connect.tls. -connect_api = "https://api.example.com/peers" # Or fetch the peer list (JSON array of URLs and/or objects) live. token = "cluster.jwt" # JWT for dials without an inline ?jwt=. id = 12345 # Stable Hop ID across restarts. max_hops = 32 # Longest accepted/forwarded hop path (1 to 32). +[cluster.api] +url = "https://api.example.com/peers" # Live JSON peer list; a local file also works. +# tls.root = ["api-ca.pem"] # API server trust, independent from connect.tls. + [cluster.lan] # Find peers on the LAN over mDNS. enabled = true # secret = "/etc/moq/cluster.key" # Optional: a file holding 64 hex chars. diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 609d9a579ee..4fd631190fe 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -57,7 +57,10 @@ between releases are listed in `moq-relay` is also a library, for an application that wants extra HTTP routes or in-process workers against the cluster origin. Load a `Relay`, take what you need, and call `run`, which keeps the listeners, QUIC workers, and -shutdown inside the relay. +shutdown inside the relay. An embedded CLI calls +`config.relay.merge(&cli_layer)?` after its own `parse_from_with_settings` to +load the relay file and environment, retain provenance, and refuse obsolete +spellings. CLI-only fields on the embedding root stay intact. ```rust use axum::routing::get; @@ -89,7 +92,7 @@ relay.with_web(web).run().await?; from `relay.shutdown().activity()` and pass `relay.shutdown().clone()` to `Connection::with_shutdown`, so the drain waits for its handshakes too. - To serve a session from any other moq-net transport, such as the in-memory - one in tests, read its SETUP with `moq_net::Server::accept_request` and pass + one in tests, enable the `test-support` feature, read its SETUP with `moq_net::Server::accept_request` and pass `Accepted::new(handshake, transport)` to `Connection::new`. The client must send its path in the SETUP. - To decide admissions yourself, leave `[auth]` empty and answer diff --git a/doc/setup/prod.md b/doc/setup/prod.md index 370cbb60d66..43ac32268e0 100644 --- a/doc/setup/prod.md +++ b/doc/setup/prod.md @@ -26,7 +26,7 @@ listen = "[::]:443" [acme] domains = ["relay.example.com"] contact = ["mailto:ops@example.com"] -dir = "/var/lib/moq-relay/acme" +cache = "/var/lib/moq-relay/acme" [auth] url = "http://127.0.0.1:4440/" diff --git a/nix/modules/moq-relay.nix b/nix/modules/moq-relay.nix index 981190d38bd..9ab5896f9e0 100644 --- a/nix/modules/moq-relay.nix +++ b/nix/modules/moq-relay.nix @@ -256,7 +256,7 @@ in "${pkgs.moq-cli}/bin/moq auth serve" "--listen 127.0.0.1:${toString cfg.auth.port}" "--key ${authKey}" - "--mtls-publish '**' --mtls-subscribe '**' --mtls-peer" + "--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer" ] ++ lib.optionals (cfg.auth.publicPath != null) [ "--public-publish '${publicPattern}' --public-subscribe '${publicPattern}'" @@ -356,7 +356,7 @@ in } // lib.optionalAttrs (cfg.acme.domains != [ ]) { MOQ_ACME_DOMAINS = lib.concatStringsSep "," cfg.acme.domains; - MOQ_ACME_DIR = "${cfg.stateDir}/acme"; + MOQ_ACME_CACHE = "${cfg.stateDir}/acme"; MOQ_WEB_HTTPS_LISTEN = "[::]:${toString cfg.acme.httpsPort}"; } // lib.optionalAttrs (cfg.acme.contact != [ ]) { diff --git a/packaging/moq-relay/relay.toml b/packaging/moq-relay/relay.toml index 249e6e2685d..2fe700f5b21 100644 --- a/packaging/moq-relay/relay.toml +++ b/packaging/moq-relay/relay.toml @@ -30,7 +30,7 @@ key = "/var/lib/moq-relay/key.pem" # [acme] # domains = ["relay.example.com"] # contact = ["mailto:ops@example.com"] -# dir = "/var/lib/moq-relay/acme" +# cache = "/var/lib/moq-relay/acme" # Uncomment to also serve plain HTTP/WebSocket on port 80. # [web.http] diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index e64b1f8ceaa..ce991f34eee 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -81,7 +81,6 @@ Rejected in planning: - [One frame rate name](/quest/m1/api-audit/frame-rate.md) - core `moq_video::Rate` becomes `FrameRate`, as the bindings name it - [CLI and gst flags](/quest/m1/api-audit/cli-flags.md) - one flag name per meaning across CLI subcommands and gst elements, and `linger` means one thing - [Transport config](/quest/m1/api-audit/transport-config.md) - moq-tokio and moq-quic configs have one meaning for unset, enums for modes, no order-dependent setters, and no leftover features -- [Relay config](/quest/m1/api-audit/relay-config.md) - moq-relay and its embedders load config through one path that refuses old spellings, under names that do not collide - [Session-only items stay private](/quest/m1/api-audit/model-private.md) - model items only the built-in sessions use are private - [Only sessions accept a request](/quest/m1/api-audit/claim-request-type.md) - accepting an application claim's request fails to compile, and the peer-route hatches are crate-private - [One range setter](/quest/m1/api-audit/serve.md) - `with_range` replaces seven cursor setters and nothing else of the serving cursor goes public diff --git a/quest/m1/api-audit/relay-config.md b/quest/m1/api-audit/relay-config.md deleted file mode 100644 index d03dd09daef..00000000000 --- a/quest/m1/api-audit/relay-config.md +++ /dev/null @@ -1,45 +0,0 @@ -# [M] Relay config - -## Goal - -moq-relay and every embedder load configuration through one call that -refuses old spellings, and relay config names do not collide with module -names or each other. - -## Plan - -Decided 2026-10-22 (API audit): - -- One loader: `Config::merge_into(&CliLayer, &EnvLayer, Option)` - takes about 25 lines of setup, so moq.pro's edge bypasses it - (`toml::from_str` plus `update_from`) and `deprecated()` never runs: - `MOQ_SERVER_BIND` or `[server]` are silently ignored there. Provide one - call both binaries use (for example `moq_tokio::cli::load::()`) with - provenance, so an embedder can tell a set value from a default without - comparing against `Default`. -- `[session]` and `[peer]` collide with `moq_relay::session` and - `cluster::Peer`; rename to `[limits.client]` and `[limits.peer]`. -- `acme.dir` (a state path, documented required but typed `Option`) beside - `acme.directory` (the URL): rename `dir` to `acme.cache`; rustls-acme moves - behind an `acme` feature. -- `cluster.connect_api_tls_root` is flat while everything else nests - `*.tls.root`: `[cluster.api] url`, `tls.root`. -- `auth.verifications`/`Auth::with_verifications(usize)` is used only by a - bench and panics on 0: a private constant. -- `[listen|connect.extensions]` stays (maintainer, 2026-10-23, reversing the - drop): it is the operator's interop knob for moq-transport extensions - ([moq-transport stays private](/quest/m1/api-audit/ietf-private.md)). -- `uring::Workers::spawn` then `serve()` is order-dependent (`serve()` alone - no-ops): typestate `spawn(self) -> Spawned`. -- `cluster::with_client_tls` repeats TLS `with_connect` holds: delete. -- mTLS: `serve::Policy::{mtls_peer, mtls_upstream}` and the matching flags - can express "upstream without peer", refused at runtime; take - `Option` and one `--mtls-link` flag, keeping the - `Grant` wire booleans. -- Public only for tests: `Cluster::admit`, `cluster::Admitted`, and - `auth::hold` (move moq.pro onto them, since it keeps copies, or make them - `pub(crate)`); `Incoming`/`Accepted` behind `test-support`. -- Extend the `demo_configs_parse` test to `packaging/` and `test/`. - -Docs: `doc/bin/relay/`. Public API: moq-relay config and Rust API breaking; -old spellings are refused by name. Wire: none. diff --git a/quest/m1/uring-listen-extensions.md b/quest/m1/uring-listen-extensions.md index 840cf5499f0..65e7a1f5b7a 100644 --- a/quest/m1/uring-listen-extensions.md +++ b/quest/m1/uring-listen-extensions.md @@ -11,7 +11,7 @@ configured, as the tokio listeners already do. and never applies `listen.extensions`; the tokio path (`rs/moq-relay/src/relay.rs`, `websocket.rs`) does. Apply them and add a uring test that a configured extension is negotiated. -[Relay config](/quest/m1/api-audit/relay-config.md) keeps extensions as the +The relay config audit ([#6535](https://github.com/moq-dev/moq/pull/6535)) keeps extensions as the operator's interop knob, so every listener must honor it. Public API: none. Wire: none. diff --git a/quest/m2/release/README.md b/quest/m2/release/README.md index a6c06b4bb11..4ec6ee3e384 100644 --- a/quest/m2/release/README.md +++ b/quest/m2/release/README.md @@ -42,3 +42,4 @@ production stability proven merely because upstream artifacts published. - [Retire the web-transport JS qmux](/quest/m2/release/qmux-js-retire.md) - once `@moq/qmux` publishes from `js/qmux`, deprecate `@moq/web-socket-stream` and stop the second publisher - [moq.sh deployment](/quest/m2/release/moq-sh-deploy.md) - verify its release-triggered publication - [Runner client bump](/quest/m2/release/runner-client-bump.md) - the upstream interop runner's Rust client moves to the published moq-tokio, so it stops stamping every object +- [Relay loader adoption](/quest/m2/release/relay-loader-adoption.md) - once their pins carry it, embedding consumers load relay config through `Config::merge` and refuse obsolete spellings diff --git a/quest/m2/release/relay-loader-adoption.md b/quest/m2/release/relay-loader-adoption.md new file mode 100644 index 00000000000..fb662c343e6 --- /dev/null +++ b/quest/m2/release/relay-loader-adoption.md @@ -0,0 +1,35 @@ +# [M] Embedders load relay config through the shared loader + +## Goal + +Condition: an embedding consumer's moq-relay pin includes +[moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535), which adds +`Config::merge(&CliLayer)`. Check each consumer's lockfile for a moq-relay +version or revision that carries it, and advance a pin that lacks it through +that consumer's normal upstream-first process. + +Then every embedding consumer loads its relay config through +`Config::merge`, an explicit obsolete file or environment spelling refuses +its startup, and its embedded CLI-only fields survive the merge. Delete this +quest once every embedder is migrated. + +## Plan + +Split from the relay config API audit (maintainer, 2026-10-23): the upstream +loader landed first, because migrating a consumer before its pin carries the +API would break it. + +For each embedding consumer: + +1. Replace its direct TOML parsing plus `update_from` (or hand-built + `merge_into`) setup with `config.relay.merge(&cli_layer)?`. +2. Migrate the renamed settings and native types listed in the Impact + section of [moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535), + in its config files, environment, and code. +3. Verify with that consumer's own checks that an explicit obsolete file or + environment spelling refuses startup, and that its root CLI-only fields + survive, reading `Config::source` provenance rather than comparing against + default values. + +Do not add compatibility aliases upstream, and do not combine unrelated +pending transport configuration changes with a consumer's migration. diff --git a/rs/moq-auth/src/serve.rs b/rs/moq-auth/src/serve.rs index 8f6bc51090f..5eb1c4f14fb 100644 --- a/rs/moq-auth/src/serve.rs +++ b/rs/moq-auth/src/serve.rs @@ -54,6 +54,35 @@ pub struct Limits { pub remote: Option, } +/// A verified certificate's cluster-link role. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Link { + /// A cluster peer that may exchange routes in either direction. + Peer, + /// A peer that is never offered routes learned from another upstream. + Upstream, +} + +impl std::str::FromStr for Link { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "peer" => Ok(Self::Peer), + "upstream" => Ok(Self::Upstream), + _ => Err("expected peer or upstream"), + } + } +} + +impl std::fmt::Display for Link { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Peer => "peer", + Self::Upstream => "upstream", + }) + } +} + /// The decisions the server answers with, evaluated in order and stopping at the /// first that applies: a JWT, then a verified certificate, then the anonymous rules. /// A malformed or expired token is a refusal, never a fall through. @@ -81,12 +110,9 @@ pub struct Policy { /// What a session presenting a verified certificate is granted, rooted at `/`; /// empty refuses it. pub mtls: Permissions, - /// Mark every session presenting a verified certificate as a cluster - /// [peer](Grant::peer), for a mesh whose relays dial each other with mTLS. - pub mtls_peer: bool, - /// Mark every certificate's peer as [upstream](Grant::upstream) too. Requires - /// [`mtls_peer`](Self::mtls_peer). - pub mtls_upstream: bool, + /// The cluster-link role granted only to sessions presenting a verified certificate. + pub mtls_link: Option, + /// The tier stamped on every grant. pub tier: Option, /// How often the relay re-checks each grant; `None` never re-checks. The contract @@ -241,8 +267,8 @@ impl Policy { }; grant.revalidate = self.revalidate; grant.tier = self.tier.clone(); - grant.peer = certificate && self.mtls_peer; - grant.upstream = certificate && self.mtls_upstream; + grant.peer = certificate && self.mtls_link.is_some(); + grant.upstream = certificate && self.mtls_link == Some(Link::Upstream); Ok((grant, root)) } @@ -432,11 +458,8 @@ pub struct Server { impl Server { /// A server answering with `policy`, refusing one that asks for a re-check without /// a bound, bounds grants past the clock's range, caps sessions without the - /// re-check that ages out a dead relay's, or marks an upstream that is not a peer. + /// re-check that ages out a dead relay's,. pub fn new(policy: Policy) -> crate::Result { - if policy.mtls_upstream && !policy.mtls_peer { - return Err(crate::Error::UpstreamWithoutPeer); - } if policy.revalidate.is_some() && policy.expires.is_none() { return Err(crate::Error::UnboundedRevalidate); } @@ -965,7 +988,7 @@ mod tests { assert_eq!(policy.decide(&request("/")).await.unwrap_err(), Denial::NoPublicGrant); } - /// `mtls_peer` and `mtls_upstream` mark a certificate's grant, and only a certificate's. + /// The link role marks only a certificate's grant, never an anonymous session or JWT. #[tokio::test] async fn mtls_peer_marks_only_certificates() { let (dir, key) = key_dir(); @@ -973,14 +996,14 @@ mod tests { keys: Some(Keys::Dir(dir.path().into())), public: rules(&["**"], &["**"]), mtls: rules(&["**"], &["**"]), - mtls_peer: true, + mtls_link: Some(Link::Peer), ..Default::default() }; let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); assert!(grant.peer && !grant.upstream); let policy = Policy { - mtls_upstream: true, + mtls_link: Some(Link::Upstream), ..policy }; let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); @@ -993,15 +1016,23 @@ mod tests { assert!(!grant.peer && !grant.upstream); } - /// An upstream that is not a peer would answer grants every relay refuses. - #[test] - fn a_server_refuses_an_upstream_that_is_not_a_peer() { - let policy = Policy { - mtls: rules(&["**"], &["**"]), - mtls_upstream: true, - ..Default::default() - }; - assert!(matches!(Server::new(policy), Err(Error::UpstreamWithoutPeer))); + /// Every representable mTLS link role produces a valid wire grant. + #[tokio::test] + async fn mtls_link_roles_preserve_the_grant_contract() { + for (link, peer, upstream) in [ + (None, false, false), + (Some(Link::Peer), true, false), + (Some(Link::Upstream), true, true), + ] { + let policy = Policy { + mtls: rules(&["**"], &["**"]), + mtls_link: link, + ..Default::default() + }; + let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); + assert_eq!((grant.peer, grant.upstream), (peer, upstream)); + grant.validate().unwrap(); + } } /// A certificate would override a JWT meant to narrow it, and a JWT would narrow or diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index d2a8416b258..9cc63db9f92 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -6,7 +6,7 @@ //! `moq fetch ` to read one group. //! //! - The MoQ side (`--connect`, the `--listen*` transport binds, `--cluster-lan`, -//! and `--cluster-connect` / `--cluster-connect-api`; all optional, at least +//! and `--cluster-connect` / `--cluster-api-url`; all optional, at least //! one) attaches the shared Origin to the MoQ network, and comes before the //! first stage. They compose: dial a relay, accept incoming sessions, mesh //! with the LAN, and join a cluster all at once. @@ -537,9 +537,9 @@ impl MoqSide { } /// Whether a WAN cluster dial is configured (`--cluster-connect` or - /// `--cluster-connect-api`). + /// `--cluster-api-url`). fn cluster_dials(&self) -> bool { - !self.cluster.connect.is_empty() || self.cluster.connect_api.is_some() + !self.cluster.connect.is_empty() || self.cluster.api.url.is_some() } /// Reject a verb that needs the MoQ network but was given no way to reach it. @@ -557,7 +557,7 @@ impl MoqSide { !(self.client.url.is_some() && self.cluster_dials() && self.client.bind.is_some_and(|bind| bind.port() != 0)), - "--connect-bind with a fixed port cannot serve both --connect and --cluster-connect / --cluster-connect-api, which each bind it: use port 0, or drop --connect (unset MOQ_CONNECT if it supplies the URL)" + "--connect-bind with a fixed port cannot serve both --connect and --cluster-connect / --cluster-api-url, which each bind it: use port 0, or drop --connect (unset MOQ_CONNECT if it supplies the URL)" ); #[cfg(feature = "cluster-lan")] { @@ -2252,11 +2252,7 @@ mod tests { ("--listen-tcp-bind", "127.0.0.1:0", "--listen-tcp-bind"), ("--broadcast", "room", "--broadcast"), ("--cluster-connect", "https://relay.example", "--cluster-connect"), - ( - "--cluster-connect-api", - "https://api.example/peers", - "--cluster-connect-api", - ), + ("--cluster-api-url", "https://api.example/peers", "--cluster-api-url"), ("--cluster-node", "https://self.example", "--cluster-node"), ("--cluster-token", "cluster.jwt", "--cluster-token"), ("--cluster-id", "1", "--cluster-id"), @@ -2406,7 +2402,7 @@ mod tests { }; for peer in [ ["--cluster-connect", "https://relay.example"], - ["--cluster-connect-api", "https://api.example/peers"], + ["--cluster-api-url", "https://api.example/peers"], ] { let both = [ &["--connect", "https://relay.example", "--connect-bind", "127.0.0.1:4443"][..], @@ -2440,7 +2436,7 @@ mod tests { .expect("--connect alone owns a fixed bind"); } - /// `--cluster-connect` / `--cluster-connect-api` attach the process as a + /// `--cluster-connect` / `--cluster-api-url` attach the process as a /// cluster peer, so they are a MoQ side on their own. `--cluster-node` is /// identity, not an attachment. #[test] @@ -2449,14 +2445,8 @@ mod tests { .expect("parse"); assert!(cli.moq.validate().is_ok(), "a cluster dial is a MoQ side on its own"); - let cli = Invocation::try_parse_from([ - "moq", - "--cluster-connect-api", - "https://api.example/peers", - "import", - "ts", - ]) - .expect("parse"); + let cli = Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) + .expect("parse"); assert!(cli.moq.validate().is_ok(), "a cluster API is a MoQ side on its own"); let cli = Invocation::try_parse_from(["moq", "--cluster-node", "https://self.example", "import", "ts"]) diff --git a/rs/moq-cli/src/auth.rs b/rs/moq-cli/src/auth.rs index 9a6c2a06560..67acd137418 100644 --- a/rs/moq-cli/src/auth.rs +++ b/rs/moq-cli/src/auth.rs @@ -397,13 +397,14 @@ pub struct Serve { #[usage(long)] mtls_subscribe: Vec, - /// Mark sessions with a verified certificate as cluster peers: relays forwarding what entered elsewhere. - #[usage(long)] - mtls_peer: bool, + /// The role verified certificates have on cluster links: peer or upstream. + #[usage(long, choices("peer", "upstream"))] + mtls_link: Option, - /// Mark those peers upstream: never offered a route learned from another upstream. Needs `--mtls-peer`. - #[usage(long)] - mtls_upstream: bool, + #[usage(long = "mtls-peer", hide = true, default_missing = "true", num_args = 0..=1, require_equals = true)] + mtls_peer: Option, + #[usage(long = "mtls-upstream", hide = true, default_missing = "true", num_args = 0..=1, require_equals = true)] + mtls_upstream: Option, /// The tier label stamped on every grant, handed to the relay's stats. #[usage(long)] @@ -444,11 +445,11 @@ impl Serve { "--limit-token and --limit-remote need --revalidate, which ages out the slots of dead relays" ); } - if self.mtls_peer && self.mtls_publish.is_empty() && self.mtls_subscribe.is_empty() { - anyhow::bail!("--mtls-peer needs --mtls-publish or --mtls-subscribe, or every certificate is refused"); + if self.mtls_peer.is_some() || self.mtls_upstream.is_some() { + anyhow::bail!("--mtls-peer and --mtls-upstream are removed; use --mtls-link peer or --mtls-link upstream"); } - if self.mtls_upstream && !self.mtls_peer { - anyhow::bail!("--mtls-upstream needs --mtls-peer; only a cluster peer can be upstream"); + if self.mtls_link.is_some() && self.mtls_publish.is_empty() && self.mtls_subscribe.is_empty() { + anyhow::bail!("--mtls-link needs --mtls-publish or --mtls-subscribe, or every certificate is refused"); } // 0.14 read `anon` as the prefix `anon/`, and a pattern reads it as exactly the // broadcast `anon`, so either silent reading would mislead someone upgrading. @@ -487,8 +488,7 @@ impl Serve { }; policy.public = rules(&self.public_publish, &self.public_subscribe); policy.mtls = rules(&self.mtls_publish, &self.mtls_subscribe); - policy.mtls_peer = self.mtls_peer; - policy.mtls_upstream = self.mtls_upstream; + policy.mtls_link = self.mtls_link; policy.tier = self.tier.clone(); policy.revalidate = revalidate; policy.expires = expires; @@ -682,8 +682,8 @@ mod tests { "**", "--mtls-subscribe", "**", - "--mtls-peer", - "--mtls-upstream", + "--mtls-link", + "upstream", "--tier", "internal", "--revalidate", @@ -704,7 +704,7 @@ mod tests { ); assert!(policy.public.publish.is_empty()); assert_eq!(policy.mtls.publish, ["**".parse().unwrap()].into_iter().collect()); - assert!(policy.mtls_peer && policy.mtls_upstream); + assert_eq!(policy.mtls_link, Some(moq_auth::serve::Link::Upstream)); assert_eq!(policy.tier.as_deref(), Some("internal")); assert_eq!(policy.revalidate, Some(std::time::Duration::from_secs(30))); assert_eq!(policy.expires, Some(std::time::Duration::from_secs(7200))); @@ -726,7 +726,7 @@ mod tests { let bare = serve(&["moq", "auth", "serve"]).policy().unwrap(); assert!(bare.keys.is_none()); assert!(bare.public.is_empty() && bare.mtls.is_empty()); - assert!(!bare.mtls_peer && !bare.mtls_upstream); + assert_eq!(bare.mtls_link, None); assert_eq!(bare.revalidate, None); assert_eq!(bare.expires, None); @@ -739,11 +739,10 @@ mod tests { (&["--limit-token", "3"], "need --revalidate"), (&["--limit-remote", "3", "--expires", "1h"], "need --revalidate"), // A peer marking with nothing granted to certificates marks nobody. - (&["--mtls-peer"], "--mtls-peer needs"), - ( - &["--mtls-subscribe", "**", "--mtls-upstream"], - "--mtls-upstream needs --mtls-peer", - ), + (&["--mtls-link", "peer"], "--mtls-link needs"), + (&["--mtls-subscribe", "**", "--mtls-upstream"], "use --mtls-link"), + (&["--mtls-peer=false"], "use --mtls-link"), + (&["--mtls-upstream=false"], "use --mtls-link"), ] { let argv: Vec<&str> = ["moq", "auth", "serve"].iter().chain(args).copied().collect(); let err = serve(&argv).policy().unwrap_err().to_string(); diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 274b5b612e5..2c8b934ab1d 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -443,9 +443,7 @@ async fn spawn_moq( directions: Directions, tasks: &mut JoinSet>, ) -> anyhow::Result { - let cluster = cluster - .with_client_tls(moq.client.tls.build()?) - .with_connect(moq.client.clone(), moq.quic.clone()); + let cluster = cluster.with_connect(moq.client.clone(), moq.quic.clone()); #[cfg(feature = "iroh")] let cluster = match net.iroh.clone() { Some(iroh) => cluster.with_iroh(iroh), @@ -1421,38 +1419,28 @@ mod tests { tasks.abort_all(); } - /// An HTTP `--cluster-connect-api` is a MoQ side, so start-up must attach + /// An HTTP `--cluster-api-url` is a MoQ side, so start-up must attach /// client TLS the way the relay does rather than refuse after validate. #[tokio::test] - async fn cluster_connect_api_http_attaches_client_tls() { + async fn cluster_api_url_http_attaches_client_tls() { let _ = moq_tokio::crypto::install_default(); - let invocation = Invocation::try_parse_from([ - "moq", - "--cluster-connect-api", - "https://api.example/peers", - "import", - "ts", - ]) - .expect("parse"); + let invocation = + Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) + .expect("parse"); assert!(invocation.moq.validate().is_ok()); - let cluster = invocation - .moq - .cluster() - .expect("cluster") - .with_connect(invocation.moq.client.clone(), invocation.moq.quic.clone()); + let cluster = invocation.moq.cluster().expect("cluster"); let err = cluster .clone() .start() .await - .expect_err("http API without TLS") + .expect_err("HTTP API needs a dial template") .to_string(); - assert!(err.contains("client TLS"), "{err}"); - + assert!(err.contains("Cluster::with_connect"), "{err}"); cluster - .with_client_tls(invocation.moq.client.tls.build().expect("tls")) + .with_connect(invocation.moq.client.clone(), invocation.moq.quic.clone()) .start() .await - .expect("http API with TLS"); + .expect("HTTP API uses the dial template's TLS"); } /// Without `--connect`, only the cluster binds `--connect-bind`, so a fixed diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 363c63a37ac..06fe21d1e52 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,7 +25,8 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] -default = ["cluster-lan", "quic", "websocket", "uds"] +default = ["acme", "cluster-lan", "quic", "websocket", "uds"] +acme = ["dep:rustls-acme"] iroh = ["moq-tokio/iroh"] # Discover cluster peers on the LAN with mDNS (`--cluster-lan`). cluster-lan = ["moq-tokio/mdns"] @@ -79,7 +80,7 @@ rand = { workspace = true } reqwest = { workspace = true, features = ["rustls"] } reqwest-middleware = "0.5" rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } -rustls-acme = { version = "0.15.4", default-features = false, features = ["aws-lc-rs", "webpki-roots"] } +rustls-acme = { version = "0.15.4", optional = true, default-features = false, features = ["aws-lc-rs", "webpki-roots"] } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["json", "base64"] } diff --git a/rs/moq-relay/README.md b/rs/moq-relay/README.md index df6d155bfcf..258ef3a41ec 100644 --- a/rs/moq-relay/README.md +++ b/rs/moq-relay/README.md @@ -62,7 +62,7 @@ HTTPS is currently not supported. Relays can be joined together to proxy announcements and subscriptions. A viewer talks to whichever relay is closest; if their broadcast lives elsewhere in the cluster, the local relay fetches it from a neighbor and caches it. Hop tracking on every broadcast keeps loops out and picks the shortest path when there's more than one. - `--cluster-connect ` lists the peers this relay dials. Repeatable; defines the topology by hand. A simple chain like `eu-west <- us-east <- us-west` lets `us-east` cache and dedup the transatlantic fetches that fan out to many `us-west` viewers. -- `--cluster-connect-api ` fetches that list from an endpoint or file instead, reloading it without a restart. +- `--cluster-api-url ` fetches that list from an endpoint or file instead, reloading it without a restart. - `--cluster-lan` finds peers on the local network over mDNS. A relay only dials peers listed this way or found on the LAN. diff --git a/rs/moq-relay/benches/auth.rs b/rs/moq-relay/benches/auth.rs index 10de51d7652..a470739d7bb 100644 --- a/rs/moq-relay/benches/auth.rs +++ b/rs/moq-relay/benches/auth.rs @@ -1,7 +1,6 @@ //! Revalidate one token while sweeping unrelated sessions and held credentials, and -//! cycle one verification slot while sweeping the waiting tokens and the slot count. +//! drain token batches while sweeping offered concurrency and payload size. //! The real relay/controller runs over loopback TCP; timings include one AUTH round trip. -#![expect(clippy::disallowed_methods, reason = "host time measures benchmark throughput")] use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use moq_auth::{Grant, lease}; @@ -145,14 +144,12 @@ fn refresh(c: &mut Criterion) { group.finish(); } -/// One session with `verifications` slots held by an unanswered decider and `waiting` -/// tokens queued behind them. Each round refuses the oldest held check, which sends the -/// oldest waiting token, and presents one more so the queue keeps its depth. +/// One real session; each measured batch offers tokens before the decider drains them, +/// so requests beyond the fixed production window wait for a verification slot. struct Queue { _connection: moq_tokio::Connection, auth: moq_tokio::connection::auth::Handle, admissions: auth::Admissions, - held: std::collections::VecDeque, presented: usize, tasks: Vec>, } @@ -166,10 +163,9 @@ impl Drop for Queue { } impl Queue { - async fn new(verifications: usize, waiting: usize) -> Self { + async fn new() -> Self { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let (auth, mut admissions) = auth::Auth::embedded("bench"); - let auth = auth.with_verifications(verifications); let mut config = moq_tokio::listen::Config::default(); config.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); let mut server = config.init(Default::default()).unwrap().listen().await.unwrap(); @@ -193,46 +189,33 @@ impl Queue { let admission = admissions.next().await.unwrap(); admission.grant(lease::Consumer::fixed(grant("initial"))); }); - let mut queue = Self { + Self { auth: connection.auth(), _connection: connection, admissions, - held: Default::default(), presented: 0, tasks: vec![listener], - }; - for _ in 0..verifications { - queue.present(); - let admission = queue.admissions.next().await.unwrap(); - queue.held.push_back(admission); - } - for _ in 0..waiting { - queue.present(); } - // Let the waiting tokens reach the relay's queue before measuring. - tokio::time::sleep(Duration::from_millis(50)).await; - queue } - fn present(&mut self) { - let auth = self.auth.clone(); - let token = format!("token{}", self.presented); - self.presented += 1; - self.tasks.push(tokio::spawn(async move { - let _ = auth.add(token).await; - })); - } - - async fn round(&mut self) { - self.held - .pop_front() - .unwrap() - .refuse(auth::Error::Refused(Default::default())); - self.present(); - let admission = self.admissions.next().await.unwrap(); - self.held.push_back(admission); - // Refused tokens finish; drop their handles so the table stays flat. - self.tasks.retain(|task| !task.is_finished()); + async fn round(&mut self, offered: usize, bytes: usize) { + let mut tasks = Vec::with_capacity(offered); + for _ in 0..offered { + let auth = self.auth.clone(); + let token = format!("{:0>width$}", self.presented, width = bytes); + self.presented += 1; + tasks.push(tokio::spawn(async move { auth.add(token).await })); + } + for _ in 0..offered { + self.admissions + .next() + .await + .unwrap() + .refuse(auth::Error::Refused(Default::default())); + } + for task in tasks { + assert!(task.await.unwrap().is_err()); + } } } @@ -245,15 +228,15 @@ fn queue(c: &mut Criterion) { group.sample_size(10); group.warm_up_time(Duration::from_millis(100)); group.measurement_time(Duration::from_millis(300)); - // QMux grants 100 bidirectional streams, and every held or waiting token holds one. - for verifications in [1, 8, 32] { - for waiting in [0, 16, 48] { - let mut fixture = runtime.block_on(Queue::new(verifications, waiting)); + for offered in [1, 128, 256] { + for bytes in [16, 256, 4096] { + let mut fixture = runtime.block_on(Queue::new()); + group.throughput(criterion::Throughput::Elements(offered as u64)); group.bench_with_input( - BenchmarkId::new(verifications.to_string(), waiting), - &(verifications, waiting), - |b, _| { - b.iter(|| runtime.block_on(fixture.round())); + BenchmarkId::new(offered.to_string(), bytes), + &(offered, bytes), + |b, &(offered, bytes)| { + b.iter(|| runtime.block_on(fixture.round(offered, bytes))); }, ); } diff --git a/rs/moq-relay/src/acme.rs b/rs/moq-relay/src/acme.rs index 62730c7dfa7..86287c216a2 100644 --- a/rs/moq-relay/src/acme.rs +++ b/rs/moq-relay/src/acme.rs @@ -1,49 +1,13 @@ -//! Automatic certificates from an ACME directory, Let's Encrypt by default. -//! -//! [rustls-acme](rustls_acme) owns the account, order, and renewal loop and -//! answers TLS-ALPN-01 challenges from its certificate resolver. The relay hands -//! that resolver to every TLS listener, so QUIC, HTTPS, and every worker serve -//! the same certificate and pick up a renewal on their next handshake. +//! Automatic certificate configuration, enabled by the default `acme` feature. -use std::{ - io, - net::SocketAddr, - path::{Path, PathBuf}, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use anyhow::Context as _; -use futures::StreamExt; -use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; -use rustls_acme::{AcmeState, EventOk}; use serde::{Deserialize, Serialize}; +use std::{path::PathBuf, time::Duration}; use url::Url; -/// How long the first issuance may take by default before startup fails. -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5 * 60); - -/// How long a validation connection may take to finish its handshake. -const CHALLENGE_TIMEOUT: Duration = Duration::from_secs(10); - -/// How long each systemd start-timeout extension lasts while issuance is pending. -#[cfg(unix)] -const EXTEND_USEC: u32 = 90_000_000; - -/// How often the start timeout is extended, well inside [`EXTEND_USEC`]. -const EXTEND_INTERVAL: Duration = Duration::from_secs(30); - -/// Challenge handshakes in flight at once; connections past this are closed unanswered, -/// so a flood cannot exhaust descriptors. Far above the handful of vantage points a -/// directory validates from, so idle scanners do not crowd the validator out. -const CHALLENGE_CONCURRENCY: usize = 1024; - -/// Failed validations the first issuance tolerates within [`FAILURE_WINDOW`] before -/// refusing to start, leaving headroom under Let's Encrypt's limit of 5 per hostname per hour. -const FAILURE_LIMIT: usize = 3; - -/// How long a failed validation counts against [`FAILURE_LIMIT`], matching Let's Encrypt's window. -const FAILURE_WINDOW: Duration = Duration::from_secs(60 * 60); +#[cfg(feature = "acme")] +mod client; +#[cfg(feature = "acme")] +pub(crate) use client::Acme; /// Automatic certificates from an ACME directory, enabled by `acme.domains`. #[serde_with::serde_as] @@ -79,10 +43,15 @@ pub struct Config { #[serde_as(as = "serde_with::OneOrMany<_>")] pub contact: Vec, - /// Directory that keeps the account key and certificate across restarts, made owner-only. - #[usage(long = "acme-dir", name = "acme-dir", env = "MOQ_ACME_DIR", setting = "acme.dir")] + /// Cache directory required when domains are configured; keeps the account key and certificate private. + #[usage( + long = "acme-cache", + name = "acme-cache", + env = "MOQ_ACME_CACHE", + setting = "acme.cache" + )] #[usage(value_hint = usage_rs::ValueHint::DirPath)] - pub dir: Option, + pub cache: Option, /// The ACME directory URL, Let's Encrypt production when unset. #[usage( @@ -101,6 +70,12 @@ pub struct Config { #[usage(long = "acme-timeout", env = "MOQ_ACME_TIMEOUT", setting = "acme.timeout")] #[serde(default, rename = "__cli_timeout", skip_serializing_if = "Option::is_none")] timeout_arg: Option, + #[usage(long = "acme-dir", env = "MOQ_ACME_DIR", hide = true)] + #[serde(skip)] + legacy_dir: Option, + #[usage(skip)] + #[serde(rename = "dir", skip_serializing)] + removed_dir: Option, } impl Config { @@ -108,366 +83,27 @@ impl Config { self.timeout_arg.map(Into::into).or(self.timeout) } - /// Validate against the listeners it must serve through and start the client, or `None` when unconfigured. - pub(crate) fn init( - &self, - listen: &moq_tokio::tls::Listen, - https: &crate::web::Https, - ) -> anyhow::Result> { - if self.domains.is_empty() { - anyhow::ensure!( - self.contact.is_empty() && self.dir.is_none() && self.directory.is_none() && self.timeout().is_none(), - "acme settings require acme.domains" - ); - return Ok(None); - } - if cfg!(not(unix)) { - anyhow::bail!("acme is only supported on unix, where acme.dir can be kept owner-only"); - } - // One canonical set, so reordering or recasing the same names reuses the cached certificate. - let mut domains: Vec = self.domains.iter().map(|domain| domain.to_ascii_lowercase()).collect(); - domains.sort(); - domains.dedup(); - for domain in &domains { - anyhow::ensure!( - !domain.contains('*'), - "acme.domains cannot include the wildcard {domain}: TLS-ALPN-01 cannot issue one" - ); - anyhow::ensure!( - matches!(ServerName::try_from(domain.as_str()), Ok(ServerName::DnsName(_))), - "acme.domains entry {domain:?} is not a DNS name" - ); - } - anyhow::ensure!( - listen.cert.is_empty() && listen.key.is_empty() && listen.generate.is_empty(), - "acme cannot be combined with listen.tls.cert, listen.tls.key, or listen.tls.generate" - ); - anyhow::ensure!( - https.listen.is_some(), - "acme requires web.https.listen: the directory validates each domain over TLS-ALPN-01 on port 443" - ); - anyhow::ensure!( - https.cert.is_empty() && https.key.is_empty(), - "acme cannot be combined with web.https.cert or web.https.key" - ); - let dir = self - .dir - .clone() - .context("acme requires acme.dir to keep the account and certificate across restarts")?; - create_private_dir(&dir).with_context(|| format!("failed to prepare acme.dir {}", dir.display()))?; - - let directory = self - .directory - .as_ref() - .map_or(rustls_acme::acme::LETS_ENCRYPT_PRODUCTION_DIRECTORY, Url::as_str); - let cache = Cache { dir }; - let failures = Failures { - path: cache.path("failures", &domains, directory), - }; - let state = rustls_acme::AcmeConfig::new(&domains) - .contact(self.contact.iter().map(Url::as_str)) - .directory(directory) - .cache(cache) - .state(); - - Ok(Some(Acme { - state, - failures, - domains: domains.join(", "), - timeout: self.timeout().unwrap_or(DEFAULT_TIMEOUT), - })) - } -} - -/// The ACME client: the certificate resolver plus the order and renewal loop behind it. -pub(crate) struct Acme { - state: AcmeState, - failures: Failures, - domains: String, - timeout: Duration, -} - -impl Acme { - /// The resolver every TLS listener serves from, which also answers the challenges. - pub(crate) fn resolver(&self) -> Arc { - self.state.resolver() - } - - /// Serve only the TLS-ALPN-01 challenge on `listener` until a certificate is deployed. - /// - /// A valid cached certificate deploys at once with no challenge. Otherwise this - /// orders one, failing with the last ACME error once the startup budget runs out - /// or once [`FAILURE_LIMIT`] validations failed within the hour, counting earlier - /// starts: a supervisor restarting the relay would otherwise spend the rest of - /// the directory's allowance and lock the hostname out for an hour. - pub(crate) async fn issue(&mut self, listener: &std::net::TcpListener) -> anyhow::Result<()> { - let addr = listener.local_addr()?; - let mut failed = self.failures.recent()?; - if failed.len() >= FAILURE_LIMIT { - return Err(self.failures.refuse(&failed, &self.domains, addr, None)); - } - - // A second descriptor for the same socket, so the web server keeps the - // original and takes over the queue once this returns. - let listener = tokio::net::TcpListener::from_std(listener.try_clone()?)?; - let challenge = tokio_rustls::TlsAcceptor::from(self.state.challenge_rustls_config()); - let mut handshakes = tokio::task::JoinSet::new(); - let health = moq_tokio::accept::Health::new("acme"); - // Readiness waits on issuance, so keep extending systemd's start timeout - // rather than letting the unit's default kill the relay inside the budget. - let mut extend = moq_clock::interval(EXTEND_INTERVAL); - let deadline = moq_clock::sleep(self.timeout); - tokio::pin!(deadline); - let mut last = None; - let mut issued = false; - - loop { - tokio::select! { - event = self.state.next() => match event.context("ACME client stopped")? { - Ok(EventOk::DeployedCachedCert) => { - tracing::info!(domains = %self.domains, "serving the cached ACME certificate"); - return self.failures.clear(); - } - // Not ready until it is saved: a certificate lost on restart would be ordered again. - Ok(EventOk::DeployedNewCert) => { - tracing::info!(domains = %self.domains, "issued an ACME certificate"); - issued = true; - } - Ok(EventOk::CertCacheStore) if issued => return self.failures.clear(), - Ok(event) => tracing::debug!(?event, "ACME"), - Err(rustls_acme::EventError::CertCacheStore(err)) => { - return Err(anyhow::Error::new(err).context("failed to save the issued ACME certificate in acme.dir")); - } - // The directory marked an authorization invalid: one failed validation. - Err(err @ rustls_acme::EventError::Order(rustls_acme::OrderError::BadAuth(_))) => { - failed = self.failures.record()?; - tracing::warn!(%err, domains = %self.domains, failed = failed.len(), "ACME validation failed"); - if failed.len() >= FAILURE_LIMIT { - return Err(self.failures.refuse(&failed, &self.domains, addr, Some(&err.to_string()))); - } - last = Some(err.to_string()); - } - Err(err) => { - tracing::warn!(%err, domains = %self.domains, "ACME issuance failed; retrying"); - last = Some(err.to_string()); - } - }, - accepted = listener.accept() => match accepted { - Ok((stream, _)) => { - health.accepted(); - if handshakes.len() < CHALLENGE_CONCURRENCY { - let handshake = challenge.accept(stream); - // The validator only needs the handshake, then closes. - handshakes.spawn(async move { - let _ = moq_clock::timeout(CHALLENGE_TIMEOUT, handshake).await; - }); - } - } - Err(err) => { - if let Some(pause) = health.failed(&err) { - moq_clock::sleep(pause).await; - } - } - }, - Some(_) = handshakes.join_next() => {} - _ = extend.tick() => { - #[cfg(unix)] - let _ = sd_notify::notify(&[sd_notify::NotifyState::ExtendTimeoutUsec(EXTEND_USEC)]); - } - _ = &mut deadline => { - let last = last.as_deref().unwrap_or("no response from the directory"); - anyhow::bail!( - "no ACME certificate for {} within {}: {last}; the directory validates each domain over TLS-ALPN-01 on port 443, and web.https.listen is {addr}", - self.domains, - humantime::format_duration(self.timeout), - ); - } - } - } - } - - /// Renew in the background forever, keeping the current certificate serving through failures. - /// - /// Never returns: the event stream is infinite, so its end panics, which stops - /// the relay through its critical tasks instead of silently ending renewal. - pub(crate) async fn run(mut self) { - loop { - let event = self.state.next().await.expect("the ACME event stream ended"); - match event { - Ok(EventOk::DeployedNewCert) => tracing::info!(domains = %self.domains, "renewed the ACME certificate"), - Ok(event) => tracing::debug!(?event, "ACME"), - Err(err) => tracing::error!( - %err, - domains = %self.domains, - "ACME renewal failed; still serving the previous certificate" - ), - } - } - } -} - -/// Create `dir` if needed and restrict it to the owner, since it holds private keys. -fn create_private_dir(dir: &Path) -> io::Result<()> { - std::fs::create_dir_all(dir)?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; - } - Ok(()) -} - -/// The failed validations recorded in `acme.dir`, so restarts share one allowance. -/// -/// One Unix timestamp per line, keyed like the certificate it was trying to issue. -struct Failures { - path: PathBuf, -} - -impl Failures { - /// The failures within [`FAILURE_WINDOW`], oldest first. - fn recent(&self) -> anyhow::Result> { - let Some(contents) = Cache::read(&self.path)? else { - return Ok(Vec::new()); - }; - let cutoff = SystemTime::from(moq_clock::wall()) - FAILURE_WINDOW; - let mut recent = Vec::new(); - for line in String::from_utf8_lossy(&contents).lines() { - let secs: u64 = line - .parse() - .with_context(|| format!("malformed ACME failure record in {}", self.path.display()))?; - let at = UNIX_EPOCH + Duration::from_secs(secs); - if at > cutoff { - recent.push(at); - } - } - recent.sort(); - Ok(recent) - } - - /// Record a failure now, returning the recent ones including it. - fn record(&self) -> anyhow::Result> { - let mut recent = self.recent()?; - recent.push(SystemTime::from(moq_clock::wall())); - let lines: String = recent - .iter() - .map(|at| format!("{}\n", at.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs())) - .collect(); - Cache::write(&self.path, lines.as_bytes())?; - Ok(recent) - } - - fn clear(&self) -> anyhow::Result<()> { - match std::fs::remove_file(&self.path) { - Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err.into()), - _ => Ok(()), + pub(crate) fn deprecated(&self) -> moq_tokio::cli::Deprecated { + let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy_dir.is_some() { + found.flag("--acme-dir", Some("MOQ_ACME_DIR"), "--acme-cache"); } - } - - /// The startup error once `failed` reaches [`FAILURE_LIMIT`]. - fn refuse(&self, failed: &[SystemTime], domains: &str, addr: SocketAddr, last: Option<&str>) -> anyhow::Error { - let retry = failed - .first() - .map_or(SystemTime::from(moq_clock::wall()), |oldest| *oldest + FAILURE_WINDOW); - let last = last.map(|last| format!("; last error: {last}")).unwrap_or_default(); - anyhow::anyhow!( - "ACME validation for {domains} failed {} times in the last hour; stopping before the directory's failed-validation limit locks the hostname out (Let's Encrypt allows 5 failed validations per hostname per hour). Check that each domain resolves to this relay and that TCP port 443 reaches web.https.listen ({addr}), then restart after {} or delete {}{last}", - failed.len(), - humantime::format_rfc3339_seconds(retry), - self.path.display(), - ) - } -} - -/// Keeps the account key and certificate in `dir`, one owner-only file each. -/// -/// Files are keyed by a hash of the contacts or domains plus the directory URL, so -/// a certificate missing a configured domain or issued by another directory is -/// simply not found and gets reissued. An expired one counts as missing too. -struct Cache { - dir: PathBuf, -} - -impl Cache { - fn path(&self, kind: &str, names: &[String], directory: &str) -> PathBuf { - let mut digest = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); - for name in names { - digest.update(name.as_bytes()); - digest.update(&[0]); + if self.removed_dir.is_some() { + found.toml("[acme] dir", "[acme] cache", None); } - digest.update(directory.as_bytes()); - self.dir.join(format!("{kind}-{}", hex::encode(digest.finish()))) + found } - fn read(path: &Path) -> io::Result>> { - match std::fs::read(path) { - Ok(contents) => Ok(Some(contents)), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } - - /// Replace `path` atomically, so a crash mid-write never leaves a torn key behind. - fn write(path: &Path, contents: &[u8]) -> io::Result<()> { - use std::io::Write; - - let partial = path.with_extension("partial"); - let mut options = std::fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); - let mut file = options.open(&partial)?; - file.write_all(contents)?; - file.sync_all()?; - std::fs::rename(partial, path) - } -} - -/// Whether the leaf in a cached key-and-chain PEM is valid right now. -fn current(pem: &[u8]) -> bool { - let Some(Ok(leaf)) = CertificateDer::pem_slice_iter(pem).next() else { - return false; - }; - x509_parser::parse_x509_certificate(&leaf).is_ok_and(|(_, cert)| cert.validity().is_valid()) -} - -#[async_trait::async_trait] -impl rustls_acme::CertCache for Cache { - type EC = io::Error; - - async fn load_cert(&self, domains: &[String], directory: &str) -> io::Result>> { - let cached = Self::read(&self.path("cert", domains, directory))?; - Ok(cached.filter(|pem| { - let current = current(pem); - if !current { - tracing::info!("the cached ACME certificate is expired or unreadable; reissuing"); - } - current - })) - } - - async fn store_cert(&self, domains: &[String], directory: &str, cert: &[u8]) -> io::Result<()> { - Self::write(&self.path("cert", domains, directory), cert) - } -} - -#[async_trait::async_trait] -impl rustls_acme::AccountCache for Cache { - type EA = io::Error; - - async fn load_account(&self, contact: &[String], directory: &str) -> io::Result>> { - Self::read(&self.path("account", contact, directory)) - } - - async fn store_account(&self, contact: &[String], directory: &str, account: &[u8]) -> io::Result<()> { - Self::write(&self.path("account", contact, directory), account) + #[cfg(not(feature = "acme"))] + pub(crate) fn refuse_unsupported(&self) -> anyhow::Result<()> { + anyhow::ensure!( + self.domains.is_empty() + && self.contact.is_empty() + && self.cache.is_none() + && self.directory.is_none() + && self.timeout().is_none(), + "acme settings require the acme feature" + ); + Ok(()) } } - -#[cfg(all(test, unix))] -#[expect( - clippy::disallowed_methods, - clippy::disallowed_types, - reason = "tests read the host clock directly" -)] -mod tests; diff --git a/rs/moq-relay/src/acme/client.rs b/rs/moq-relay/src/acme/client.rs new file mode 100644 index 00000000000..54991e16fa9 --- /dev/null +++ b/rs/moq-relay/src/acme/client.rs @@ -0,0 +1,413 @@ +//! Automatic certificates from an ACME directory, Let's Encrypt by default. +//! +//! [rustls-acme](rustls_acme) owns the account, order, and renewal loop and +//! answers TLS-ALPN-01 challenges from its certificate resolver. The relay hands +//! that resolver to every TLS listener, so QUIC, HTTPS, and every worker serve +//! the same certificate and pick up a renewal on their next handshake. + +use std::{ + io, + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::Context as _; +use futures::StreamExt; +use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; +use rustls_acme::{AcmeState, EventOk}; +use url::Url; + +/// How long the first issuance may take by default before startup fails. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +/// How long a validation connection may take to finish its handshake. +const CHALLENGE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long each systemd start-timeout extension lasts while issuance is pending. +#[cfg(unix)] +const EXTEND_USEC: u32 = 90_000_000; + +/// How often the start timeout is extended, well inside [`EXTEND_USEC`]. +const EXTEND_INTERVAL: Duration = Duration::from_secs(30); + +/// Challenge handshakes in flight at once; connections past this are closed unanswered, +/// so a flood cannot exhaust descriptors. Far above the handful of vantage points a +/// directory validates from, so idle scanners do not crowd the validator out. +const CHALLENGE_CONCURRENCY: usize = 1024; + +/// Failed validations the first issuance tolerates within [`FAILURE_WINDOW`] before +/// refusing to start, leaving headroom under Let's Encrypt's limit of 5 per hostname per hour. +const FAILURE_LIMIT: usize = 3; + +/// How long a failed validation counts against [`FAILURE_LIMIT`], matching Let's Encrypt's window. +const FAILURE_WINDOW: Duration = Duration::from_secs(60 * 60); + +use super::Config; + +impl Config { + /// Validate against the listeners it must serve through and start the client, or `None` when unconfigured. + pub(crate) fn init( + &self, + listen: &moq_tokio::tls::Listen, + https: &crate::web::Https, + ) -> anyhow::Result> { + if self.domains.is_empty() { + anyhow::ensure!( + self.contact.is_empty() && self.cache.is_none() && self.directory.is_none() && self.timeout().is_none(), + "acme settings require acme.domains" + ); + return Ok(None); + } + if cfg!(not(unix)) { + anyhow::bail!("acme is only supported on unix, where acme.cache can be kept owner-only"); + } + // One canonical set, so reordering or recasing the same names reuses the cached certificate. + let mut domains: Vec = self.domains.iter().map(|domain| domain.to_ascii_lowercase()).collect(); + domains.sort(); + domains.dedup(); + for domain in &domains { + anyhow::ensure!( + !domain.contains('*'), + "acme.domains cannot include the wildcard {domain}: TLS-ALPN-01 cannot issue one" + ); + anyhow::ensure!( + matches!(ServerName::try_from(domain.as_str()), Ok(ServerName::DnsName(_))), + "acme.domains entry {domain:?} is not a DNS name" + ); + } + anyhow::ensure!( + listen.cert.is_empty() && listen.key.is_empty() && listen.generate.is_empty(), + "acme cannot be combined with listen.tls.cert, listen.tls.key, or listen.tls.generate" + ); + anyhow::ensure!( + https.listen.is_some(), + "acme requires web.https.listen: the directory validates each domain over TLS-ALPN-01 on port 443" + ); + anyhow::ensure!( + https.cert.is_empty() && https.key.is_empty(), + "acme cannot be combined with web.https.cert or web.https.key" + ); + let dir = self + .cache + .clone() + .context("acme requires acme.cache to keep the account and certificate across restarts")?; + create_private_dir(&dir).with_context(|| format!("failed to prepare acme.cache {}", dir.display()))?; + + let directory = self + .directory + .as_ref() + .map_or(rustls_acme::acme::LETS_ENCRYPT_PRODUCTION_DIRECTORY, Url::as_str); + let cache = Cache { dir }; + let failures = Failures { + path: cache.path("failures", &domains, directory), + }; + let state = rustls_acme::AcmeConfig::new(&domains) + .contact(self.contact.iter().map(Url::as_str)) + .directory(directory) + .cache(cache) + .state(); + + Ok(Some(Acme { + state, + failures, + domains: domains.join(", "), + timeout: self.timeout().unwrap_or(DEFAULT_TIMEOUT), + })) + } +} + +/// The ACME client: the certificate resolver plus the order and renewal loop behind it. +pub(crate) struct Acme { + state: AcmeState, + failures: Failures, + domains: String, + timeout: Duration, +} + +impl Acme { + /// The resolver every TLS listener serves from, which also answers the challenges. + pub(crate) fn resolver(&self) -> Arc { + self.state.resolver() + } + + /// Serve only the TLS-ALPN-01 challenge on `listener` until a certificate is deployed. + /// + /// A valid cached certificate deploys at once with no challenge. Otherwise this + /// orders one, failing with the last ACME error once the startup budget runs out + /// or once [`FAILURE_LIMIT`] validations failed within the hour, counting earlier + /// starts: a supervisor restarting the relay would otherwise spend the rest of + /// the directory's allowance and lock the hostname out for an hour. + pub(crate) async fn issue(&mut self, listener: &std::net::TcpListener) -> anyhow::Result<()> { + let addr = listener.local_addr()?; + let mut failed = self.failures.recent()?; + if failed.len() >= FAILURE_LIMIT { + return Err(self.failures.refuse(&failed, &self.domains, addr, None)); + } + + // A second descriptor for the same socket, so the web server keeps the + // original and takes over the queue once this returns. + let listener = tokio::net::TcpListener::from_std(listener.try_clone()?)?; + let challenge = tokio_rustls::TlsAcceptor::from(self.state.challenge_rustls_config()); + let mut handshakes = tokio::task::JoinSet::new(); + let health = moq_tokio::accept::Health::new("acme"); + // Readiness waits on issuance, so keep extending systemd's start timeout + // rather than letting the unit's default kill the relay inside the budget. + let mut extend = moq_clock::interval(EXTEND_INTERVAL); + let deadline = moq_clock::sleep(self.timeout); + tokio::pin!(deadline); + let mut last = None; + let mut issued = false; + + loop { + tokio::select! { + event = self.state.next() => match event.context("ACME client stopped")? { + Ok(EventOk::DeployedCachedCert) => { + tracing::info!(domains = %self.domains, "serving the cached ACME certificate"); + return self.failures.clear(); + } + // Not ready until it is saved: a certificate lost on restart would be ordered again. + Ok(EventOk::DeployedNewCert) => { + tracing::info!(domains = %self.domains, "issued an ACME certificate"); + issued = true; + } + Ok(EventOk::CertCacheStore) if issued => return self.failures.clear(), + Ok(event) => tracing::debug!(?event, "ACME"), + Err(rustls_acme::EventError::CertCacheStore(err)) => { + return Err(anyhow::Error::new(err).context("failed to save the issued ACME certificate in acme.cache")); + } + // The directory marked an authorization invalid: one failed validation. + Err(err @ rustls_acme::EventError::Order(rustls_acme::OrderError::BadAuth(_))) => { + failed = self.failures.record()?; + tracing::warn!(%err, domains = %self.domains, failed = failed.len(), "ACME validation failed"); + if failed.len() >= FAILURE_LIMIT { + return Err(self.failures.refuse(&failed, &self.domains, addr, Some(&err.to_string()))); + } + last = Some(err.to_string()); + } + Err(err) => { + tracing::warn!(%err, domains = %self.domains, "ACME issuance failed; retrying"); + last = Some(err.to_string()); + } + }, + accepted = listener.accept() => match accepted { + Ok((stream, _)) => { + health.accepted(); + if handshakes.len() < CHALLENGE_CONCURRENCY { + let handshake = challenge.accept(stream); + // The validator only needs the handshake, then closes. + handshakes.spawn(async move { + let _ = moq_clock::timeout(CHALLENGE_TIMEOUT, handshake).await; + }); + } + } + Err(err) => { + if let Some(pause) = health.failed(&err) { + moq_clock::sleep(pause).await; + } + } + }, + Some(_) = handshakes.join_next() => {} + _ = extend.tick() => { + #[cfg(unix)] + let _ = sd_notify::notify(&[sd_notify::NotifyState::ExtendTimeoutUsec(EXTEND_USEC)]); + } + _ = &mut deadline => { + let last = last.as_deref().unwrap_or("no response from the directory"); + anyhow::bail!( + "no ACME certificate for {} within {}: {last}; the directory validates each domain over TLS-ALPN-01 on port 443, and web.https.listen is {addr}", + self.domains, + humantime::format_duration(self.timeout), + ); + } + } + } + } + + /// Renew in the background forever, keeping the current certificate serving through failures. + /// + /// Never returns: the event stream is infinite, so its end panics, which stops + /// the relay through its critical tasks instead of silently ending renewal. + pub(crate) async fn run(mut self) { + loop { + let event = self.state.next().await.expect("the ACME event stream ended"); + match event { + Ok(EventOk::DeployedNewCert) => tracing::info!(domains = %self.domains, "renewed the ACME certificate"), + Ok(event) => tracing::debug!(?event, "ACME"), + Err(err) => tracing::error!( + %err, + domains = %self.domains, + "ACME renewal failed; still serving the previous certificate" + ), + } + } + } +} + +/// Create `dir` if needed and restrict it to the owner, since it holds private keys. +fn create_private_dir(dir: &Path) -> io::Result<()> { + std::fs::create_dir_all(dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +/// The failed validations recorded in `acme.cache`, so restarts share one allowance. +/// +/// One Unix timestamp per line, keyed like the certificate it was trying to issue. +struct Failures { + path: PathBuf, +} + +impl Failures { + /// The failures within [`FAILURE_WINDOW`], oldest first. + fn recent(&self) -> anyhow::Result> { + let Some(contents) = Cache::read(&self.path)? else { + return Ok(Vec::new()); + }; + let cutoff = SystemTime::from(moq_clock::wall()) - FAILURE_WINDOW; + let mut recent = Vec::new(); + for line in String::from_utf8_lossy(&contents).lines() { + let secs: u64 = line + .parse() + .with_context(|| format!("malformed ACME failure record in {}", self.path.display()))?; + let at = UNIX_EPOCH + Duration::from_secs(secs); + if at > cutoff { + recent.push(at); + } + } + recent.sort(); + Ok(recent) + } + + /// Record a failure now, returning the recent ones including it. + fn record(&self) -> anyhow::Result> { + let mut recent = self.recent()?; + recent.push(SystemTime::from(moq_clock::wall())); + let lines: String = recent + .iter() + .map(|at| format!("{}\n", at.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs())) + .collect(); + Cache::write(&self.path, lines.as_bytes())?; + Ok(recent) + } + + fn clear(&self) -> anyhow::Result<()> { + match std::fs::remove_file(&self.path) { + Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err.into()), + _ => Ok(()), + } + } + + /// The startup error once `failed` reaches [`FAILURE_LIMIT`]. + fn refuse(&self, failed: &[SystemTime], domains: &str, addr: SocketAddr, last: Option<&str>) -> anyhow::Error { + let retry = failed + .first() + .map_or(SystemTime::from(moq_clock::wall()), |oldest| *oldest + FAILURE_WINDOW); + let last = last.map(|last| format!("; last error: {last}")).unwrap_or_default(); + anyhow::anyhow!( + "ACME validation for {domains} failed {} times in the last hour; stopping before the directory's failed-validation limit locks the hostname out (Let's Encrypt allows 5 failed validations per hostname per hour). Check that each domain resolves to this relay and that TCP port 443 reaches web.https.listen ({addr}), then restart after {} or delete {}{last}", + failed.len(), + humantime::format_rfc3339_seconds(retry), + self.path.display(), + ) + } +} + +/// Keeps the account key and certificate in `dir`, one owner-only file each. +/// +/// Files are keyed by a hash of the contacts or domains plus the directory URL, so +/// a certificate missing a configured domain or issued by another directory is +/// simply not found and gets reissued. An expired one counts as missing too. +struct Cache { + dir: PathBuf, +} + +impl Cache { + fn path(&self, kind: &str, names: &[String], directory: &str) -> PathBuf { + let mut digest = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); + for name in names { + digest.update(name.as_bytes()); + digest.update(&[0]); + } + digest.update(directory.as_bytes()); + self.dir.join(format!("{kind}-{}", hex::encode(digest.finish()))) + } + + fn read(path: &Path) -> io::Result>> { + match std::fs::read(path) { + Ok(contents) => Ok(Some(contents)), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + /// Replace `path` atomically, so a crash mid-write never leaves a torn key behind. + fn write(path: &Path, contents: &[u8]) -> io::Result<()> { + use std::io::Write; + + let partial = path.with_extension("partial"); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(&partial)?; + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(partial, path) + } +} + +/// Whether the leaf in a cached key-and-chain PEM is valid right now. +fn current(pem: &[u8]) -> bool { + let Some(Ok(leaf)) = CertificateDer::pem_slice_iter(pem).next() else { + return false; + }; + x509_parser::parse_x509_certificate(&leaf).is_ok_and(|(_, cert)| cert.validity().is_valid()) +} + +#[async_trait::async_trait] +impl rustls_acme::CertCache for Cache { + type EC = io::Error; + + async fn load_cert(&self, domains: &[String], directory: &str) -> io::Result>> { + let cached = Self::read(&self.path("cert", domains, directory))?; + Ok(cached.filter(|pem| { + let current = current(pem); + if !current { + tracing::info!("the cached ACME certificate is expired or unreadable; reissuing"); + } + current + })) + } + + async fn store_cert(&self, domains: &[String], directory: &str, cert: &[u8]) -> io::Result<()> { + Self::write(&self.path("cert", domains, directory), cert) + } +} + +#[async_trait::async_trait] +impl rustls_acme::AccountCache for Cache { + type EA = io::Error; + + async fn load_account(&self, contact: &[String], directory: &str) -> io::Result>> { + Self::read(&self.path("account", contact, directory)) + } + + async fn store_account(&self, contact: &[String], directory: &str, account: &[u8]) -> io::Result<()> { + Self::write(&self.path("account", contact, directory), account) + } +} + +#[cfg(all(test, unix))] +#[expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "tests read the host clock directly" +)] +#[path = "tests.rs"] +mod tests; diff --git a/rs/moq-relay/src/acme/tests.rs b/rs/moq-relay/src/acme/tests.rs index 4358c8aaab0..8e4b647a47e 100644 --- a/rs/moq-relay/src/acme/tests.rs +++ b/rs/moq-relay/src/acme/tests.rs @@ -18,6 +18,7 @@ use axum::{ response::{IntoResponse, Response}, routing::{get, post}, }; +use serde::Deserialize; use serde_json::{Value, json}; use super::*; @@ -365,7 +366,7 @@ fn relay_config(directory: &Url, dir: &Path, domains: &[&str]) -> RelayConfig { config.web.https.listen = Some("127.0.0.1:0".parse().unwrap()); config.auth.public = vec![moq_auth::Pattern::all()]; config.acme.domains = domains.iter().map(|domain| domain.to_string()).collect(); - config.acme.dir = Some(dir.to_path_buf()); + config.acme.cache = Some(dir.to_path_buf()); config.acme.directory = Some(directory.clone()); config } @@ -715,7 +716,7 @@ fn config_refuses_what_acme_cannot_serve() { let mut config = RelayConfig::default(); config.web.https.listen = Some("127.0.0.1:0".parse().unwrap()); config.acme.domains = vec![DOMAIN.into()]; - config.acme.dir = Some(dir.path().to_path_buf()); + config.acme.cache = Some(dir.path().to_path_buf()); config }; let init = |config: RelayConfig| { @@ -738,7 +739,7 @@ fn config_refuses_what_acme_cannot_serve() { config.web.https.cert = vec!["cert.pem".into()] }), ("wildcard", |config| config.acme.domains = vec!["*.relay.test".into()]), - ("acme.dir", |config| config.acme.dir = None), + ("acme.cache", |config| config.acme.cache = None), ("require acme.domains", |config| config.acme.domains.clear()), ]; for (expected, mutate) in cases { diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index 91fe9e66069..3801095c5f9 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -24,7 +24,7 @@ const ATTACH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); /// Where every session's grant comes from. Exactly one of `url` and the public /// patterns is set; [`validate`](Self::validate) refuses anything else. #[serde_as] -#[derive(usage_rs::Args, Clone, Debug, Serialize, Deserialize)] +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default)] #[non_exhaustive] @@ -72,17 +72,9 @@ pub struct Config { #[serde_as(as = "OneOrMany<_>")] pub public_publish: Vec, - /// Maximum in-band tokens one session has the auth server verifying at once, - /// counting checks still running past the admission timeout. Later tokens wait - /// for a slot. Also the requests one HTTP/2 connection carries at once, each - /// admitted on its own. Defaults to 128. - #[usage( - long = "auth-verifications", - env = "MOQ_AUTH_VERIFICATIONS", - default = "128", - setting = "auth.verifications" - )] - pub verifications: usize, + #[usage(skip)] + #[serde(rename = "verifications", skip_serializing)] + pub(crate) removed_verifications: Option, /// The 0.14 flags and env vars #3688 removed, kept parsing but hidden. Never /// read as settings: [`deprecated`](Self::deprecated) names what replaced each. @@ -112,26 +104,8 @@ pub struct Config { pub(crate) tls: Option, } -impl Default for Config { - fn default() -> Self { - Self { - url: None, - public: Vec::new(), - public_subscribe: Vec::new(), - public_publish: Vec::new(), - verifications: VERIFICATIONS, - legacy: Legacy::default(), - key: None, - key_dir: None, - auth_api: None, - domains: None, - mtls_tier: None, - tls: None, - } - } -} - -/// The default for [`Config::verifications`]. +/// Maximum in-band token verifications one session, or requests one HTTP/2 +/// connection, may hold at its decider. const VERIFICATIONS: usize = 128; /// The `--auth-*` flags 0.14 had and #3688 removed, with their env vars: a relay @@ -139,6 +113,9 @@ const VERIFICATIONS: usize = 128; #[derive(Clone, Debug, Default, usage_rs::Args)] #[usage(unknown_flags = "error", args_override_self = false)] pub(crate) struct Legacy { + #[usage(long = "auth-verifications", env = "MOQ_AUTH_VERIFICATIONS", hide = true)] + verifications: Option, + #[usage(name = "auth-key", long = "auth-key", env = "MOQ_AUTH_KEY", hide = true)] key: Option, #[usage(name = "auth-key-dir", long = "auth-key-dir", env = "MOQ_AUTH_KEY_DIR", hide = true)] @@ -205,6 +182,22 @@ impl Config { /// ignored. pub fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy.verifications.is_some() { + found.changed( + "--auth-verifications", + Some("MOQ_AUTH_VERIFICATIONS"), + "the fixed per-session verification policy", + "the configurable verification window is removed", + ); + } + if self.removed_verifications.is_some() { + found.toml( + "[auth] verifications", + "the fixed per-session verification policy", + None, + ); + } + let legacy = &self.legacy; for (set, flag, env, toml, new) in [ ( @@ -286,6 +279,8 @@ impl Config { /// prefix `anon/`, and a pattern reads it as exactly the broadcast `anon`, so /// either silent reading would mislead someone upgrading. pub fn validate(&self, client_ca: bool) -> anyhow::Result<()> { + let deprecated = self.deprecated(); + anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); let flags = [ ("--auth-public", &self.public), ("--auth-public-publish", &self.public_publish), @@ -312,9 +307,6 @@ impl Config { ), _ => {} } - if self.verifications == 0 { - anyhow::bail!("--auth-verifications must be at least 1, or no in-band token is ever verified"); - } Ok(()) } @@ -341,8 +333,7 @@ impl Config { ), (None, None) => unreachable!("validated above"), }; - let (auth, admissions) = Auth::embedded(node); - let mut auth = auth.with_verifications(self.verifications); + let (mut auth, admissions) = Auth::embedded(node); auth.in_band = matches!(decider, Decider::Server(_)); decider.spawn(admissions); Ok(auth) @@ -759,7 +750,8 @@ pub(crate) fn error(reason: &lease::Reason) -> moq_net::Error { /// Work is dropped when the lease expires, narrows, or is revoked. A completed /// work future ends the lease with zero byte totals. A gateway that tracks /// transport totals can use [`Lease::ended`] and [`Lease::close`] directly. -pub async fn hold(mut lease: Lease, work: impl std::future::Future) -> Result { +#[cfg(test)] +pub(crate) async fn hold(mut lease: Lease, work: impl std::future::Future) -> Result { tokio::select! { biased; reason = lease.ended() => { @@ -852,6 +844,7 @@ pub struct Auth { /// Whether the decider can verify a token presented in band; public rules cannot. in_band: bool, /// How many in-band tokens one session may have the decider verifying at once. + #[cfg(test)] verifications: usize, } @@ -865,20 +858,27 @@ impl Auth { admissions: sender, node: Arc::from(node.into()), in_band: true, + #[cfg(test)] verifications: VERIFICATIONS, }; (auth, Admissions(receiver)) } - /// Verify at most `verifications` in-band tokens per session at once, as - /// `--auth-verifications` does for a configured `Auth`; later ones wait for a slot. - /// Panics on 0, which would leave every in-band token waiting forever. - pub fn with_verifications(mut self, verifications: usize) -> Self { - assert!(verifications > 0, "an Auth must verify at least one token at a time"); + #[cfg(test)] + pub(crate) fn with_verifications(mut self, verifications: usize) -> Self { + assert!(verifications > 0); self.verifications = verifications; self } + /// How many credentials one session, or one HTTP/2 connection, may have the decider verifying at once. + pub(crate) fn verifications(&self) -> usize { + #[cfg(test)] + return self.verifications; + #[cfg(not(test))] + VERIFICATIONS + } + /// An `Auth` that refuses every session a server or a public grant would have /// decided, admitting only what the relay decides for itself (a LAN peer). /// Must be called within a Tokio runtime, which drives the refusal decider. @@ -893,11 +893,6 @@ impl Auth { &self.node } - /// How many credentials one session may have the decider verifying at once. - pub(crate) fn verifications(&self) -> usize { - self.verifications - } - /// A fresh `connect` request for this relay, before the transport's facts are filled in. pub fn request(&self, transport: moq_auth::Transport, path: impl Into) -> Request { Request::new(self.node.as_ref(), transport, path) @@ -1062,6 +1057,7 @@ pub fn request_for(auth: &Auth, request: &moq_tokio::server::Request) -> Request } /// Build an auth request from the path, credential, and role carried in SETUP. +#[cfg(any(feature = "test-support", all(target_os = "linux", feature = "_uring")))] pub(crate) fn request_for_setup( auth: &Auth, setup: &moq_net::server::Handshake, @@ -1194,18 +1190,15 @@ mod tests { assert!(grant.publish.is_empty()); } - /// The verification concurrency reads from `[auth]` and refuses 0, - /// which would leave every in-band token waiting forever. + /// Removed explicit window values, including zero, are refused by name. #[test] - fn verifications_parse_and_refuse_zero() { - let config = config(Some("http://127.0.0.1:4440/"), &[]); - let parsed: Config = toml::from_str("url = \"http://127.0.0.1:4440/\"\nverifications = 4\n").unwrap(); - assert_eq!(parsed.verifications, 4); - let zero = Config { - verifications: 0, - ..config - }; - assert!(zero.validate(false).is_err()); + fn configurable_verifications_are_refused() { + for value in [0, 4] { + let parsed: Config = + toml::from_str(&format!("url = \"http://127.0.0.1:4440/\"\nverifications = {value}\n")).unwrap(); + let error = parsed.validate(false).unwrap_err(); + assert!(error.to_string().contains("verifications"), "{error}"); + } } /// Public rules grant a certificate what they grant anyone, so a client CA @@ -1580,3 +1573,7 @@ mod outcome_tests { assert_eq!(presence.sessions_ended, 1); } } + +#[cfg(test)] +#[expect(clippy::disallowed_methods, reason = "tests read the host clock directly")] +mod admission_tests; diff --git a/rs/moq-relay/tests/integration/auth_embedder.rs b/rs/moq-relay/src/auth/admission_tests.rs similarity index 98% rename from rs/moq-relay/tests/integration/auth_embedder.rs rename to rs/moq-relay/src/auth/admission_tests.rs index c3bb2bbe23d..1bec4316f4a 100644 --- a/rs/moq-relay/tests/integration/auth_embedder.rs +++ b/rs/moq-relay/src/auth/admission_tests.rs @@ -1,7 +1,7 @@ //! Admission and lease behavior for a gateway embedding the relay. +use crate::{auth, cluster}; use moq_auth::{Grant, Pattern, Patterns, Role, Transport, lease}; -use moq_relay::{auth, cluster}; fn patterns(texts: &[&str]) -> Patterns { texts.iter().map(|text| text.parse::().unwrap()).collect() @@ -184,8 +184,10 @@ async fn gateway_admission_refuses_once_in_the_shared_stats_registry() { async fn embedded_expired_refusals_keep_verified_root_and_tier() { tokio::time::pause(); let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); - let mut config = moq_relay::stats::Config::default(); - config.enabled = true; + let config = crate::stats::Config { + enabled: true, + ..Default::default() + }; let producer = config.build(cluster.origin.clone()).unwrap(); let registry = producer.registry().clone(); let cluster = cluster.with_stats(producer); diff --git a/rs/moq-relay/src/auth/refresh.rs b/rs/moq-relay/src/auth/refresh.rs index a356e9f4158..1a6b523eea2 100644 --- a/rs/moq-relay/src/auth/refresh.rs +++ b/rs/moq-relay/src/auth/refresh.rs @@ -250,7 +250,7 @@ impl Refresh { let Some(source) = self.source.as_ref() else { return; }; - while self.pending.len() + self.settling.len() < source.auth.verifications { + while self.pending.len() + self.settling.len() < source.auth.verifications() { let Some(waiting) = self.waiting.pop_front() else { return; }; diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 8d34ffd7655..3c24c5291d2 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -19,11 +19,14 @@ use url::Url; use crate::auth; +/// Dynamic cluster peer-list configuration. +pub mod api; + /// The request path prefix a LAN mesh dial presents, marking it as a peer rather /// than an ordinary publisher or viewer on the same listener. pub(crate) const CLUSTER_PATH: &str = "/.cluster"; -/// How often the relay re-checks an http(s) `--cluster-connect-api` endpoint. The +/// How often the relay re-checks an http(s) `--cluster-api-url` endpoint. The /// HTTP cache middleware suppresses the actual network round-trip while the cached /// list is still fresh (per the response's `Cache-Control`), so this is the floor /// on responsiveness, not on origin load: a tighter `max-age` means more of these @@ -31,7 +34,7 @@ pub(crate) const CLUSTER_PATH: &str = "/.cluster"; const CONNECT_API_POLL_INTERVAL: Duration = Duration::from_secs(30); /// One cluster peer to dial, as listed in [`Config::connect`] or returned -/// by a `connect_api` endpoint. +/// by an `api.url` endpoint. /// /// Accepts a URL string or an object with `url` plus /// policy: `cost` prices the link, `egress` declares this relay's own price to @@ -365,7 +368,7 @@ fn parse_peer_list(list: Vec, node: Option<&str>) -> anyhow::Result, - /// Fetch the list of peers to dial from an HTTP(S) URL or a local file, - /// reloading at runtime without a restart. The source returns a JSON array - /// of peers: bare URL strings and/or objects with `url`, `cost`, `egress`, - /// `token`, and `upstream`, exactly as [`Self::connect`] accepts, e.g. - /// `["https://a.pop.example/?cost=1", {"url": "https://b.pop.example/", "cost": 2}]`. - /// An http(s) URL is re-checked on a fixed cadence, with caching, conditional revalidation - /// (`ETag` / `Last-Modified`), and stale-if-error handled by the shared HTTP - /// cache client, so the response's `Cache-Control` controls how often a real - /// fetch hits the endpoint; a local path is watched via OS filesystem - /// notifications (with a periodic re-check fallback). This relay's own - /// [`Self::node`] value, when set, is sent as a `?node=` query param so the - /// server can return this node's peers. The relay keeps the last good list if - /// a fetch fails. Composes with [`Self::connect`]. - #[usage( - name = "cluster-connect-api", - long = "cluster-connect-api", - env = "MOQ_CLUSTER_CONNECT_API", - setting = "cluster.connect_api" - )] - pub connect_api: Option, + /// Dynamic peer-list source and its independent TLS trust. + #[usage(flatten)] + pub api: api::Config, - /// Trust these PEM roots for the peer API instead of the system roots. - /// The API hostname comes from its URL; mesh TLS trust is independent. + #[usage(long = "cluster-connect-api", env = "MOQ_CLUSTER_CONNECT_API", hide = true)] + #[serde(skip)] + pub(crate) legacy_api: Option, #[usage( - name = "cluster-connect-api-tls-root", long = "cluster-connect-api-tls-root", env = "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", - setting = "cluster.connect_api_tls_root" + hide = true )] - #[usage(value_hint = usage_rs::ValueHint::FilePath, extensions("pem", "crt", "cer"))] - #[serde_as(as = "serde_with::OneOrMany<_>")] - pub connect_api_tls_root: Vec, + #[serde(skip)] + pub(crate) legacy_api_root: Vec, + #[usage(skip)] + #[serde(rename = "connect_api", skip_serializing)] + pub(crate) removed_api: Option, + #[usage(skip)] + #[serde(rename = "connect_api_tls_root", skip_serializing)] + pub(crate) removed_api_root: Option, /// This relay's own externally-reachable URL (identity). Sent to - /// [`Self::connect_api`] as a `?node=` query param so the endpoint can return + /// [`Self::api`] as a `?node=` query param so the endpoint can return /// this node's peers, and advertised over mDNS when LAN discovery is on. On /// its own it neither opens nor accepts a connection. #[usage( @@ -847,6 +838,27 @@ impl Config { /// Released spellings this config was parsed from, each paired with what replaced it. pub fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy_api.is_some() { + found.flag( + "--cluster-connect-api", + Some("MOQ_CLUSTER_CONNECT_API"), + "--cluster-api-url", + ); + } + if !self.legacy_api_root.is_empty() { + found.flag( + "--cluster-connect-api-tls-root", + Some("MOQ_CLUSTER_CONNECT_API_TLS_ROOT"), + "--cluster-api-tls-root", + ); + } + if self.removed_api.is_some() { + found.toml("[cluster] connect_api", "[cluster.api] url", None); + } + if self.removed_api_root.is_some() { + found.toml("[cluster] connect_api_tls_root", "[cluster.api.tls] root", None); + } + if self.connect.iter().any(|peer| is_legacy_peer(peer.url())) { found.changed( "--cluster-connect", @@ -977,7 +989,7 @@ struct LanAuth { #[non_exhaustive] pub struct Options { /// Request caps for ordinary client sessions. - pub session: crate::limits::Session, + pub session: crate::limits::Client, /// Request caps and QUIC limits for authenticated peers and outbound cluster links. pub peer: crate::limits::Peer, /// Cluster identity, peers, and discovery. @@ -1128,11 +1140,6 @@ pub struct Cluster { /// view always points at the same session in the logs. connection_ids: Arc, - /// Client TLS config used to build the `--cluster-connect-api` HTTP client, so - /// peer-list fetches present the same cluster cert the QUIC dials do. `Arc` so - /// cloning a `Cluster` per connection stays cheap. - client_tls: Option>, - /// The origin driver, cache governor, and certificate reload: a panic in any /// of them stops [`Relay::run`](crate::Relay::run). pub(crate) critical: crate::critical::Critical, @@ -1158,7 +1165,7 @@ pub struct Cluster { /// A gateway or network session admitted with its lease, scoped origins, and stats. #[non_exhaustive] -pub struct Admitted { +pub(crate) struct Admitted { /// The live authorization that must be held for the session's lifetime. pub lease: auth::Lease, /// Where an admitted publisher writes its broadcasts. @@ -1223,7 +1230,7 @@ impl Cluster { let deprecated = config.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Reject a repeated static identity with conflicting policy up front, - // matching the `--cluster-connect-api` validation, instead of silently + // matching the `--cluster-api-url` validation, instead of silently // keeping the first entry when dials spawn. parse_peer_list(config.connect.clone(), None).context("invalid --cluster-connect peer list")?; peer.quic.validate()?; @@ -1262,7 +1269,6 @@ impl Cluster { nodes, refusals: Default::default(), connection_ids: Arc::default(), - client_tls: None, critical, origin, stats: moq_net::stats::Registry::disabled(), @@ -1274,7 +1280,8 @@ impl Cluster { /// /// Waits on the decider for as long as the caller does; the caller owns the /// deadline, as for [`auth::Auth::admit`]. - pub async fn admit(&self, auth: &auth::Auth, request: moq_auth::Request) -> Result { + #[cfg(test)] + pub(crate) async fn admit(&self, auth: &auth::Auth, request: moq_auth::Request) -> Result { let result = async { if !matches!(request.event, moq_auth::Event::Connect) { return Err(auth::Error::Request("admission requires a connect event".into())); @@ -1403,16 +1410,6 @@ impl Cluster { self } - /// Attach the client TLS config used for `--cluster-connect-api` peer-list - /// fetches. Only its client identity is shared; the API has separate server - /// trust and verifies the hostname in its URL. Required for an HTTP(S) API; pass the TLS config of - /// the [`with_connect`](Self::with_connect) template so the endpoint sees this - /// relay's cluster certificate. - pub fn with_client_tls(mut self, tls: rustls::ClientConfig) -> Self { - self.client_tls = Some(Arc::new(tls)); - self - } - /// Reserve the next `conn` id, the handle a session is logged under. /// /// One counter serves inbound accepts and outbound cluster dials, so an id in @@ -1605,7 +1602,7 @@ impl Cluster { } } - let can_dial = !self.config.connect.is_empty() || self.config.connect_api.is_some() || lan; + let can_dial = !self.config.connect.is_empty() || self.config.api.url.is_some() || lan; if !can_dial { tracing::info!("no cluster peers configured; running standalone"); return Ok(Started { @@ -1615,12 +1612,14 @@ impl Cluster { } // Build here so malformed API roots fail startup, before background work. - let http = if self.config.connect_api.as_deref().is_some_and(connect_api_is_http) { + let http = if self.config.api.url.as_deref().is_some_and(connect_api_is_http) { let tls = self - .client_tls + .connect .as_ref() - .context("cluster.connect_api with an http(s) URL needs client TLS (call Cluster::with_client_tls)")?; - Some(crate::http_client::build(tls, &self.config.connect_api_tls_root)?) + .context("cluster.api.url with an http(s) URL needs a dial template (call Cluster::with_connect)")? + .tls + .build()?; + Some(crate::http_client::build(&tls, &self.config.api.tls.root)?) } else { None }; @@ -1657,7 +1656,7 @@ impl Cluster { // last, so a start that fails earlier never binds a fixed `[connect]` port. // Static and API peers share this client; LAN dials each build their own, // so a LAN-only node builds one only to check the template. - let statics = !self.config.connect.is_empty() || self.config.connect_api.is_some(); + let statics = !self.config.connect.is_empty() || self.config.api.url.is_some(); let client = self .peer_client(!statics, None) .context("cluster peers configured but the dial template is unusable")?; @@ -1684,7 +1683,7 @@ impl Cluster { } /// Runs the cluster event loop against the work [`start`](Self::start) - /// resolved: dials the static peers, then keeps the `connect_api` list and + /// resolved: dials the static peers, then keeps the `api.url` list and /// LAN discovery reconciled into the same dial set. async fn run_work(self, work: Work) -> anyhow::Result<()> { let Work { @@ -1716,7 +1715,7 @@ impl Cluster { dialed.insert(target, handle, DialSource::Static); } - if let Some(source) = self.config.connect_api.clone() { + if let Some(source) = self.config.api.url.clone() { let this = self.clone(); let token = token.clone(); let dialed = dialed.clone(); @@ -1796,7 +1795,7 @@ impl Cluster { anyhow::bail!("LAN discovery stopped unexpectedly"); } - /// Drive `--cluster-connect-api`: an http(s) URL is polled, a local path (or + /// Drive `--cluster-api-url`: an http(s) URL is polled, a local path (or /// `file://` URL) is watched for changes. Either way the source yields a JSON /// array of peers that's reconciled into the shared dial map. async fn run_connect_api( @@ -1814,7 +1813,7 @@ impl Cluster { } Ok(url) if url.scheme() == "file" => match url.to_file_path() { Ok(path) => self.run_connect_api_file(path, node, token, dialed).await, - Err(()) => tracing::error!(%source, "cluster.connect_api file URL is not a valid local path"), + Err(()) => tracing::error!(%source, "cluster.api.url file URL is not a valid local path"), }, // Anything that isn't a URL we recognize is treated as a filesystem path. _ => { @@ -1853,7 +1852,7 @@ impl Cluster { match Self::fetch_peer_list(&http, req_url).await { Ok(list) => self.apply_peer_list(list, &node, &token, &dialed), - Err(err) => tracing::warn!(%err, "cluster.connect_api fetch failed; keeping current peers"), + Err(err) => tracing::warn!(%err, "cluster.api.url fetch failed; keeping current peers"), } } } @@ -1868,7 +1867,7 @@ impl Cluster { let mut watcher = match moq_tokio::watch::Files::new(std::slice::from_ref(&path)) { Ok(watcher) => watcher, Err(err) => { - tracing::error!(%err, ?path, "failed to watch cluster.connect_api file; updates disabled"); + tracing::error!(%err, ?path, "failed to watch cluster.api.url file; updates disabled"); return; } }; @@ -1888,10 +1887,10 @@ impl Cluster { Ok(body) => match serde_json::from_str::>(&body) { Ok(list) => self.apply_peer_list(list, node, token, dialed), Err(err) => { - tracing::warn!(%err, ?path, "cluster.connect_api file is not a JSON array of peers; keeping current peers") + tracing::warn!(%err, ?path, "cluster.api.url file is not a JSON array of peers; keeping current peers") } }, - Err(err) => tracing::warn!(%err, ?path, "failed to read cluster.connect_api file; keeping current peers"), + Err(err) => tracing::warn!(%err, ?path, "failed to read cluster.api.url file; keeping current peers"), } } @@ -1903,14 +1902,14 @@ impl Cluster { .get(url) .send() .await - .context("cluster.connect_api request failed")? + .context("cluster.api.url request failed")? .error_for_status() - .context("cluster.connect_api returned an error status")? + .context("cluster.api.url returned an error status")? .text() .await - .context("failed to read cluster.connect_api body")?; + .context("failed to read cluster.api.url body")?; - serde_json::from_str(&body).context("cluster.connect_api response is not a JSON array of peers") + serde_json::from_str(&body).context("cluster.api.url response is not a JSON array of peers") } /// Reconcile a freshly fetched peer list into the dial map: dial peers that @@ -1923,13 +1922,13 @@ impl Cluster { let desired = match parse_peer_list(list, node.as_deref()) { Ok(desired) => desired, Err(err) => { - tracing::warn!(%err, "invalid cluster.connect_api peer list; keeping current peers"); + tracing::warn!(%err, "invalid cluster.api.url peer list; keeping current peers"); return; } }; dialed.reconcile_api(&desired, |target| { - tracing::info!(peer = %target.key, "cluster.connect_api peer; dialing"); + tracing::info!(peer = %target.key, "cluster.api.url peer; dialing"); let handle = tokio::spawn(self.clone().supervise_remote(target, token.to_string())); handle.abort_handle() }); @@ -2229,7 +2228,7 @@ fn strip_jwt(url: &mut Url) { } } -/// Whether a `--cluster-connect-api` source is an http(s) URL (otherwise it's +/// Whether a `--cluster-api-url` source is an http(s) URL (otherwise it's /// treated as a local file path, which needs no TLS client). fn connect_api_is_http(source: &str) -> bool { Url::parse(source).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) @@ -2239,7 +2238,7 @@ fn connect_api_is_http(source: &str) -> bool { /// /// The modern form is a full URL, e.g. `https://host/?jwt=TOKEN`, which is used /// verbatim. A bare host or `host:port` is wrapped in `https://.../` for -/// `--cluster-connect-api` lists; user-supplied `--cluster-connect` entries +/// `--cluster-api-url` lists; user-supplied `--cluster-connect` entries /// refuse that form through [`Config::deprecated`]. fn peer_url(peer: &str) -> anyhow::Result { // A full URL has a scheme separator; a bare host or `host:port` does not @@ -2409,6 +2408,7 @@ mod tests { } /// The next route and whether it is active. + #[cfg(feature = "cluster-lan")] async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { match announced.next().await? { moq_net::announce::Event::Start(route) @@ -3076,7 +3076,7 @@ mod tests { } /// Static `--cluster-connect` entries get the same all-or-nothing - /// validation as `--cluster-connect-api` lists: a repeated identity with + /// validation as `--cluster-api-url` lists: a repeated identity with /// conflicting policy fails construction instead of silently keeping the /// first entry, while equivalent entries dedupe. #[tokio::test] @@ -3558,7 +3558,7 @@ mod tests { #[test] fn cluster_connect_api_survives_toml_merge() { - let toml = "[cluster]\nconnect_api = \"https://api.example.com/cluster/connect\"\nconnect_api_tls_root = [\"api.pem\"]\n"; + let toml = "[cluster.api]\nurl = \"https://api.example.com/cluster/connect\"\ntls.root = [\"api.pem\"]\n"; let dir = std::env::temp_dir().join("moq-relay-cluster-test"); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("cluster-connect-api-toml.toml"); @@ -3567,10 +3567,10 @@ mod tests { let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = RelayConfig::parse_and_merge(args).expect("config load"); assert_eq!( - config.cluster.connect_api.as_deref(), + config.cluster.api.url.as_deref(), Some("https://api.example.com/cluster/connect") ); - assert_eq!(config.cluster.connect_api_tls_root, vec![PathBuf::from("api.pem")]); + assert_eq!(config.cluster.api.tls.root, vec![PathBuf::from("api.pem")]); } /// Two in-process origins share broadcasts both ways over one @@ -3755,7 +3755,10 @@ mod tests { std::fs::write(&peers, format!(r#"["moql://127.0.0.1:{port}/"]"#)).expect("write peers"); let config = Config { - connect_api: Some(peers.display().to_string()), + api: api::Config { + url: Some(peers.display().to_string()), + ..Default::default() + }, ..Default::default() }; let mut quic = moq_tokio::quic::Config::default(); diff --git a/rs/moq-relay/src/cluster/api.rs b/rs/moq-relay/src/cluster/api.rs new file mode 100644 index 00000000000..3714285b10b --- /dev/null +++ b/rs/moq-relay/src/cluster/api.rs @@ -0,0 +1,47 @@ +//! The dynamic peer-list source and its own server trust. + +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Dynamic peer-list source, independent from mesh TLS trust. +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Config { + /// Fetch the list of peers to dial from an HTTP(S) URL or a local file, + /// reloading at runtime without a restart. The source returns a JSON array + /// of peers: bare URL strings and/or objects with `url`, `cost`, `egress`, + /// `token`, and `upstream`, exactly as [`super::Config::connect`] accepts, e.g. + /// `["https://a.pop.example/?cost=1", {"url": "https://b.pop.example/", "cost": 2}]`. + /// An http(s) URL is re-checked on a fixed cadence, with caching, conditional revalidation + /// (`ETag` / `Last-Modified`), and stale-if-error handled by the shared HTTP + /// cache client, so the response's `Cache-Control` controls how often a real + /// fetch hits the endpoint; a local path is watched via OS filesystem + /// notifications (with a periodic re-check fallback). This relay's own + /// [`super::Config::node`] value, when set, is sent as a `?node=` query param so the + /// server can return this node's peers. The relay keeps the last good list if + /// a fetch fails. Composes with [`super::Config::connect`]. + + #[usage(long = "cluster-api-url", env = "MOQ_CLUSTER_API_URL", setting = "cluster.api.url")] + pub url: Option, + /// TLS trust used only for the peer-list API. + #[usage(flatten)] + pub tls: Tls, +} + +/// Server roots for a peer-list API; its client identity comes from the dial template. +#[serde_with::serde_as] +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Tls { + /// PEM roots to use instead of system roots, verifying the API URL's hostname. + #[usage( + long = "cluster-api-tls-root", + env = "MOQ_CLUSTER_API_TLS_ROOT", + setting = "cluster.api.tls.root" + )] + #[usage(value_hint = usage_rs::ValueHint::FilePath, extensions("pem", "crt", "cer"))] + #[serde_as(as = "serde_with::OneOrMany<_>")] + pub root: Vec, +} diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index d58ee5ea68e..c49f4c68584 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -12,8 +12,8 @@ use crate::{auth, cache, cluster, internal, stats, web}; /// name, a version, the completion command) belong to whichever binary owns the /// process, and a `Cli` cannot be `#[usage(flatten)]`ed into another one. Keeping /// them off this type is what lets an embedder put the relay's whole flag surface -/// inside its own CLI -- moq.pro's `edge` does exactly that -- instead of -/// re-declaring it and drifting on every flag added here. This binary wraps it in +/// inside its own CLI instead of re-declaring it and drifting on every flag added +/// here. This binary wraps it in /// a private `Cli` that adds those program-level parts; [`spec`] exposes the /// resulting command line. #[derive(usage_rs::Args, Clone, Debug, Deserialize, Serialize)] @@ -21,13 +21,16 @@ use crate::{auth, cache, cluster, internal, stats, web}; #[serde(deny_unknown_fields, default)] #[non_exhaustive] pub struct Config { - /// Request caps for ordinary client sessions. + /// Request and transport limits grouped by client and cluster-peer role. #[usage(flatten)] - pub session: crate::limits::Session, + pub limits: crate::limits::Config, - /// Request caps for authenticated peers and outgoing cluster links. - #[usage(flatten)] - pub peer: crate::limits::Peer, + #[usage(skip)] + #[serde(rename = "session", skip_serializing)] + legacy_session: Option, + #[usage(skip)] + #[serde(rename = "peer", skip_serializing)] + legacy_peer: Option, /// The QUIC/TLS configuration for the server. #[usage(flatten)] @@ -147,8 +150,9 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { - session: Default::default(), - peer: Default::default(), + limits: Default::default(), + legacy_session: None, + legacy_peer: None, listen: Default::default(), server: None, connect: Default::default(), @@ -266,8 +270,18 @@ impl Config { anyhow::bail!("{}", answer.message()); } }; - let env = usage_rs::config::EnvLayer::from_process(); - let file_path = cli.config.file.clone(); + let mut config = cli.config; + config.merge(&cli_layer)?; + Ok(config) + } + + /// Load this relay's optional file, merge explicit CLI and environment values, and refuse obsolete settings. + /// + /// Call once after the embedding CLI's `parse_from_with_settings`; only this relay + /// fragment is replaced, so the root's CLI-only fields retain their parsed values. + /// [`Self::source`] reports where each merged setting came from. + pub fn merge(&mut self, cli: &usage_rs::config::CliLayer) -> anyhow::Result<()> { + let file_path = self.file.clone(); let file_body = file_path .as_ref() .map(|path| std::fs::read_to_string(path).map(|source| (path.clone(), source))) @@ -283,16 +297,14 @@ impl Config { path: std::path::Path::new(path), value, }); - let mut config = cli.config; - config.merge_into(&cli_layer, &env, file)?; - Ok(config) + self.merge_into(cli, &usage_rs::config::EnvLayer::from_process(), file) } /// Merge CLI, environment, and optional TOML values into this relay fragment. /// /// An embedding binary can parse its own flattened CLI once, then merge only /// its `relay` field. Other CLI-only fields remain untouched. - pub fn merge_into( + fn merge_into( &mut self, cli: &usage_rs::config::CliLayer, env: &usage_rs::config::EnvLayer, @@ -342,10 +354,18 @@ impl Config { /// replaced it. fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut deprecated = self.quic.deprecated(); + deprecated.extend(self.limits.legacy.deprecated()); + if self.legacy_session.is_some() { + deprecated.toml("[session]", "[limits.client]", None); + } + if self.legacy_peer.is_some() { + deprecated.toml("[peer]", "[limits.peer]", None); + } deprecated.extend(self.listen.deprecated()); deprecated.extend(self.connect.deprecated()); deprecated.extend(self.cluster.deprecated()); deprecated.extend(self.auth.deprecated()); + deprecated.extend(self.acme.deprecated()); if let Some(server) = &self.server { deprecated.toml("[server]", "[listen]", None); deprecated.extend(server.deprecated()); @@ -367,9 +387,11 @@ impl Config { let deprecated = self.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Here rather than at an endpoint, since io_uring workers and lazily built - // dial clients would otherwise check `[quic]` late or never. `[peer.quic]` + // dial clients would otherwise check `[quic]` late or never. `[limits.peer.quic]` // is checked by `Cluster::new`, which embedders call directly. self.quic.validate().context("invalid [quic]")?; + #[cfg(not(feature = "acme"))] + self.acme.refuse_unsupported()?; self.quic .max_streams @@ -384,6 +406,113 @@ impl Config { mod tests { use super::*; + /// Each renamed setting: its old flag, old variable, a value equal to the old default, + /// and the replacement its refusal names. + const OBSOLETE_ROLE_SOURCES: [(&str, &str, &str, &str); 6] = [ + ( + "--session-announces", + "MOQ_SESSION_ANNOUNCES", + "1000", + "--limits-client-announces", + ), + ( + "--peer-announces", + "MOQ_PEER_ANNOUNCES", + "100000", + "--limits-peer-announces", + ), + ("--acme-dir", "MOQ_ACME_DIR", "cache", "--acme-cache"), + ( + "--cluster-connect-api", + "MOQ_CLUSTER_CONNECT_API", + "peers.json", + "--cluster-api-url", + ), + ( + "--cluster-connect-api-tls-root", + "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", + "api.pem", + "--cluster-api-tls-root", + ), + ( + "--auth-verifications", + "MOQ_AUTH_VERIFICATIONS", + "128", + "fixed per-session verification policy", + ), + ]; + + /// Renames refuse an explicit variable even when its value equals the old default. + #[test] + fn role_config_refuses_obsolete_env() { + let env = OBSOLETE_ROLE_SOURCES + .iter() + .fold(test_env::Env::default(), |env, (_, var, value, _)| env.set(var, value)); + if !env.isolate() { + return; + } + let error = Config::parse_and_merge(["moq-relay"]).unwrap_err().to_string(); + for (_, var, _, replacement) in OBSOLETE_ROLE_SOURCES { + assert!(error.contains(var) && error.contains(replacement), "{var}: {error}"); + } + } + + /// Renames refuse an explicit flag or file setting even when its value equals the old default. + #[test] + fn role_config_refuses_obsolete_sources() { + if !test_env::Env::without(&OBSOLETE_ROLE_SOURCES.map(|(_, var, _, _)| var)).isolate() { + return; + } + for (flag, _, value, replacement) in OBSOLETE_ROLE_SOURCES { + let error = Config::parse_and_merge(["moq-relay", flag, value]) + .unwrap_err() + .to_string(); + assert!(error.contains(flag) && error.contains(replacement), "{error}"); + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("obsolete.toml"); + for (body, replacement) in [ + ("[session]\nannounces = 1000", "[limits.client]"), + ("[peer]\nannounces = 100000", "[limits.peer]"), + ("[acme]\ndir = 'cache'", "[acme] cache"), + ("[cluster]\nconnect_api = 'peers.json'", "[cluster.api] url"), + ("[cluster]\nconnect_api_tls_root = []", "[cluster.api.tls] root"), + ("[auth]\nverifications = 128", "fixed per-session verification policy"), + ] { + std::fs::write(&path, body).unwrap(); + let error = + Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap_err(); + assert!(error.to_string().contains(replacement), "{body}: {error}"); + } + } + + #[cfg(not(feature = "acme"))] + #[test] + fn an_acme_free_build_refuses_configured_issuance() { + let mut config = Config::default(); + config.resolve().unwrap(); + config.acme.domains.push("relay.example.com".into()); + let error = config.resolve().unwrap_err(); + assert!(error.to_string().contains("acme feature"), "{error}"); + } + + /// The moq-transport extension choices are file-only, so the loader must carry them through. + #[test] + fn loader_keeps_toml_extension_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write( + &path, + "[listen.extensions]\nauth = false\n\n[connect.extensions]\nsolicit = false\n", + ) + .unwrap(); + let config = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.into()]).unwrap(); + assert!(!config.listen.extensions.auth); + assert!(config.listen.extensions.solicit); + assert!(!config.connect.extensions.solicit); + assert!(config.connect.extensions.auth); + } + #[test] fn stun_binding_merges_file_and_cli() { if !test_env::Env::without(&["MOQ_STUN_BIND"]).isolate() { @@ -416,10 +545,10 @@ mod tests { #[test] fn session_limits_merge_without_crossing_peer_policy() { if !test_env::Env::without(&[ - "MOQ_SESSION_ANNOUNCES", - "MOQ_SESSION_REQUESTS", - "MOQ_PEER_ANNOUNCES", - "MOQ_PEER_REQUESTS", + "MOQ_LIMITS_CLIENT_ANNOUNCES", + "MOQ_LIMITS_CLIENT_REQUESTS", + "MOQ_LIMITS_PEER_ANNOUNCES", + "MOQ_LIMITS_PEER_REQUESTS", ]) .isolate() { @@ -429,35 +558,35 @@ mod tests { let path = dir.path().join("limits.toml"); std::fs::write( &path, - "[session]\nannounces = 7\nrequests = 8\n[peer]\nannounces = 90\nrequests = 91\n", + "[limits.client]\nannounces = 7\nrequests = 8\n[limits.peer]\nannounces = 90\nrequests = 91\n", ) .unwrap(); let config = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap(); - assert_eq!(config.session.resolve().announces, 7); - assert_eq!(config.session.resolve().requests, 8); - assert_eq!(config.peer.resolve().announces, 90); - assert_eq!(config.peer.resolve().requests, 91); + assert_eq!(config.limits.client.resolve().announces, 7); + assert_eq!(config.limits.client.resolve().requests, 8); + assert_eq!(config.limits.peer.resolve().announces, 90); + assert_eq!(config.limits.peer.resolve().requests, 91); let config = Config::parse_and_merge([ std::ffi::OsString::from("moq-relay"), path.into(), - "--session-announces=0".into(), - "--peer-requests=1000".into(), + "--limits-client-announces=0".into(), + "--limits-peer-requests=1000".into(), ]) .unwrap(); - assert_eq!(config.session.resolve().announces, 0); - assert_eq!(config.session.resolve().requests, 8); - assert_eq!(config.peer.resolve().announces, 90); - assert_eq!(config.peer.resolve().requests, 1000); + assert_eq!(config.limits.client.resolve().announces, 0); + assert_eq!(config.limits.client.resolve().requests, 8); + assert_eq!(config.limits.peer.resolve().announces, 90); + assert_eq!(config.limits.peer.resolve().requests, 1000); } - /// `[peer.quic]` mirrors `[quic]`'s keys without touching `[quic]` itself. + /// `[limits.peer.quic]` mirrors `[quic]`'s keys without touching `[quic]` itself. #[test] fn peer_quic_table_merges_with_flags() { if !test_env::Env::without(&[ "MOQ_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_RECEIVE_WINDOW", - "MOQ_PEER_QUIC_SEND_WINDOW", + "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", + "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", ]) .isolate() { @@ -467,30 +596,30 @@ mod tests { let path = dir.path().join("peer-quic.toml"); std::fs::write( &path, - "[quic]\nmax_streams = 4\n[peer.quic]\nmax_streams = 64\nsend_window = 1000\n", + "[quic]\nmax_streams = 4\n[limits.peer.quic]\nmax_streams = 64\nsend_window = 1000\n", ) .unwrap(); let config = Config::parse_and_merge([ std::ffi::OsString::from("moq-relay"), path.into(), - "--peer-quic-receive-window=2000".into(), + "--limits-peer-quic-receive-window=2000".into(), ]) .unwrap(); assert_eq!(config.quic.max_streams, Some(4)); - assert_eq!(config.peer.quic.max_streams.get(), 64); - assert_eq!(config.peer.quic.receive_window.get(), 2000); - assert_eq!(config.peer.quic.send_window.get(), 1000); + assert_eq!(config.limits.peer.quic.max_streams.get(), 64); + assert_eq!(config.limits.peer.quic.receive_window.get(), 2000); + assert_eq!(config.limits.peer.quic.send_window.get(), 1000); - assert!(toml::from_str::("[peer]\nmax_streams = 64\n").is_err()); + assert!(toml::from_str::("[limits.peer]\nmax_streams = 64\n").is_err()); } /// A zero peer stream limit or window would wedge the connection, so it never parses. #[test] fn zero_peer_quic_limits_are_refused() { if !test_env::Env::without(&[ - "MOQ_PEER_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_RECEIVE_WINDOW", - "MOQ_PEER_QUIC_SEND_WINDOW", + "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", + "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", ]) .isolate() { @@ -498,14 +627,14 @@ mod tests { } for key in ["max_streams", "receive_window", "send_window"] { assert!( - toml::from_str::(&format!("[peer.quic]\n{key} = 0\n")).is_err(), + toml::from_str::(&format!("[limits.peer.quic]\n{key} = 0\n")).is_err(), "{key}" ); } for flag in [ - "--peer-quic-max-streams=0", - "--peer-quic-receive-window=0", - "--peer-quic-send-window=0", + "--limits-peer-quic-max-streams=0", + "--limits-peer-quic-receive-window=0", + "--limits-peer-quic-send-window=0", ] { assert!( Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), flag.into()]).is_err(), @@ -746,26 +875,41 @@ root = ["ca.pem"] assert_eq!(config.quic.max_streams, Some(128)); } - /// Every config under `demo/relay/` still parses. - /// - /// These are the configs a reader copies, and a rename that lands in the code - /// but not in them is invisible until someone's relay refuses to boot. A move - /// across tables (`[server.quic]` to a top-level `[quic]`) is the case serde - /// aliases cannot cover, which is exactly why this reads the real files. + /// Real relay examples, packaging and test fixtures use the same loader as an embedding CLI. #[test] - fn demo_configs_parse() { - let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../demo/relay"); - let mut checked = 0; - for entry in std::fs::read_dir(&dir).expect("demo/relay") { - let path = entry.expect("dir entry").path(); - if path.extension().is_none_or(|ext| ext != "toml") { - continue; + fn repository_configs_load() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + // Only repository inputs belong here; installed clients and build output below + // test/ can contain unrelated TOML. Git's tracked list also includes new fixtures. + let tracked = std::process::Command::new("git") + .args(["ls-files", "-z", "--", "demo/relay", "packaging", "test"]) + .current_dir(&root) + .output() + .expect("list repository fixtures"); + assert!(tracked.status.success()); + let tracked = String::from_utf8(tracked.stdout).unwrap(); + for area in ["demo/relay", "packaging", "test"] { + let mut checked = 0; + for name in tracked.split('\0').filter(|name| name.starts_with(area)) { + let path = root.join(name); + if path.extension().is_none_or(|ext| ext != "toml") + || matches!( + path.file_name().and_then(|name| name.to_str()), + Some("Cargo.toml" | "rust-toolchain.toml" | ".rustfmt.toml") + ) { + continue; + } + let result = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]); + if path.file_name().unwrap() == "smoke-legacy.toml" { + let error = result.expect_err("the published legacy fixture must be refused"); + assert!(error.to_string().contains("[server]"), "{}: {error}", path.display()); + } else { + result.unwrap_or_else(|error| panic!("{}: {error}", path.display())); + } + checked += 1; } - let toml = std::fs::read_to_string(&path).expect("read config"); - toml::from_str::(&toml).unwrap_or_else(|err| panic!("{}: {err}", path.display())); - checked += 1; + assert!(checked > 0, "no relay fixtures found in {area}"); } - assert!(checked > 0, "no demo configs found in {}", dir.display()); } /// Bare defaults loaded from TOML survive when the CLI does not mention them. @@ -841,7 +985,7 @@ ws = false let dir = std::env::temp_dir().join("moq-relay-config-test"); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("runtime-toml-wins.toml"); - std::fs::write(&path, &toml).unwrap(); + std::fs::write(&path, toml.as_bytes()).unwrap(); let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = Config::parse_and_merge(args).expect("config load"); @@ -915,7 +1059,7 @@ ws = false /// This is the reason [`Config`] derives `Args` rather than `Cli`: a `Cli` /// cannot be flattened, so declaring the program-level parts on it would force /// every embedder to re-declare the relay's flags and drift on each one added - /// here. moq.pro's `edge` is the embedder this exists for. + /// here. #[test] fn the_config_can_be_embedded() { /// A stand-in for an embedder's command line: the relay's flags plus one of @@ -974,7 +1118,7 @@ ws = false "Top-level relay configuration, loadable from CLI arguments, environment variables, or a TOML file." ); // The tell that a developer-facing doc comment reached this surface. - for leak in ["[`", "moq.pro", "implementation detail"] { + for leak in ["[`", "implementation detail"] { assert!( !shown.contains(leak), "{leak:?} leaked into the help description: {shown}" diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index e6863f9d957..26107cfa340 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -30,7 +30,7 @@ impl From for StatusError { /// publish/subscribe origins, and serve the session until it closes. /// /// The request is a moq-tokio [`Request`](moq_tokio::server::Request), or an -/// [`Accepted`](crate::Accepted) session from any moq-net transport. +/// in-memory accepted session when the `test-support` feature is enabled. pub struct Connection { request: R, lifecycle: Lifecycle, diff --git a/rs/moq-relay/src/incoming.rs b/rs/moq-relay/src/incoming.rs index 6b6fe853b51..bd54b3fc010 100644 --- a/rs/moq-relay/src/incoming.rs +++ b/rs/moq-relay/src/incoming.rs @@ -5,22 +5,25 @@ use axum::http; /// A session whose SETUP awaits the relay's reply: what a [`Connection`](crate::Connection) serves. /// -/// Sealed: implemented by [`moq_tokio::server::Request`] and [`Accepted`]. +/// Sealed: native requests, plus in-memory accepted sessions with `test-support`. pub trait Incoming: Sealed {} impl Incoming for moq_tokio::server::Request {} +#[cfg(feature = "test-support")] impl Incoming for Accepted {} /// A session accepted on any moq-net transport, its SETUP awaiting the relay's reply. /// /// The auth decider sees the path, query, token, and role the SETUP carried, so the /// client must send its path in-band. +#[cfg(feature = "test-support")] pub struct Accepted { handshake: moq_net::server::Handshake, transport: moq_auth::Transport, } +#[cfg(feature = "test-support")] impl Accepted { /// Wrap `handshake`, reported to the auth decider as arriving over `transport`. pub fn new(handshake: moq_net::server::Handshake, transport: moq_auth::Transport) -> Self { @@ -147,6 +150,7 @@ impl Sealed for moq_tokio::server::Request { } } +#[cfg(feature = "test-support")] impl Sealed for Accepted { /// Polled on the connection's own task, so it needs no runtime to spawn on. type Driver = moq_net::Driver; diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index a0a3d17ffde..f9541bc7a43 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -61,6 +61,7 @@ fn configured_tier(label: Option) -> moq_net::stats::Tier { pub use config::*; pub use connection::*; pub use fetch::*; +#[cfg(feature = "test-support")] pub use incoming::{Accepted, Incoming}; pub use relay::*; #[cfg(feature = "test-support")] diff --git a/rs/moq-relay/src/limits.rs b/rs/moq-relay/src/limits.rs index dba45b090ea..15634ecc581 100644 --- a/rs/moq-relay/src/limits.rs +++ b/rs/moq-relay/src/limits.rs @@ -3,30 +3,46 @@ use anyhow::Context; use serde::{Deserialize, Serialize}; +/// Session and transport limits grouped by the authorized connection's role. +#[derive(usage_rs::Args, Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Config { + /// Session limits for ordinary clients. + #[usage(flatten)] + pub client: Client, + /// Session and QUIC limits for authenticated cluster peers. + #[usage(flatten)] + pub peer: Peer, + #[usage(flatten)] + #[serde(skip)] + pub(crate) legacy: Legacy, +} + /// Session limits for ordinary client sessions. #[derive(usage_rs::Args, Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] -pub struct Session { +pub struct Client { /// Maximum simultaneously announced broadcasts or namespaces. #[usage( - long = "session-announces", - env = "MOQ_SESSION_ANNOUNCES", + long = "limits-client-announces", + env = "MOQ_LIMITS_CLIENT_ANNOUNCES", default = "1000", - setting = "session.announces" + setting = "limits.client.announces" )] pub announces: usize, /// Maximum other simultaneously held moq-transport requests, sizing the drafts 14 to 16 request window. #[usage( - long = "session-requests", - env = "MOQ_SESSION_REQUESTS", + long = "limits-client-requests", + env = "MOQ_LIMITS_CLIENT_REQUESTS", default = "10000", - setting = "session.requests" + setting = "limits.client.requests" )] pub requests: usize, } -impl Default for Session { +impl Default for Client { fn default() -> Self { Self { announces: 1000, @@ -35,7 +51,7 @@ impl Default for Session { } } -impl Session { +impl Client { pub(crate) fn resolve(&self) -> moq_net::session::Limits { resolve(self.announces, self.requests) } @@ -48,21 +64,21 @@ impl Session { pub struct Peer { /// Maximum simultaneously announced broadcasts or namespaces. #[usage( - long = "peer-announces", - env = "MOQ_PEER_ANNOUNCES", + long = "limits-peer-announces", + env = "MOQ_LIMITS_PEER_ANNOUNCES", default = "100000", - setting = "peer.announces" + setting = "limits.peer.announces" )] pub announces: usize, /// Maximum other simultaneously held moq-transport requests, sizing the drafts 14 to 16 request window. #[usage( - long = "peer-requests", - env = "MOQ_PEER_REQUESTS", + long = "limits-peer-requests", + env = "MOQ_LIMITS_PEER_REQUESTS", default = "20000", - setting = "peer.requests" + setting = "limits.peer.requests" )] pub requests: usize, - /// QUIC limits a peer session widens to (`[peer.quic]`). + /// QUIC limits a peer session widens to (`[limits.peer.quic]`). #[usage(flatten)] pub quic: Quic, } @@ -90,26 +106,26 @@ impl Peer { pub struct Quic { /// Maximum concurrent QUIC streams the peer may open, per direction. #[usage( - long = "peer-quic-max-streams", - env = "MOQ_PEER_QUIC_MAX_STREAMS", + long = "limits-peer-quic-max-streams", + env = "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", default = "100000", - setting = "peer.quic.max_streams" + setting = "limits.peer.quic.max_streams" )] pub max_streams: std::num::NonZeroU64, /// Connection-wide QUIC receive window, in bytes. #[usage( - long = "peer-quic-receive-window", - env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", + long = "limits-peer-quic-receive-window", + env = "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", default = "671088640", - setting = "peer.quic.receive_window" + setting = "limits.peer.quic.receive_window" )] pub receive_window: std::num::NonZeroU64, /// Cap on unacknowledged outgoing QUIC data, in bytes. #[usage( - long = "peer-quic-send-window", - env = "MOQ_PEER_QUIC_SEND_WINDOW", + long = "limits-peer-quic-send-window", + env = "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", default = "671088640", - setting = "peer.quic.send_window" + setting = "limits.peer.quic.send_window" )] pub send_window: std::num::NonZeroU64, } @@ -133,7 +149,7 @@ impl Quic { pub(crate) fn validate(&self) -> anyhow::Result<()> { let mut quic = moq_tokio::quic::Config::default(); self.dial(&mut quic); - quic.validate().context("invalid [peer.quic]") + quic.validate().context("invalid [limits.peer.quic]") } /// Apply these limits to the `[quic]` section a peer link is dialed with. @@ -160,3 +176,59 @@ fn resolve(announces: usize, requests: usize) -> moq_net::session::Limits { limits.requests = requests; limits } + +#[derive(usage_rs::Args, Clone, Debug, Default)] +pub(crate) struct Legacy { + #[usage(long = "session-announces", env = "MOQ_SESSION_ANNOUNCES", hide = true)] + session_announces: Option, + #[usage(long = "peer-announces", env = "MOQ_PEER_ANNOUNCES", hide = true)] + peer_announces: Option, + #[usage(long = "peer-quic-max-streams", env = "MOQ_PEER_QUIC_MAX_STREAMS", hide = true)] + peer_quic_max_streams: Option, + #[usage(long = "peer-quic-receive-window", env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", hide = true)] + peer_quic_receive_window: Option, + #[usage(long = "peer-quic-send-window", env = "MOQ_PEER_QUIC_SEND_WINDOW", hide = true)] + peer_quic_send_window: Option, +} + +impl Legacy { + pub(crate) fn deprecated(&self) -> moq_tokio::cli::Deprecated { + let mut found = moq_tokio::cli::Deprecated::default(); + if self.session_announces.is_some() { + found.flag( + "--session-announces", + Some("MOQ_SESSION_ANNOUNCES"), + "--limits-client-announces", + ); + } + if self.peer_announces.is_some() { + found.flag( + "--peer-announces", + Some("MOQ_PEER_ANNOUNCES"), + "--limits-peer-announces", + ); + } + if self.peer_quic_max_streams.is_some() { + found.flag( + "--peer-quic-max-streams", + Some("MOQ_PEER_QUIC_MAX_STREAMS"), + "--limits-peer-quic-max-streams", + ); + } + if self.peer_quic_receive_window.is_some() { + found.flag( + "--peer-quic-receive-window", + Some("MOQ_PEER_QUIC_RECEIVE_WINDOW"), + "--limits-peer-quic-receive-window", + ); + } + if self.peer_quic_send_window.is_some() { + found.flag( + "--peer-quic-send-window", + Some("MOQ_PEER_QUIC_SEND_WINDOW"), + "--limits-peer-quic-send-window", + ); + } + found + } +} diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index af35a5fb115..25ec1304790 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -81,6 +81,7 @@ pub struct Relay { internal: internal::Internal, web: web::Web, /// The ACME client, issuing before [`Self::run`] serves and renewing after. + #[cfg(feature = "acme")] acme: Option, stun: Option, addr: Option, @@ -101,7 +102,7 @@ pub struct Relay { #[cfg(feature = "_quic")] workers: Option, /// The io_uring QUIC workers, already bound and waiting for - /// [`serve`](crate::uring::Workers::serve). `None` unless both + /// [`serve`](crate::uring::Spawned::serve). `None` unless both /// `runtime.workers` and `runtime.io_uring` are configured, in which case /// they own the QUIC listen address instead of `workers`. #[cfg(all(target_os = "linux", feature = "_uring"))] @@ -128,7 +129,9 @@ impl Relay { config.listen.extensions.ietf = true; config.connect.extensions.ietf = true; // Every TLS listener serves the ACME certificate, so install it before any is built. + #[cfg(feature = "acme")] let acme = config.acme.init(&config.listen.tls, &config.web.https)?; + #[cfg(feature = "acme")] if let Some(acme) = &acme { config.listen.tls.resolver = Some(acme.resolver()); } @@ -288,10 +291,9 @@ impl Relay { advertise = advertise.with_fingerprint(fingerprint); } let mut cluster_options = cluster::Options::new(config.cluster).with_cache(cache); - cluster_options.session = config.session; - cluster_options.peer = config.peer; + cluster_options.session = config.limits.client; + cluster_options.peer = config.limits.peer; let cluster = cluster::Cluster::new(cluster_options)? - .with_client_tls(config.connect.tls.build()?) .with_connect(config.connect.clone(), config.quic.clone()) .with_advertise(advertise); #[cfg(feature = "iroh")] @@ -307,13 +309,14 @@ impl Relay { let sessions = crate::session::Registry::new(); let (ready, _) = tokio::sync::watch::channel(false); let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web) - .with_acme(acme.as_ref().map(crate::acme::Acme::resolver)) .with_shutdown(shutdown.clone()) .with_versions(server_versions) .with_extensions(server_extensions) .with_timeout(handshake_timeout) - .with_sessions(sessions.clone()) - .bind()?; + .with_sessions(sessions.clone()); + #[cfg(feature = "acme")] + let web = web.with_acme(acme.as_ref().map(crate::acme::Acme::resolver)); + let web = web.bind()?; // `bind`, not `listen`: the TCP/Unix accept loops handshake as soon as // they run, and `load` is not yet willing to take a session. `run` // starts them on its first accept. @@ -361,6 +364,7 @@ impl Relay { stats, internal, web, + #[cfg(feature = "acme")] acme, stun, addr, @@ -537,6 +541,7 @@ impl Relay { cluster, internal, web, + #[cfg(feature = "acme")] acme, stun, shutdown, @@ -566,6 +571,7 @@ impl Relay { // Nothing else serves or reports ready until the certificate exists: a // self-signed placeholder would only fail every browser that connects. + #[cfg(feature = "acme")] if let Some(mut acme) = acme { let listener = web.https_listener().context("ACME needs the HTTPS listener")?; // Issuance can take minutes, and as a container's PID 1 the relay @@ -593,13 +599,13 @@ impl Relay { // identity rustls will not load fails here, and reporting ready first would // release the units depending on a relay that is about to exit. #[cfg(all(target_os = "linux", feature = "_uring"))] - let mut uring = uring; - #[cfg(all(target_os = "linux", feature = "_uring"))] - if let Some(uring) = uring.as_mut() { - uring - .spawn(cluster.clone(), auth.clone(), shutdown.clone(), sessions.clone()) - .context("failed to start the io_uring QUIC workers")?; - } + let mut uring = uring + .map(|uring| { + uring + .spawn(cluster.clone(), auth.clone(), shutdown.clone(), sessions.clone()) + .context("failed to start the io_uring QUIC workers") + }) + .transpose()?; // Without ACME, after the last step that can fail, since a handler outlives a // failed `run`, and before anything accepts a session a signal would kill diff --git a/rs/moq-relay/src/settings.rs b/rs/moq-relay/src/settings.rs index 232dadb2a60..b7b224b16b7 100644 --- a/rs/moq-relay/src/settings.rs +++ b/rs/moq-relay/src/settings.rs @@ -13,9 +13,7 @@ #[derive(usage_rs::Config)] pub struct Settings { #[usage(flatten)] - session: Session, - #[usage(flatten)] - peer: Peer, + limits: Limits, #[usage(flatten)] listen: moq_tokio::settings::Listen, @@ -88,11 +86,8 @@ struct Cluster { #[usage(env = "MOQ_CLUSTER_CONNECT", cli("--cluster-connect"), parse = "list_by_comma")] connect: Option>, - #[usage(env = "MOQ_CLUSTER_CONNECT_API", cli("--cluster-connect-api"))] - connect_api: Option, - - #[usage(env = "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", cli("--cluster-connect-api-tls-root"))] - connect_api_tls_root: Option>, + #[usage(flatten)] + api: ClusterApi, #[usage(env = "MOQ_CLUSTER_NODE", cli("--cluster-node"))] node: Option, @@ -108,6 +103,22 @@ struct Cluster { lan: ClusterLan, } +#[derive(usage_rs::Config)] +#[usage(prefix = "cluster.api")] +struct ClusterApi { + #[usage(env = "MOQ_CLUSTER_API_URL", cli("--cluster-api-url"))] + url: Option, + #[usage(flatten)] + tls: ClusterApiTls, +} + +#[derive(usage_rs::Config)] +#[usage(prefix = "cluster.api.tls")] +struct ClusterApiTls { + #[usage(env = "MOQ_CLUSTER_API_TLS_ROOT", cli("--cluster-api-tls-root"))] + root: Option>, +} + #[cfg(feature = "cluster-lan")] #[derive(usage_rs::Config)] #[usage(prefix = "cluster.lan")] @@ -144,9 +155,6 @@ struct Auth { parse = "list_by_comma" )] public_publish: Option>, - - #[usage(env = "MOQ_AUTH_VERIFICATIONS", cli("--auth-verifications"))] - verifications: Option, } #[derive(usage_rs::Config)] @@ -194,8 +202,8 @@ struct Acme { #[usage(env = "MOQ_ACME_CONTACT", cli("--acme-contact"), parse = "list_by_comma")] contact: Option>, - #[usage(env = "MOQ_ACME_DIR", cli("--acme-dir"))] - dir: Option, + #[usage(env = "MOQ_ACME_CACHE", cli("--acme-cache"))] + cache: Option, #[usage(env = "MOQ_ACME_DIRECTORY", cli("--acme-directory"))] directory: Option, @@ -247,33 +255,44 @@ struct Internal { } #[derive(usage_rs::Config)] -#[usage(prefix = "session")] -struct Session { - #[usage(env = "MOQ_SESSION_ANNOUNCES", cli("--session-announces"))] +struct Limits { + #[usage(flatten)] + client: Client, + #[usage(flatten)] + peer: Peer, +} + +#[derive(usage_rs::Config)] +#[usage(prefix = "limits.client")] +struct Client { + #[usage(env = "MOQ_LIMITS_CLIENT_ANNOUNCES", cli("--limits-client-announces"))] announces: Option, - #[usage(env = "MOQ_SESSION_REQUESTS", cli("--session-requests"))] + #[usage(env = "MOQ_LIMITS_CLIENT_REQUESTS", cli("--limits-client-requests"))] requests: Option, } #[derive(usage_rs::Config)] -#[usage(prefix = "peer")] +#[usage(prefix = "limits.peer")] struct Peer { - #[usage(env = "MOQ_PEER_ANNOUNCES", cli("--peer-announces"))] + #[usage(env = "MOQ_LIMITS_PEER_ANNOUNCES", cli("--limits-peer-announces"))] announces: Option, - #[usage(env = "MOQ_PEER_REQUESTS", cli("--peer-requests"))] + #[usage(env = "MOQ_LIMITS_PEER_REQUESTS", cli("--limits-peer-requests"))] requests: Option, #[usage(flatten)] quic: PeerQuic, } #[derive(usage_rs::Config)] -#[usage(prefix = "peer.quic")] +#[usage(prefix = "limits.peer.quic")] struct PeerQuic { - #[usage(env = "MOQ_PEER_QUIC_MAX_STREAMS", cli("--peer-quic-max-streams"))] + #[usage(env = "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", cli("--limits-peer-quic-max-streams"))] max_streams: Option, - #[usage(env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", cli("--peer-quic-receive-window"))] + #[usage( + env = "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + cli("--limits-peer-quic-receive-window") + )] receive_window: Option, - #[usage(env = "MOQ_PEER_QUIC_SEND_WINDOW", cli("--peer-quic-send-window"))] + #[usage(env = "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", cli("--limits-peer-quic-send-window"))] send_window: Option, } diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index db9ef8480d0..01cc99297d7 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -87,7 +87,13 @@ struct Serve { /// [`bind`](Self::bind) opens every socket, so a returned value already owns /// the listen address and a port conflict is an error here rather than a /// worker that quietly died. [`spawn`](Self::spawn) starts every thread, and -/// nothing is served until [`serve`](Self::serve). +/// nothing is served until [`Spawned::serve`]. +/// +/// ```compile_fail +/// fn serve_before_spawn(mut bound: moq_relay::uring::Workers) { +/// bound.serve(); +/// } +/// ``` pub struct Workers { members: Vec, addr: SocketAddr, @@ -111,7 +117,7 @@ pub struct Workers { stops: Vec>, /// Releases each spawned worker to accept; dropped unsent, it exits instead. starts: Vec>, - /// A worker's fatal error, reported to whoever polls [`failed`](Self::failed). + /// A worker's fatal error, reported to whoever polls [`Spawned::failed`]. failures: tokio::sync::mpsc::UnboundedReceiver, failures_tx: tokio::sync::mpsc::UnboundedSender, @@ -280,14 +286,14 @@ impl Workers { /// authentication and session supervision. Returns once every worker is /// set up; a worker that cannot start (an old kernel, a ring failure) is /// an error here rather than a thread that quietly died. Nothing accepts - /// until [`serve`](Self::serve), so a failure here has served nobody. + /// until [`Spawned::serve`], so a failure here has served nobody. pub fn spawn( - &mut self, + mut self, cluster: cluster::Cluster, auth: auth::Auth, shutdown: shutdown::Observer, sessions: crate::session::Registry, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { let serve = Serve { cluster, auth, @@ -349,11 +355,11 @@ impl Workers { self.starts.push(start_tx); } - Ok(()) + Ok(Spawned { workers: self }) } - /// Start every [`spawn`](Self::spawn)ed worker accepting. - pub fn serve(&mut self) { + /// Start every [`Workers::spawn`]ed worker accepting. + fn serve(&mut self) { for start in self.starts.drain(..) { // A worker that died since setup is reported through `failed`. let _ = start.send(()); @@ -362,7 +368,7 @@ impl Workers { /// The first worker's fatal error. Pends until one dies, which a healthy /// group never does, so this composes into a `select!` loop. - pub async fn failed(&mut self) -> anyhow::Error { + async fn failed(&mut self) -> anyhow::Error { match self.failures.recv().await { Some(err) => err, // Senders live in `self`, so this is unreachable; pend regardless. @@ -371,7 +377,7 @@ impl Workers { } /// Stop every worker and wait for its thread, off the caller's runtime. - pub async fn shutdown(mut self) { + async fn shutdown(mut self) { let threads = self.signal(); if threads.is_empty() { return; @@ -389,6 +395,26 @@ impl Workers { } } +/// Workers that have all initialized and can now begin accepting sessions. +pub struct Spawned { + workers: Workers, +} + +impl Spawned { + /// Release every initialized worker to accept sessions. + pub fn serve(&mut self) { + self.workers.serve(); + } + /// Wait for the first worker's fatal error. + pub async fn failed(&mut self) -> anyhow::Error { + self.workers.failed().await + } + /// Stop every worker and wait until its thread has ended. + pub async fn shutdown(self) { + self.workers.shutdown().await; + } +} + impl Drop for Workers { fn drop(&mut self) { join(self.signal()); @@ -485,7 +511,7 @@ struct Spawn { /// Reports setup back to [`Workers::spawn`], which blocks on it so a /// worker that cannot start is an error there rather than a dead thread. ready: std::sync::mpsc::Sender>, - /// Fires on [`Workers::serve`]; closes when the group stops first. + /// Fires on [`Spawned::serve`]; closes when the group stops first. start: std::sync::mpsc::Receiver<()>, /// Where a fatal error goes once the worker is running. failures: tokio::sync::mpsc::UnboundedSender, @@ -990,9 +1016,9 @@ mod tests { let mut server = moq_tokio::server::Config::default(); server.listen.bind = Some("127.0.0.1:0".parse().unwrap()); server.listen.tls.generate = vec!["localhost".into()]; - let mut workers = Workers::bind(server, moq_tokio::worker::Config::new(2)).unwrap(); + let workers = Workers::bind(server, moq_tokio::worker::Config::new(2)).unwrap(); let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); - workers + let workers = workers .spawn( cluster, auth::Auth::refuse("test"), diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 283feaeeb46..bdacc4a7865 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -279,12 +279,14 @@ impl Web { } /// Serve HTTPS from the ACME resolver, which also answers TLS-ALPN-01 challenges. + #[cfg(feature = "acme")] pub(crate) fn with_acme(mut self, resolver: Option>) -> Self { self.acme = resolver; self } /// The bound HTTPS socket, where ACME answers challenges before serving starts. + #[cfg(feature = "acme")] pub(crate) fn https_listener(&self) -> Option<&net::TcpListener> { self.https_listener.as_ref() } @@ -307,7 +309,7 @@ impl Web { } /// Choose which moq-transport extensions WebSocket sessions offer in SETUP. - pub fn with_extensions(mut self, extensions: moq_net::setup::Extensions) -> Self { + pub(crate) fn with_extensions(mut self, extensions: moq_net::setup::Extensions) -> Self { self.extensions = extensions; self } @@ -575,11 +577,7 @@ fn build_acme_config( tls.resolver = Some(resolver); tls.root = root.to_vec(); - let alpn = vec![ - b"h2".to_vec(), - b"http/1.1".to_vec(), - rustls_acme::acme::ACME_TLS_ALPN_NAME.to_vec(), - ]; + let alpn = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"acme-tls/1".to_vec()]; tls.server_config(alpn).context("failed to build https TLS config") } @@ -741,7 +739,7 @@ impl HttpStream for TlsStream { self.get_ref().1.alpn_protocol() == Some(b"h2") } fn acme_challenge(&self) -> bool { - self.get_ref().1.alpn_protocol() == Some(rustls_acme::acme::ACME_TLS_ALPN_NAME) + self.get_ref().1.alpn_protocol() == Some(b"acme-tls/1") } fn server_name(&self) -> Option<&str> { self.get_ref().1.server_name() diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index df712a47ab0..1b1f16d8cdc 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1185,7 +1185,7 @@ async fn a_certificate_admits_only_what_the_server_grants() { relay.abort(); } -/// A relay dialing in with a certificate, admitted by `moq auth serve --mtls-peer`, +/// A relay dialing in with a certificate, admitted by `moq auth serve --mtls-link peer`, /// is a cluster peer: what it announces entered the cluster elsewhere. #[tokio::test] async fn an_mtls_peer_is_a_cluster_peer() { @@ -1196,14 +1196,14 @@ async fn an_mtls_peer_is_a_cluster_peer() { ); } -/// Without `--mtls-peer`, a certificate identifies a client ingesting here. +/// Without `--mtls-link peer`, a certificate identifies a client ingesting here. #[tokio::test] async fn an_mtls_client_ingests_here() { assert_eq!(mtls_route_source(false).await, moq_net::origin::Source::Local); } /// The source the relay records for a broadcast announced over a certificate -/// session, admitted by `moq auth serve` with `mtls_peer` set as given. +/// session, admitted by `moq auth serve` with the peer link role when requested. async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let dir = tempfile::tempdir().expect("tempdir"); @@ -1211,7 +1211,7 @@ async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { let mut policy = moq_auth::serve::Policy::default(); policy.mtls = moq_auth::Permissions::new(all(), all()); - policy.mtls_peer = mtls_peer; + policy.mtls_link = mtls_peer.then_some(moq_auth::serve::Link::Peer); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let auth_url: url::Url = format!("http://{}/", listener.local_addr().unwrap()).parse().unwrap(); let server = moq_auth::serve::Server::new(policy).unwrap(); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 749c0c5fa0d..1fbd0758454 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -466,24 +466,43 @@ struct EmbeddedConfig { /// A flattened relay merges its settings without resetting the embedder's flags. #[test] fn embedded_cli_merges_only_relay_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write(&path, "[cluster]\nid = 7\n[stats]\nenabled = false\n").unwrap(); let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[ + path.as_os_str(), std::ffi::OsStr::new("--worker-name"), std::ffi::OsStr::new("recorder"), std::ffi::OsStr::new("--cluster-id"), std::ffi::OsStr::new("9"), ]) .expect("parse embedding CLI"); - let file = toml::from_str::("[cluster]\nid = 7\n").unwrap(); - let source = moq_tokio::cli::FileSource { - path: std::path::Path::new("relay.toml"), - value: &file, - }; - parsed - .relay - .merge_into(&cli, &usage_rs::config::EnvLayer::from_process(), Some(source)) - .unwrap(); + parsed.relay.merge(&cli).unwrap(); assert_eq!(parsed.worker_name.as_deref(), Some("recorder")); assert_eq!(parsed.relay.cluster.id, Some(9)); + assert_eq!(parsed.relay.source("cluster.id"), Some("--cluster-id")); + assert!(!parsed.relay.stats.enabled); + assert!(parsed.relay.source("stats.enabled").unwrap().contains("enabled")); + // The explicit file value equals the runtime default but still has file provenance. + let (mut defaults, cli) = EmbeddedConfig::parse_from_with_settings(&[]).unwrap(); + defaults.relay.merge(&cli).unwrap(); + assert_eq!(defaults.relay.stats.enabled, parsed.relay.stats.enabled); + assert_ne!( + defaults.relay.source("stats.enabled"), + parsed.relay.source("stats.enabled") + ); +} + +/// A composed CLI refuses obsolete file keys before constructing a relay or initializing logging. +#[test] +fn embedded_loader_refuses_obsolete_file_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write(&path, "[server]\nbind = \"127.0.0.1:0\"\n").unwrap(); + let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[path.as_os_str()]).unwrap(); + let error = parsed.relay.merge(&cli).unwrap_err(); + assert!(error.to_string().contains("[server]"), "{error}"); + assert!(error.to_string().contains("[listen]"), "{error}"); } /// The next route and whether it is active. diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 3cb044e9813..34e9d88dbac 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -10,13 +10,14 @@ reason = "tests read the host clock directly" )] -mod auth_embedder; mod auth_lifetime; mod cluster_close; mod cluster_drain; mod cluster_idle; mod cluster_unknown; +#[cfg(feature = "test-support")] mod connection; +#[cfg(feature = "test-support")] mod drain; mod embed; mod goaway_cluster; diff --git a/rs/moq-relay/tests/integration/memory.rs b/rs/moq-relay/tests/integration/memory.rs index f06174d4b30..4ff69f4c170 100644 --- a/rs/moq-relay/tests/integration/memory.rs +++ b/rs/moq-relay/tests/integration/memory.rs @@ -3,7 +3,9 @@ //! No sockets, so a test on Tokio's paused clock never waits on the host: the clock //! only advances once every message in flight has landed. +#[cfg(feature = "test-support")] use moq_net::transport::memory; +#[cfg(feature = "test-support")] use moq_relay::{Accepted, Connection, auth, cluster}; /// The version a test speaks unless it pins one: the newest the relay offers. @@ -17,6 +19,7 @@ pub fn newest() -> moq_net::Version { /// The relay's half runs as [`Connection::run`] on its own task, reported to the /// decider as QUIC. Lite 05+ answers SETUP before the verdict, so a refused session /// may connect and then close. +#[cfg(feature = "test-support")] pub async fn connect( client: moq_net::Client, version: moq_net::Version, @@ -47,6 +50,7 @@ pub async fn connect( /// An `Auth` granting everything under `public`, `public_publish`, and /// `public_subscribe` to anyone, as the `[auth]` config does. +#[cfg(feature = "test-support")] pub fn public(public: &[&str], publish: &[&str], subscribe: &[&str]) -> auth::Auth { let patterns = |texts: &[&str]| texts.iter().map(|text| text.parse().expect("pattern")).collect(); let mut config = auth::Config::default(); @@ -59,6 +63,7 @@ pub fn public(public: &[&str], publish: &[&str], subscribe: &[&str]) -> auth::Au } /// A standalone cluster with default options. +#[cfg(feature = "test-support")] pub fn cluster() -> cluster::Cluster { cluster::Cluster::new(cluster::Options::default()).expect("cluster init") } diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index 04edfbcd013..f6115559ea8 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -37,7 +37,7 @@ pub(crate) fn config(dir: &std::path::Path) -> Config { config.listen.tls.cert = vec![cert]; config.listen.tls.key = vec![key]; config.quic.max_streams = Some(VIEWER_STREAMS); - config.peer.quic.max_streams = std::num::NonZeroU64::new(4 * PROBE as u64).unwrap(); + config.limits.peer.quic.max_streams = std::num::NonZeroU64::new(4 * PROBE as u64).unwrap(); config } @@ -165,30 +165,39 @@ async fn refused_at_load(set: impl FnOnce(&mut Config)) -> String { } } -/// A limit QUIC cannot advertise or honor is refused at load, in `[peer.quic]` +/// A limit QUIC cannot advertise or honor is refused at load, in `[limits.peer.quic]` /// and `[quic]` alike, rather than wedging the peer or the handshake later. #[tokio::test] async fn unadvertisable_quic_limits_are_refused_at_load() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let err = refused_at_load(|config| { - config.peer.quic.receive_window = std::num::NonZeroU64::new(1 << 62).unwrap(); + config.limits.peer.quic.receive_window = std::num::NonZeroU64::new(1 << 62).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("receive_window"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("receive_window"), + "{err}" + ); // Incomplete messages hold connection credit, so the window has to fit the largest. let err = refused_at_load(|config| { let window = moq_net::transport::MIN_RECEIVE_WINDOW - 1; - config.peer.quic.receive_window = std::num::NonZeroU64::new(window).unwrap(); + config.limits.peer.quic.receive_window = std::num::NonZeroU64::new(window).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("largest message"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("largest message"), + "{err}" + ); let err = refused_at_load(|config| { - config.peer.quic.max_streams = std::num::NonZeroU64::new((1 << 60) + 1).unwrap(); + config.limits.peer.quic.max_streams = std::num::NonZeroU64::new((1 << 60) + 1).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("max_streams"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("max_streams"), + "{err}" + ); // Zero lets the peer open nothing, so it is refused too rather than wedging the connection. for max_streams in [0, (1 << 60) + 1] {