From 4aa1a4703a8bf05290b2aa8631d54b28207878fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 02:36:22 -0700 Subject: [PATCH 1/8] chore(quest): claim quest/m1/api-audit/relay-config.md Quest-Claim: df801c987323c3fd From 7774c3b4a843ce36dcd1c15e6371286c570dbdf8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 03:20:14 -0700 Subject: [PATCH 2/8] refactor(relay)!: consolidate configuration loading and roles Co-Authored-By: GPT-6 --- demo/relay/leaf0.toml | 4 +- demo/relay/leaf1.toml | 4 +- demo/relay/prod.toml | 2 +- demo/relay/root.toml | 4 +- doc/bin/cli.md | 2 +- doc/bin/relay/auth.md | 15 +- doc/bin/relay/cluster.md | 20 +- doc/bin/relay/config.md | 39 +- doc/bin/relay/index.md | 7 +- nix/modules/moq-relay.nix | 4 +- quest/m1/api-audit/relay-config.md | 71 +-- rs/moq-auth/MIGRATE.md | 8 + rs/moq-auth/src/serve.rs | 79 +++- rs/moq-cli/MIGRATE.md | 14 +- rs/moq-cli/src/args.rs | 28 +- rs/moq-cli/src/auth.rs | 41 +- rs/moq-cli/src/main.rs | 32 +- rs/moq-relay/Cargo.toml | 5 +- rs/moq-relay/MIGRATE.md | 45 +- rs/moq-relay/benches/auth.rs | 78 ++-- rs/moq-relay/src/acme.rs | 436 ++---------------- rs/moq-relay/src/acme/client.rs | 409 ++++++++++++++++ rs/moq-relay/src/acme/tests.rs | 7 +- rs/moq-relay/src/auth.rs | 107 ++--- .../auth/admission_tests.rs} | 8 +- rs/moq-relay/src/auth/refresh.rs | 2 +- rs/moq-relay/src/cluster.rs | 163 +++---- rs/moq-relay/src/cluster/api.rs | 47 ++ rs/moq-relay/src/config.rs | 278 ++++++++--- rs/moq-relay/src/connection.rs | 2 +- rs/moq-relay/src/incoming.rs | 7 +- rs/moq-relay/src/lib.rs | 1 + rs/moq-relay/src/limits.rs | 168 +++++-- rs/moq-relay/src/relay.rs | 34 +- rs/moq-relay/src/settings.rs | 71 +-- rs/moq-relay/src/uring.rs | 52 ++- rs/moq-relay/src/web.rs | 12 +- .../tests/integration/auth_lifetime.rs | 4 +- rs/moq-relay/tests/integration/embed.rs | 37 +- rs/moq-relay/tests/integration/main.rs | 3 +- rs/moq-relay/tests/integration/memory.rs | 5 + rs/moq-relay/tests/integration/peer_limits.rs | 25 +- rs/moq-tokio/MIGRATE.md | 8 + rs/moq-tokio/src/connect.rs | 25 +- rs/moq-tokio/src/listen.rs | 20 +- 45 files changed, 1454 insertions(+), 979 deletions(-) create mode 100644 rs/moq-relay/src/acme/client.rs rename rs/moq-relay/{tests/integration/auth_embedder.rs => src/auth/admission_tests.rs} (98%) create mode 100644 rs/moq-relay/src/cluster/api.rs diff --git a/demo/relay/leaf0.toml b/demo/relay/leaf0.toml index 5aab70afd91..e18d51e2741 100644 --- a/demo/relay/leaf0.toml +++ b/demo/relay/leaf0.toml @@ -16,7 +16,7 @@ tls.key = ["leaf0.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # tls.root = ["ca.pem"] @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/"] public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/demo/relay/leaf1.toml b/demo/relay/leaf1.toml index 5e80c9b1421..b888ff38256 100644 --- a/demo/relay/leaf1.toml +++ b/demo/relay/leaf1.toml @@ -16,7 +16,7 @@ tls.key = ["leaf1.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # tls.root = ["ca.pem"] @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/", "https://localhost:4444/"] public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/demo/relay/prod.toml b/demo/relay/prod.toml index 1529ac7550d..d30593368f0 100644 --- a/demo/relay/prod.toml +++ b/demo/relay/prod.toml @@ -39,7 +39,7 @@ key = "key.pem" # moq auth serve --listen 127.0.0.1:4440 --key root.jwk \ # --public-subscribe 'anon/**' --public-subscribe 'demo/**' \ # --public-publish 'anon/**' --public-publish 'demo/viewer/**' \ -# --mtls-publish '**' --mtls-subscribe '**' --mtls-peer +# --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer # # For multiple keys with rotation, give it `--key-dir /path/to/keys/` instead: # keys are named {kid}.jwk and resolved by the kid in the JWT header. diff --git a/demo/relay/root.toml b/demo/relay/root.toml index 0043426f588..9fb262e472d 100644 --- a/demo/relay/root.toml +++ b/demo/relay/root.toml @@ -17,7 +17,7 @@ tls.key = ["root.key"] # To authenticate peers by certificate, trust client certificates signed by # this CA and switch `[auth]` below to the auth server, which grants them # as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and -# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a +# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a # client CA alongside them. # `just ca` generates the CA. # tls.root = ["ca.pem"] @@ -32,6 +32,6 @@ listen = "[::]:4443" public = "**" # To exercise JWT + public-pattern auth instead, drop `public` above, run -# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`, +# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`, # point the relay at it, and uncomment `listen.tls.root` above: # url = "http://127.0.0.1:4440/" diff --git a/doc/bin/cli.md b/doc/bin/cli.md index b42a7222f3b..70ba5b305cd 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -387,7 +387,7 @@ MoQ process and `moq-relay` on the LAN, with no other configuration. Without `--cluster-lan-secret`, anyone who can reach the listener joins, so leave it unset only on networks you trust. `--cluster-connect` is a MoQ side on its own, so it needs no `--connect`. WAN cluster dials (`--cluster-connect`, -`--cluster-connect-api`) share `--connect-bind` with `--connect`, and each binds +`--cluster-api-url`) share `--connect-bind` with `--connect`, and each binds it, while LAN dials take an ephemeral port. So a fixed `--connect-bind` port with both `--connect` and a WAN cluster peer is refused at startup: use port 0, or drop `--connect` (unset `MOQ_CONNECT` if it supplies the URL). diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index e368101b94f..19a1d7db24b 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -160,7 +160,7 @@ cannot verify a presented token; both refuse it as unsupported. A moq-transport same way on any draft, but its grant authorizes only that request and ends with it; it never joins the union. A `PUBLISH_NAMESPACE` grant must cover the whole namespace: `room/**`, not just `room`. A session verifies at most -`verifications` tokens at once (`--auth-verifications`, default 128), counting +128 tokens at once, counting checks still running past the admission timeout. Later tokens wait their turn rather than being refused. A waiting token the peer withdraws never reaches the auth server. The ten-second admission timeout counts from a token's arrival, so @@ -259,11 +259,11 @@ verifies client certificates, so a relay without `listen.bind` refuses `listen.tls.root` too. Cluster peers are admitted the same way, so a mesh runs -`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer` (or a +`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` (or a server granting its cluster CA everything with `peer: true`); see -[Clustering](/bin/relay/cluster). `--mtls-peer` marks every certificate as +[Clustering](/bin/relay/cluster). `--mtls-link peer` marks every certificate as another relay; leave it off when certificates identify clients. -`--mtls-upstream`, which needs `--mtls-peer`, also marks them +`--mtls-link upstream` also marks them [upstream](/bin/relay/cluster#upstream-links). Changing either flag ends each live mTLS session at its next re-check (the `--revalidate` cadence or a `moq auth revalidate` push), so the mesh redials once. @@ -290,7 +290,7 @@ map future transport names to `unknown`, which your policy should handle explici moq auth serve --listen 127.0.0.1:4440 \ --key-dir /etc/moq/keys \ --public-subscribe 'anon/**' --public-publish 'anon/**' \ - --mtls-publish '**' --mtls-subscribe '**' --mtls-peer \ + --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer \ --tier edge --expires 1d --revalidate 1m --limit-remote 64 ``` @@ -340,7 +340,7 @@ patterns: `--auth-public 'PREFIX/**'`. | `--auth-public-publish` / `--auth-public-subscribe` | `--public-publish` / `--public-subscribe`, as patterns | | `--auth-public-api URL` | your own server answering the contract | | `--auth-mtls-tier LABEL` | `--tier LABEL` (one tier per server) | -| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-peer` | +| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` | | `--auth-api` (token or proxy mode), `Cache-Control` | `--auth-url` pointed at any server answering the contract; `revalidate` and `expires` in the grant | | `--auth-domain` | your server reads `server_name` and decides | @@ -365,8 +365,7 @@ session at the grant's `expires`, and a connection whose admission is still unanswered at its `listen.timeout` handshake deadline is closed; a token presented in band is refused after ten seconds. `Admission::attached()` marks a token presented [in band](#in-band-renewal) for a session already admitted under the same id, -so count sessions only on the others. Gateway accept loops can call `Cluster::admit` for the same -scoped session as a native connection. See +so count sessions only on the others. See [docs.rs/moq-relay](https://docs.rs/moq-relay) and [moq-auth](/lib/rs/moq-auth). diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 7b52fae52f6..6b62a584862 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -67,7 +67,7 @@ that is another source, so it restarts. ## Topology List the peers each relay dials. That's the whole topology: a relay dials only -peers from `connect`, [`connect_api`](#dynamic-peer-lists), or +peers from `connect`, [`api.url`](#dynamic-peer-lists), or [LAN discovery](#lan-discovery), never a URL learned from an announcement. ```toml @@ -79,7 +79,7 @@ connect = ["https://us-east.example.com/"] A chain (`eu-west <- us-east <- us-west`) dedupes fetches through the middle; a full mesh trades that for one fewer hop. Mix shapes as your traffic demands. -For a full mesh, list every other relay, or serve the list from `connect_api`. +For a full mesh, list every other relay, or serve the list from `api.url`. A session carries both directions, so one dial per pair is enough; listing a pair on both sides opens a redundant second session. @@ -105,7 +105,7 @@ connect = [ The mark belongs to the link, whichever side dialed: a dialed peer is upstream when its `connect` entry says so, and a peer that dials in when its [grant](/bin/relay/auth#the-contract) sets `"upstream": true` beside -`"peer": true`. `moq auth serve --mtls-peer --mtls-upstream` grants that to +`"peer": true`. `moq auth serve --mtls-link upstream` grants that to every certificate, so use it only where cores are the only relays dialing in with mTLS: on a hub that leaf relays dial into, it marks every leaf upstream and the hub stops forwarding between them. A relay that predates the mark treats every link as @@ -199,7 +199,7 @@ different `app` never discover each other. ## Dynamic peer lists -Point `connect_api` at an HTTP(S) endpoint or local file returning a JSON +Point `api.url` at an HTTP(S) endpoint or local file returning a JSON array of peers: bare URL strings and/or the same objects `connect` accepts. The relay re-checks it (honoring `Cache-Control`, or watching the file) and reconciles: new peers are dialed, missing ones dropped, @@ -211,15 +211,17 @@ changed URLs redialed. A bad fetch keeps the last good list. ```toml [cluster] -connect_api = "https://api.example.com/cluster/peers" node = "https://us-west.example.com/" + +[cluster.api] +url = "https://api.example.com/cluster/peers" ``` HTTPS peer-list fetches present the same client certificate as cluster dials, but authenticate the API with system roots and the hostname in its URL. Mesh roots, fingerprints, insecure mode, and the mesh hostname override do not -apply to the API. For a private API, set `cluster.connect_api_tls_root` to PEM -paths (or repeat `--cluster-connect-api-tls-root`); these replace system trust +apply to the API. For a private API, set `cluster.api.tls.root` to PEM +paths (or repeat `--cluster-api-tls-root`); these replace system trust for API fetches only. Malformed roots fail startup, and files reload for new connections when rotated. @@ -252,9 +254,9 @@ Peers dial with **mTLS** (recommended: `listen.tls.root` on the listener, URL, `token` on a peer object, or a shared `cluster.token` file for every listed peer). The accepting relay admits a peer like any client, so a mesh needs an auth server that grants the cluster CA, such as -`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`. An +`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`. An accepted peer counts as a cluster peer only when its grant sets `peer: true`, -as `--mtls-peer` does; otherwise what it announces counts as ingest here, like +as `--mtls-link peer` does; otherwise what it announces counts as ingest here, like a client's. Dials retry forever with capped backoff, so a rejected peer is loud in the logs rather than fatal. See [Authentication](/bin/relay/auth#mtls). diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index b76ef9e601c..f14a20b9ec7 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -99,7 +99,7 @@ Raise the receive windows when a fat, long path idles below the link rate. The 128 MiB `receive_window` default carries several Gbps at a 100 ms RTT and bounds how much unread data and incomplete messages one peer can make the relay buffer. It has to fit the largest message, just over 64 MiB, so a smaller -value, here or in `[peer.quic]`, is refused at startup. When incomplete +value, here or in `[limits.peer.quic]`, is refused at startup. When incomplete messages fill the window, the relay cancels the least urgent stream: the lowest subscription priority, then the group its subscription would deliver last. Over iroh the window bounds only @@ -107,40 +107,40 @@ unread data, and over WebSocket or TCP nothing bounds incomplete messages. `stream_receive_window` only paces one stream: its credit returns as bytes are read, whether or not their message is complete. -## \[session] and \[peer] +## \[limits.client] and \[limits.peer] Request caps apply separately to each session on every transport. The relay -uses `[peer]` for an admitted grant marked `peer` or `upstream`, including +uses `[limits.peer]` for an admitted grant marked `peer` or `upstream`, including verified LAN membership, and for links it dials through the cluster. A client certificate, route hop, or cluster-looking URL alone does not select peer caps. ```toml -[session] +[limits.client] announces = 1000 # Simultaneous broadcasts or namespaces. subscriptions = 1000 # Simultaneously held subscriptions. fetches = 10000 # Independent simultaneous FETCH requests. -[peer] +[limits.peer] announces = 100000 subscriptions = 10000 fetches = 10000 -[peer.quic] +[limits.peer.quic] max_streams = 100000 # Concurrent QUIC streams per direction. receive_window = 671088640 # Connection-wide receive window (640 MiB). send_window = 671088640 # Unacknowledged outgoing data (640 MiB). ``` -`[peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`; +`[limits.peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`; any other `[quic]` key is refused. They replace the `[quic]` values for a peer's QUIC connection once its grant is known, before the SETUP reply. Links the relay dials get them from the handshake on. A link carrying a whole cluster is then not throttled by `[quic]` limits sized for one viewer, for example -`--peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep +`--limits-peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep their transport's own limits. -These are the defaults. For example, `--peer-subscriptions=50000` raises the -subscription cap on cluster links. `--session-fetches` and `--peer-fetches` +These are the defaults. For example, `--limits-peer-subscriptions=50000` raises the +subscription cap on cluster links. `--limits-client-fetches` and `--limits-peer-fetches` set the independent FETCH caps; concurrent callers of the same group each hold a slot. Size peer caps for the combined traffic they carry, and use session peak counters to check announcement and subscription headroom. @@ -204,13 +204,14 @@ See [HTTP endpoints](/bin/relay/http). ## \[acme] The relay provisions and renews a publicly trusted certificate itself, for QUIC -and HTTPS alike (Unix only, where the key can be kept owner-only): +and HTTPS alike (Unix only, where the key can be kept owner-only). This needs +the default `acme` Cargo feature; builds without it refuse configured issuance: ```toml [acme] domains = ["relay.example.com"] # Enables ACME. No wildcards. contact = ["mailto:ops@example.com"] # Optional account contact. -dir = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only. +cache = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only. # directory = "https://acme-staging-v02.api.letsencrypt.org/directory" # Default: Let's Encrypt production. # timeout = "5m" # How long the first issuance may take before startup fails. ``` @@ -220,11 +221,11 @@ domain, so `web.https.listen` is required and must be reachable there, directly or forwarded. `[acme]` cannot be combined with `listen.tls.cert`, `key`, or `generate`, or with `web.https.cert` and `key`. -With no usable certificate in `dir` (missing, expired, missing a domain, or +With no usable certificate in `cache` (missing, expired, missing a domain, or from another directory), startup answers only the challenge until one is issued; QUIC, the other routes, and readiness wait for it, and startup fails with the ACME error once `timeout` runs out. It also fails after 3 failed -validations within an hour, counting earlier starts recorded in `dir`, so a +validations within an hour, counting earlier starts recorded in `cache`, so a restart loop stops short of Let's Encrypt's limit of 5 per hostname per hour. Renewal runs in the background well before expiry and reaches every listener on its next handshake. A failed renewal keeps the current certificate serving and retries. @@ -267,7 +268,6 @@ url = "http://127.0.0.1:4440/" # An auth server asked once per session eve # public = "anon/**" # Or a static anonymous grant rooted at /, publish and subscribe alike. # public_subscribe = ["anon/**", "demo/**"] # Or split them; patterns, `foo/**` for a subtree. # public_publish = ["anon/**"] -verifications = 128 # In-band tokens one session verifies at once; later ones wait. ``` See [Authentication](/bin/relay/auth). @@ -278,11 +278,13 @@ See [Authentication](/bin/relay/auth). [cluster] connect = ["https://us-east.example.com/?cost=10"] # Peers to dial. ?cost prices the link, or use {url, cost, egress, token, upstream} objects. node = "https://us-west.example.com/" # This relay's own URL. -# connect_api_tls_root = ["api-ca.pem"] # Private API roots, independent from connect.tls. -connect_api = "https://api.example.com/peers" # Or fetch the peer list (JSON array of URLs and/or objects) live. token = "cluster.jwt" # JWT for dials without an inline ?jwt=. id = 12345 # Stable Hop ID across restarts. +[cluster.api] +url = "https://api.example.com/peers" # Live JSON peer list; a local file also works. +# tls.root = ["api-ca.pem"] # API server trust, independent from connect.tls. + [cluster.lan] # Find peers on the LAN over mDNS. enabled = true # secret = "/etc/moq/cluster.key" # Optional: a file holding 64 hex chars. @@ -303,9 +305,6 @@ tls.cert = "relay.pem" # Present a client certificate (mTLS to pee tls.key = "relay.key" goaway.redirect = "same-host" # How far to trust a draining peer's redirect URI. goaway.handover = "10s" # Cap on how long the drained upstream keeps serving. -extensions.auth = false # Decline MoQ Auth on outbound moq-transport sessions. -extensions.solicit = false # Decline MoQ Solicit. -extensions.epoch = false # Decline MoQ Epoch. All default to on, as in [listen.extensions]. ``` A draining upstream may name a replacement URI. `same-host` follows it only diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 9359869eca1..a41f925e3dc 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -57,7 +57,10 @@ between releases are listed in `moq-relay` is also a library, for an application that wants extra HTTP routes or in-process workers against the cluster origin. Load a `Relay`, take what you need, and call `run`, which keeps the listeners, QUIC workers, and -shutdown inside the relay. +shutdown inside the relay. An embedded CLI calls +`config.relay.merge(&cli_layer)?` after its own `parse_from_with_settings` to +load the relay file and environment, retain provenance, and refuse obsolete +spellings. CLI-only fields on the embedding root stay intact. ```rust use axum::routing::get; @@ -86,7 +89,7 @@ relay.with_web(web).run().await?; from `relay.shutdown().activity()` and pass `relay.shutdown().clone()` to `Connection::with_shutdown`, so the drain waits for its handshakes too. - To serve a session from any other moq-net transport, such as the in-memory - one in tests, read its SETUP with `moq_net::Server::accept_request` and pass + one in tests, enable the `test-support` feature, read its SETUP with `moq_net::Server::accept_request` and pass `Accepted::new(handshake, transport)` to `Connection::new`. The client must send its path in the SETUP. - To decide admissions yourself, leave `[auth]` empty and answer diff --git a/nix/modules/moq-relay.nix b/nix/modules/moq-relay.nix index 981190d38bd..9ab5896f9e0 100644 --- a/nix/modules/moq-relay.nix +++ b/nix/modules/moq-relay.nix @@ -256,7 +256,7 @@ in "${pkgs.moq-cli}/bin/moq auth serve" "--listen 127.0.0.1:${toString cfg.auth.port}" "--key ${authKey}" - "--mtls-publish '**' --mtls-subscribe '**' --mtls-peer" + "--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer" ] ++ lib.optionals (cfg.auth.publicPath != null) [ "--public-publish '${publicPattern}' --public-subscribe '${publicPattern}'" @@ -356,7 +356,7 @@ in } // lib.optionalAttrs (cfg.acme.domains != [ ]) { MOQ_ACME_DOMAINS = lib.concatStringsSep "," cfg.acme.domains; - MOQ_ACME_DIR = "${cfg.stateDir}/acme"; + MOQ_ACME_CACHE = "${cfg.stateDir}/acme"; MOQ_WEB_HTTPS_LISTEN = "[::]:${toString cfg.acme.httpsPort}"; } // lib.optionalAttrs (cfg.acme.contact != [ ]) { diff --git a/quest/m1/api-audit/relay-config.md b/quest/m1/api-audit/relay-config.md index 08e0072eb52..50c4db09e37 100644 --- a/quest/m1/api-audit/relay-config.md +++ b/quest/m1/api-audit/relay-config.md @@ -8,37 +8,40 @@ names or each other. ## Plan -Decided 2026-10-22 (API audit): - -- One loader: `Config::merge_into(&CliLayer, &EnvLayer, Option)` - takes about 25 lines of setup, so moq.pro's edge bypasses it - (`toml::from_str` plus `update_from`) and `deprecated()` never runs: - `MOQ_SERVER_BIND` or `[server]` are silently ignored there. Provide one - call both binaries use (for example `moq_tokio::cli::load::()`) with - provenance, so an embedder can tell a set value from a default without - comparing against `Default`. -- `[session]` and `[peer]` collide with `moq_relay::session` and - `cluster::Peer`; rename to `[limits.client]` and `[limits.peer]`. -- `acme.dir` (a state path, documented required but typed `Option`) beside - `acme.directory` (the URL): rename `dir` to `acme.cache`; rustls-acme moves - behind an `acme` feature. -- `cluster.connect_api_tls_root` is flat while everything else nests - `*.tls.root`: `[cluster.api] url`, `tls.root`. -- `auth.verifications`/`Auth::with_verifications(usize)` is used only by a - bench and panics on 0: a private constant. -- `[listen|connect.extensions]` is TOML-only and unset everywhere: drop the - config field and `Web::with_extensions`, keep the Rust API. -- `uring::Workers::spawn` then `serve()` is order-dependent (`serve()` alone - no-ops): typestate `spawn(self) -> Spawned`. -- `cluster::with_client_tls` repeats TLS `with_connect` holds: delete. -- mTLS: `serve::Policy::{mtls_peer, mtls_upstream}` and the matching flags - can express "upstream without peer", refused at runtime; take - `Option` and one `--mtls-link` flag, keeping the - `Grant` wire booleans. -- Public only for tests: `Cluster::admit`, `cluster::Admitted`, and - `auth::hold` (move moq.pro onto them, since it keeps copies, or make them - `pub(crate)`); `Incoming`/`Accepted` behind `test-support`. -- Extend the `demo_configs_parse` test to `packaging/` and `test/`. - -Docs: `doc/bin/relay/`. Public API: moq-relay config and Rust API breaking; -old spellings are refused by name. Wire: none. +Decided 2026-10-22 (API audit). The public implementation is checkpointed in +[moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535): + +- `Config::merge(&CliLayer)` reads the optional file and environment, reuses + the existing Merge/Resolved provenance, and retains embedded CLI-only fields. +- Limits live under `[limits.client]` / `[limits.peer]`; ACME uses `cache` and + a default-enabled `acme` dependency feature; dynamic peers use + `[cluster.api] url` and `tls.root`. +- Verification concurrency is a private production constant, with a private + test override; the benchmark sweeps offered concurrency and token payload. +- Extension fields were not unused: the relay enables IETF and forwards its + offers to Web. Preserve the typed Rust fields and native setup API, refuse + their TOML surface including explicit false, and keep Web setup crate-private. +- Worker binding and spawning use `Workers` → `Spawned`; only the spawned + owner serves. Cluster API TLS comes from the existing dial template. +- mTLS policy uses one optional `Link::{Peer, Upstream}` role, preserving + the `Grant` JSON booleans. Test-only admission helpers are private; + in-memory `Incoming` / `Accepted` exports require `test-support`. +- Tracked demo, packaging, and test relay fixtures use the shared loader; + the intentional published legacy fixture is checked for refusal. + +Remaining completion prerequisite, not an unresolved public API policy: + +1. Land the upstream implementation. +2. Advance each pinned embedding consumer's dependency through its normal + upstream-first process. +3. Replace that consumer's direct TOML/update setup with the shared loader, + migrate renamed settings and native types, and run its own required checks. + Verify explicit obsolete file/environment spellings refuse startup and + embedded CLI-only fields survive, without default-value comparisons. +4. Remove this quest once every embedder uses the one-call loader. + +Do not pre-migrate consumers while their dependency pin lacks this API, add +compatibility aliases, or combine unrelated pending transport changes here. + +Docs: `doc/bin/relay/`. Public API: moq-relay/moq-auth native configuration and +moq CLI breaking, with package migrations. Wire: unchanged. diff --git a/rs/moq-auth/MIGRATE.md b/rs/moq-auth/MIGRATE.md index 59084abfc31..bffb5bb646c 100644 --- a/rs/moq-auth/MIGRATE.md +++ b/rs/moq-auth/MIGRATE.md @@ -4,6 +4,14 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A certificate has one cluster-link role (#6535) + +Replace `serve::Policy.mtls_peer` and `.mtls_upstream` with `.mtls_link`: +use `None` for ordinary certificate grants, `Some(serve::Link::Peer)` for peers, +and `Some(serve::Link::Upstream)` for upstream peers. `Grant.peer` and +`Grant.upstream` retain their existing JSON boolean contract. + + ### `Client::connect` has no timeout of its own (#5765) `Client::connect` waits on the auth server for as long as its caller does, and dropping the future drops the request. diff --git a/rs/moq-auth/src/serve.rs b/rs/moq-auth/src/serve.rs index 8b58905f935..d7b7c183a52 100644 --- a/rs/moq-auth/src/serve.rs +++ b/rs/moq-auth/src/serve.rs @@ -54,6 +54,35 @@ pub struct Limits { pub remote: Option, } +/// A verified certificate's cluster-link role. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Link { + /// A cluster peer that may exchange routes in either direction. + Peer, + /// A peer that is never offered routes learned from another upstream. + Upstream, +} + +impl std::str::FromStr for Link { + type Err = &'static str; + fn from_str(value: &str) -> Result { + match value { + "peer" => Ok(Self::Peer), + "upstream" => Ok(Self::Upstream), + _ => Err("expected peer or upstream"), + } + } +} + +impl std::fmt::Display for Link { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Peer => "peer", + Self::Upstream => "upstream", + }) + } +} + /// The decisions the server answers with, evaluated in order and stopping at the /// first that applies: a JWT, then a verified certificate, then the anonymous rules. /// A malformed or expired token is a refusal, never a fall through. @@ -81,12 +110,9 @@ pub struct Policy { /// What a session presenting a verified certificate is granted, rooted at `/`; /// empty refuses it. pub mtls: Permissions, - /// Mark every session presenting a verified certificate as a cluster - /// [peer](Grant::peer), for a mesh whose relays dial each other with mTLS. - pub mtls_peer: bool, - /// Mark every certificate's peer as [upstream](Grant::upstream) too. Requires - /// [`mtls_peer`](Self::mtls_peer). - pub mtls_upstream: bool, + /// The cluster-link role granted only to sessions presenting a verified certificate. + pub mtls_link: Option, + /// The tier stamped on every grant. pub tier: Option, /// How often the relay re-checks each grant; `None` never re-checks. The contract @@ -241,8 +267,8 @@ impl Policy { }; grant.revalidate = self.revalidate; grant.tier = self.tier.clone(); - grant.peer = certificate && self.mtls_peer; - grant.upstream = certificate && self.mtls_upstream; + grant.peer = certificate && self.mtls_link.is_some(); + grant.upstream = certificate && self.mtls_link == Some(Link::Upstream); Ok((grant, root)) } @@ -432,11 +458,8 @@ pub struct Server { impl Server { /// A server answering with `policy`, refusing one that asks for a re-check without /// a bound, bounds grants past the clock's range, caps sessions without the - /// re-check that ages out a dead relay's, or marks an upstream that is not a peer. + /// re-check that ages out a dead relay's,. pub fn new(policy: Policy) -> crate::Result { - if policy.mtls_upstream && !policy.mtls_peer { - return Err(crate::Error::UpstreamWithoutPeer); - } if policy.revalidate.is_some() && policy.expires.is_none() { return Err(crate::Error::UnboundedRevalidate); } @@ -965,7 +988,7 @@ mod tests { assert_eq!(policy.decide(&request("/")).await.unwrap_err(), Refusal::NoPublicGrant); } - /// `mtls_peer` and `mtls_upstream` mark a certificate's grant, and only a certificate's. + /// The link role marks only a certificate's grant, never an anonymous session or JWT. #[tokio::test] async fn mtls_peer_marks_only_certificates() { let (dir, key) = key_dir(); @@ -973,14 +996,14 @@ mod tests { keys: Some(Keys::Dir(dir.path().into())), public: rules(&["**"], &["**"]), mtls: rules(&["**"], &["**"]), - mtls_peer: true, + mtls_link: Some(Link::Peer), ..Default::default() }; let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); assert!(grant.peer && !grant.upstream); let policy = Policy { - mtls_upstream: true, + mtls_link: Some(Link::Upstream), ..policy }; let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); @@ -993,15 +1016,23 @@ mod tests { assert!(!grant.peer && !grant.upstream); } - /// An upstream that is not a peer would answer grants every relay refuses. - #[test] - fn a_server_refuses_an_upstream_that_is_not_a_peer() { - let policy = Policy { - mtls: rules(&["**"], &["**"]), - mtls_upstream: true, - ..Default::default() - }; - assert!(matches!(Server::new(policy), Err(Error::UpstreamWithoutPeer))); + /// Every representable mTLS link role produces a valid wire grant. + #[tokio::test] + async fn mtls_link_roles_preserve_the_grant_contract() { + for (link, peer, upstream) in [ + (None, false, false), + (Some(Link::Peer), true, false), + (Some(Link::Upstream), true, true), + ] { + let policy = Policy { + mtls: rules(&["**"], &["**"]), + mtls_link: link, + ..Default::default() + }; + let grant = policy.decide(&with_peer(request("/"), None)).await.unwrap(); + assert_eq!((grant.peer, grant.upstream), (peer, upstream)); + grant.validate().unwrap(); + } } /// A certificate would override a JWT meant to narrow it, and a JWT would narrow or diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 215b3a099ff..748a3223db6 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,18 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### Relay config and certificate roles (#6535) + +Replace `moq auth serve --mtls-peer` with `--mtls-link peer`, and the combined +peer/upstream flags with `--mtls-link upstream`. Omit the flag for ordinary +certificate grants. Both roles still need `--mtls-publish` or `--mtls-subscribe`. +The removed flags are refused even when explicitly false. + +Cluster peer-list flags are now `--cluster-api-url` and `--cluster-api-tls-root`; +the matching environment variables are `MOQ_CLUSTER_API_URL` and +`MOQ_CLUSTER_API_TLS_ROOT`. TOML uses `[cluster.api] url` and `tls.root`. + + ### `import fmp4` and `import hls` unwrap to Legacy (#6250) Both now publish one `legacy` frame per sample instead of one `cmaf` frame per fragment. @@ -45,7 +57,7 @@ Add the `openh264` feature, such as `cargo install moq-cli --features transcode, ### A fixed `--connect-bind` refuses a WAN cluster (#5674) -`--connect` and `--cluster-connect` or `--cluster-connect-api` used to share one client; now each binds its own, so a fixed `--connect-bind` port with both stops startup. +`--connect` and `--cluster-connect` or `--cluster-api-url` used to share one client; now each binds its own, so a fixed `--connect-bind` port with both stops startup. Use port 0, or drop `--connect`. ### `moq export ts --delay` defaults to 1 s and includes the send-ahead (#5553) diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index ae4ad8b10b3..5e5ae60e74d 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -6,7 +6,7 @@ //! `moq fetch ` to read one group. //! //! - The MoQ side (`--connect`, the `--listen*` transport binds, `--cluster-lan`, -//! and `--cluster-connect` / `--cluster-connect-api`; all optional, at least +//! and `--cluster-connect` / `--cluster-api-url`; all optional, at least //! one) attaches the shared Origin to the MoQ network, and comes before the //! first stage. They compose: dial a relay, accept incoming sessions, mesh //! with the LAN, and join a cluster all at once. @@ -547,9 +547,9 @@ impl MoqSide { } /// Whether a WAN cluster dial is configured (`--cluster-connect` or - /// `--cluster-connect-api`). + /// `--cluster-api-url`). fn cluster_dials(&self) -> bool { - !self.cluster.connect.is_empty() || self.cluster.connect_api.is_some() + !self.cluster.connect.is_empty() || self.cluster.api.url.is_some() } /// Reject a verb that needs the MoQ network but was given no way to reach it. @@ -567,7 +567,7 @@ impl MoqSide { !(self.client.url.is_some() && self.cluster_dials() && self.client.bind.is_some_and(|bind| bind.port() != 0)), - "--connect-bind with a fixed port cannot serve both --connect and --cluster-connect / --cluster-connect-api, which each bind it: use port 0, or drop --connect (unset MOQ_CONNECT if it supplies the URL)" + "--connect-bind with a fixed port cannot serve both --connect and --cluster-connect / --cluster-api-url, which each bind it: use port 0, or drop --connect (unset MOQ_CONNECT if it supplies the URL)" ); #[cfg(feature = "cluster-lan")] { @@ -2294,11 +2294,7 @@ mod tests { ("--listen-tcp-bind", "127.0.0.1:0", "--listen-tcp-bind"), ("--broadcast", "room", "--broadcast"), ("--cluster-connect", "https://relay.example", "--cluster-connect"), - ( - "--cluster-connect-api", - "https://api.example/peers", - "--cluster-connect-api", - ), + ("--cluster-api-url", "https://api.example/peers", "--cluster-api-url"), ("--cluster-node", "https://self.example", "--cluster-node"), ("--cluster-token", "cluster.jwt", "--cluster-token"), ("--cluster-id", "1", "--cluster-id"), @@ -2449,7 +2445,7 @@ mod tests { }; for peer in [ ["--cluster-connect", "https://relay.example"], - ["--cluster-connect-api", "https://api.example/peers"], + ["--cluster-api-url", "https://api.example/peers"], ] { let both = [ &["--connect", "https://relay.example", "--connect-bind", "127.0.0.1:4443"][..], @@ -2483,7 +2479,7 @@ mod tests { .expect("--connect alone owns a fixed bind"); } - /// `--cluster-connect` / `--cluster-connect-api` attach the process as a + /// `--cluster-connect` / `--cluster-api-url` attach the process as a /// cluster peer, so they are a MoQ side on their own. `--cluster-node` is /// identity, not an attachment. #[test] @@ -2492,14 +2488,8 @@ mod tests { .expect("parse"); assert!(cli.moq.validate().is_ok(), "a cluster dial is a MoQ side on its own"); - let cli = Invocation::try_parse_from([ - "moq", - "--cluster-connect-api", - "https://api.example/peers", - "import", - "ts", - ]) - .expect("parse"); + let cli = Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) + .expect("parse"); assert!(cli.moq.validate().is_ok(), "a cluster API is a MoQ side on its own"); let cli = Invocation::try_parse_from(["moq", "--cluster-node", "https://self.example", "import", "ts"]) diff --git a/rs/moq-cli/src/auth.rs b/rs/moq-cli/src/auth.rs index 2b17f8aeb7a..9f0ca4cfb1d 100644 --- a/rs/moq-cli/src/auth.rs +++ b/rs/moq-cli/src/auth.rs @@ -397,13 +397,14 @@ pub struct Serve { #[usage(long)] mtls_subscribe: Vec, - /// Mark sessions with a verified certificate as cluster peers: relays forwarding what entered elsewhere. - #[usage(long)] - mtls_peer: bool, + /// The role verified certificates have on cluster links: peer or upstream. + #[usage(long, choices("peer", "upstream"))] + mtls_link: Option, - /// Mark those peers upstream: never offered a route learned from another upstream. Needs `--mtls-peer`. - #[usage(long)] - mtls_upstream: bool, + #[usage(long = "mtls-peer", hide = true, default_missing = "true", num_args = 0..=1, require_equals = true)] + mtls_peer: Option, + #[usage(long = "mtls-upstream", hide = true, default_missing = "true", num_args = 0..=1, require_equals = true)] + mtls_upstream: Option, /// The tier label stamped on every grant, handed to the relay's stats. #[usage(long)] @@ -444,11 +445,11 @@ impl Serve { "--limit-token and --limit-remote need --revalidate, which ages out the slots of dead relays" ); } - if self.mtls_peer && self.mtls_publish.is_empty() && self.mtls_subscribe.is_empty() { - anyhow::bail!("--mtls-peer needs --mtls-publish or --mtls-subscribe, or every certificate is refused"); + if self.mtls_peer.is_some() || self.mtls_upstream.is_some() { + anyhow::bail!("--mtls-peer and --mtls-upstream are removed; use --mtls-link peer or --mtls-link upstream"); } - if self.mtls_upstream && !self.mtls_peer { - anyhow::bail!("--mtls-upstream needs --mtls-peer; only a cluster peer can be upstream"); + if self.mtls_link.is_some() && self.mtls_publish.is_empty() && self.mtls_subscribe.is_empty() { + anyhow::bail!("--mtls-link needs --mtls-publish or --mtls-subscribe, or every certificate is refused"); } // 0.14 read `anon` as the prefix `anon/`, and a pattern reads it as exactly the // broadcast `anon`, so either silent reading would mislead someone upgrading. @@ -487,8 +488,7 @@ impl Serve { }; policy.public = rules(&self.public_publish, &self.public_subscribe); policy.mtls = rules(&self.mtls_publish, &self.mtls_subscribe); - policy.mtls_peer = self.mtls_peer; - policy.mtls_upstream = self.mtls_upstream; + policy.mtls_link = self.mtls_link; policy.tier = self.tier.clone(); policy.revalidate = revalidate; policy.expires = expires; @@ -682,8 +682,8 @@ mod tests { "**", "--mtls-subscribe", "**", - "--mtls-peer", - "--mtls-upstream", + "--mtls-link", + "upstream", "--tier", "internal", "--revalidate", @@ -704,7 +704,7 @@ mod tests { ); assert!(policy.public.publish.is_empty()); assert_eq!(policy.mtls.publish, ["**".parse().unwrap()].into_iter().collect()); - assert!(policy.mtls_peer && policy.mtls_upstream); + assert_eq!(policy.mtls_link, Some(moq_auth::serve::Link::Upstream)); assert_eq!(policy.tier.as_deref(), Some("internal")); assert_eq!(policy.revalidate, Some(std::time::Duration::from_secs(30))); assert_eq!(policy.expires, Some(std::time::Duration::from_secs(7200))); @@ -726,7 +726,7 @@ mod tests { let bare = serve(&["moq", "auth", "serve"]).policy().unwrap(); assert!(bare.keys.is_none()); assert!(bare.public.is_empty() && bare.mtls.is_empty()); - assert!(!bare.mtls_peer && !bare.mtls_upstream); + assert_eq!(bare.mtls_link, None); assert_eq!(bare.revalidate, None); assert_eq!(bare.expires, None); @@ -739,11 +739,10 @@ mod tests { (&["--limit-token", "3"], "need --revalidate"), (&["--limit-remote", "3", "--expires", "1h"], "need --revalidate"), // A peer marking with nothing granted to certificates marks nobody. - (&["--mtls-peer"], "--mtls-peer needs"), - ( - &["--mtls-subscribe", "**", "--mtls-upstream"], - "--mtls-upstream needs --mtls-peer", - ), + (&["--mtls-link", "peer"], "--mtls-link needs"), + (&["--mtls-subscribe", "**", "--mtls-upstream"], "use --mtls-link"), + (&["--mtls-peer=false"], "use --mtls-link"), + (&["--mtls-upstream=false"], "use --mtls-link"), ] { let argv: Vec<&str> = ["moq", "auth", "serve"].iter().chain(args).copied().collect(); let err = serve(&argv).policy().unwrap_err().to_string(); diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index a885a71be06..6c49d26939d 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -449,9 +449,7 @@ async fn spawn_moq( directions: Directions, tasks: &mut JoinSet>, ) -> anyhow::Result { - let cluster = cluster - .with_client_tls(moq.client.tls.build()?) - .with_connect(moq.client.clone(), moq.quic.clone()); + let cluster = cluster.with_connect(moq.client.clone(), moq.quic.clone()); #[cfg(feature = "iroh")] let cluster = match net.iroh.clone() { Some(iroh) => cluster.with_iroh(iroh), @@ -1428,38 +1426,28 @@ mod tests { tasks.abort_all(); } - /// An HTTP `--cluster-connect-api` is a MoQ side, so start-up must attach + /// An HTTP `--cluster-api-url` is a MoQ side, so start-up must attach /// client TLS the way the relay does rather than refuse after validate. #[tokio::test] async fn cluster_connect_api_http_attaches_client_tls() { let _ = moq_tokio::crypto::install_default(); - let invocation = Invocation::try_parse_from([ - "moq", - "--cluster-connect-api", - "https://api.example/peers", - "import", - "ts", - ]) - .expect("parse"); + let invocation = + Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) + .expect("parse"); assert!(invocation.moq.validate().is_ok()); - let cluster = invocation - .moq - .cluster() - .expect("cluster") - .with_connect(invocation.moq.client.clone(), invocation.moq.quic.clone()); + let cluster = invocation.moq.cluster().expect("cluster"); let err = cluster .clone() .start() .await - .expect_err("http API without TLS") + .expect_err("HTTP API needs a dial template") .to_string(); - assert!(err.contains("client TLS"), "{err}"); - + assert!(err.contains("Cluster::with_connect"), "{err}"); cluster - .with_client_tls(invocation.moq.client.tls.build().expect("tls")) + .with_connect(invocation.moq.client.clone(), invocation.moq.quic.clone()) .start() .await - .expect("http API with TLS"); + .expect("HTTP API uses the dial template's TLS"); } /// Without `--connect`, only the cluster binds `--connect-bind`, so a fixed diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index f672b08bdda..cad099fd9f3 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -25,7 +25,8 @@ doc = false [features] # A programmatic relay fixture with generated TLS and ephemeral ports. test-support = [] -default = ["cluster-lan", "quic", "websocket", "uds"] +default = ["acme", "cluster-lan", "quic", "websocket", "uds"] +acme = ["dep:rustls-acme"] iroh = ["moq-tokio/iroh"] # Discover cluster peers on the LAN with mDNS (`--cluster-lan`). cluster-lan = ["moq-tokio/mdns"] @@ -77,7 +78,7 @@ qmux = { workspace = true, features = ["ws"], optional = true } reqwest = { workspace = true, features = ["rustls"] } reqwest-middleware = "0.5" rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } -rustls-acme = { version = "0.15.4", default-features = false, features = ["aws-lc-rs", "webpki-roots"] } +rustls-acme = { version = "0.15.4", optional = true, default-features = false, features = ["aws-lc-rs", "webpki-roots"] } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true, features = ["json", "base64"] } diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 70f6f6e236c..63c68428e12 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,9 +5,44 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### One loader and role-scoped config (#6535) + +After an embedded CLI's `parse_from_with_settings`, call `config.relay.merge(&cli_layer)?`. +It reads `relay.file`, applies CLI > environment > file > defaults, refuses obsolete +settings, and retains provenance through `Config::source`; root CLI-only fields stay intact. +Replace direct `merge_into` setup or TOML plus `update_from` with that call. + +Move `[session]` to `[limits.client]` and `[peer]` to `[limits.peer]`, including its +`quic` table. Flags and environment variables use `--limits-client-*` / +`MOQ_LIMITS_CLIENT_*` and `--limits-peer-*` / `MOQ_LIMITS_PEER_*`. +Rust uses `config.limits.client` / `config.limits.peer` and `limits::Client` +in place of `limits::Session`. + +Rename `acme.dir` / `--acme-dir` / `MOQ_ACME_DIR` to `acme.cache` / +`--acme-cache` / `MOQ_ACME_CACHE`. The default `acme` feature preserves automatic +certificates; builds without it refuse configured ACME settings. +Move `cluster.connect_api` to `cluster.api.url` and `cluster.connect_api_tls_root` +to `cluster.api.tls.root`; use `--cluster-api-url`, `--cluster-api-tls-root`, +`MOQ_CLUSTER_API_URL`, and `MOQ_CLUSTER_API_TLS_ROOT`. +Remove `Cluster::with_client_tls`: the peer-list API presents `with_connect`'s +client identity, with its own `api.tls.root` server trust. + +Remove `auth.verifications` / `--auth-verifications` / `MOQ_AUTH_VERIFICATIONS` +and `Auth::with_verifications`; the verification window is a private production policy. +Remove `[listen.extensions]` and `[connect.extensions]` from TOML. The typed Rust +fields and native endpoint setup APIs remain; the relay still offers IETF support. +`Web::with_extensions` is private to the relay's own setup. + +`Workers::spawn(self, …)` now returns `uring::Spawned`; call `serve`, `failed`, +and consuming `shutdown` on that value. A merely bound worker group cannot serve. +`Incoming` and `Accepted` require the `test-support` feature for in-memory tests. +`Cluster::admit`, `cluster::Admitted`, and `auth::hold` are private; real admissions +use the existing `Auth::embedded` / `Admissions` and `Lease` lifecycle. + + ### `receive_window` has a floor (#5878) -The `[quic]` and `[peer.quic]` `receive_window` must be at least 67108880 bytes (64 MiB plus 16), or the relay refuses to start; the `[quic]` default is now 128 MiB. +The `[quic]` and `[limits.peer.quic]` `receive_window` must be at least 67108880 bytes (64 MiB plus 16), or the relay refuses to start; the `[quic]` default is now 128 MiB. Raise an explicit value below that, or remove it. ### WebSocket clients must offer a QMux subprotocol (#6231) @@ -43,8 +78,8 @@ Write the hex to a file (`openssl rand -hex 32 > cluster.key`) and pass its path `listen.timeout` now bounds auth admission as part of the handshake, replacing the fixed ten-second admission timeout, so `"0"` also lets admission wait forever. HTTP reads bound admission by the web listener's timeout. -`Auth::admit` and `Cluster::admit` no longer time out on their own, so a slow embedded decider no longer refuses after ten seconds: the caller owns the deadline. -Bound them with `moq_tokio::server::Request::within` on an accepted session, or wrap them in `tokio::time::timeout`; dropping the future withdraws the admission from the decider. +`Auth::admit` no longer times out on its own, so a slow embedded decider no longer refuses after ten seconds: the caller owns the deadline. +Bound it with `moq_tokio::server::Request::within` on an accepted session, or wrap it in `tokio::time::timeout`; dropping the future withdraws the admission from the decider. Tokens attached in band keep their ten-second bound and are refused as `auth::Error::Unavailable` past it. ### Usage types come from usage-rs 7 (#6060) @@ -115,7 +150,7 @@ Each used to be ignored: drop it, or add the listener it configures. ### `--cluster-mesh` and `--cluster-linger` are unknown flags (#4818) They, TOML `mesh` and `linger`, and `MOQ_CLUSTER_MESH` / `MOQ_CLUSTER_LINGER` are no longer read. -List every peer with `--cluster-connect` or `--cluster-connect-api`. +List every peer with `--cluster-connect` or `--cluster-api-url`. In Rust, `cluster::Config` has no `mesh` or `linger` field. ### A route without a publisher keeps hop 0 (#4969) @@ -137,7 +172,7 @@ Set `listen` and `quic` on a `moq_tokio::server::Config` and pass it to `bind`. ### Gossip discovery is removed (#4601) A relay dials only the peers it lists or finds on the LAN, never a URL learned from an announcement, and no longer announces `.internal/origins`. -`--cluster-mesh` and TOML `mesh` are refused by name: list every peer with `--cluster-connect` or `--cluster-connect-api`. +`--cluster-mesh` and TOML `mesh` are refused by name: list every peer with `--cluster-connect` or `--cluster-api-url`. The `/nodes` endpoint lists only peers this relay dialed. ## 0.15 -> 0.16 diff --git a/rs/moq-relay/benches/auth.rs b/rs/moq-relay/benches/auth.rs index adb43093382..50b148bbf71 100644 --- a/rs/moq-relay/benches/auth.rs +++ b/rs/moq-relay/benches/auth.rs @@ -1,5 +1,5 @@ //! Revalidate one token while sweeping unrelated sessions and held credentials, and -//! cycle one verification slot while sweeping the waiting tokens and the slot count. +//! drain token batches while sweeping offered concurrency and payload size. //! The real relay/controller runs over loopback TCP; timings include one AUTH round trip. #![allow(clippy::disallowed_methods, clippy::disallowed_types)] @@ -145,14 +145,12 @@ fn refresh(c: &mut Criterion) { group.finish(); } -/// One session with `verifications` slots held by an unanswered decider and `waiting` -/// tokens queued behind them. Each round refuses the oldest held check, which sends the -/// oldest waiting token, and presents one more so the queue keeps its depth. +/// One real session; each measured batch offers tokens before the decider drains them, +/// so requests beyond the fixed production window wait for a verification slot. struct Queue { _connection: moq_tokio::Connection, auth: moq_tokio::connection::auth::Handle, admissions: auth::Admissions, - held: std::collections::VecDeque, presented: usize, tasks: Vec>, } @@ -166,10 +164,9 @@ impl Drop for Queue { } impl Queue { - async fn new(verifications: usize, waiting: usize) -> Self { + async fn new() -> Self { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let (auth, mut admissions) = auth::Auth::embedded("bench"); - let auth = auth.with_verifications(verifications); let mut config = moq_tokio::listen::Config::default(); config.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); let mut server = config.init(Default::default()).unwrap().listen().await.unwrap(); @@ -193,46 +190,33 @@ impl Queue { let admission = admissions.next().await.unwrap(); admission.grant(lease::Consumer::fixed(grant("initial"))); }); - let mut queue = Self { + Self { auth: connection.auth(), _connection: connection, admissions, - held: Default::default(), presented: 0, tasks: vec![listener], - }; - for _ in 0..verifications { - queue.present(); - let admission = queue.admissions.next().await.unwrap(); - queue.held.push_back(admission); - } - for _ in 0..waiting { - queue.present(); } - // Let the waiting tokens reach the relay's queue before measuring. - tokio::time::sleep(Duration::from_millis(50)).await; - queue } - fn present(&mut self) { - let auth = self.auth.clone(); - let token = format!("token{}", self.presented); - self.presented += 1; - self.tasks.push(tokio::spawn(async move { - let _ = auth.add(token).await; - })); - } - - async fn round(&mut self) { - self.held - .pop_front() - .unwrap() - .refuse(auth::Error::Refused(Default::default())); - self.present(); - let admission = self.admissions.next().await.unwrap(); - self.held.push_back(admission); - // Refused tokens finish; drop their handles so the table stays flat. - self.tasks.retain(|task| !task.is_finished()); + async fn round(&mut self, offered: usize, bytes: usize) { + let mut tasks = Vec::with_capacity(offered); + for _ in 0..offered { + let auth = self.auth.clone(); + let token = format!("{:0>width$}", self.presented, width = bytes); + self.presented += 1; + tasks.push(tokio::spawn(async move { auth.add(token).await })); + } + for _ in 0..offered { + self.admissions + .next() + .await + .unwrap() + .refuse(auth::Error::Refused(Default::default())); + } + for task in tasks { + assert!(task.await.unwrap().is_err()); + } } } @@ -245,15 +229,15 @@ fn queue(c: &mut Criterion) { group.sample_size(10); group.warm_up_time(Duration::from_millis(100)); group.measurement_time(Duration::from_millis(300)); - // QMux grants 100 bidirectional streams, and every held or waiting token holds one. - for verifications in [1, 8, 32] { - for waiting in [0, 16, 48] { - let mut fixture = runtime.block_on(Queue::new(verifications, waiting)); + for offered in [1, 128, 256] { + for bytes in [16, 256, 4096] { + let mut fixture = runtime.block_on(Queue::new()); + group.throughput(criterion::Throughput::Elements(offered as u64)); group.bench_with_input( - BenchmarkId::new(verifications.to_string(), waiting), - &(verifications, waiting), - |b, _| { - b.iter(|| runtime.block_on(fixture.round())); + BenchmarkId::new(offered.to_string(), bytes), + &(offered, bytes), + |b, &(offered, bytes)| { + b.iter(|| runtime.block_on(fixture.round(offered, bytes))); }, ); } diff --git a/rs/moq-relay/src/acme.rs b/rs/moq-relay/src/acme.rs index 46a4be65aab..3d38c53ed89 100644 --- a/rs/moq-relay/src/acme.rs +++ b/rs/moq-relay/src/acme.rs @@ -1,49 +1,13 @@ -//! Automatic certificates from an ACME directory, Let's Encrypt by default. -//! -//! [rustls-acme](rustls_acme) owns the account, order, and renewal loop and -//! answers TLS-ALPN-01 challenges from its certificate resolver. The relay hands -//! that resolver to every TLS listener, so QUIC, HTTPS, and every worker serve -//! the same certificate and pick up a renewal on their next handshake. +//! Automatic certificate configuration, enabled by the default `acme` feature. -use std::{ - io, - net::SocketAddr, - path::{Path, PathBuf}, - sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use anyhow::Context as _; -use futures::StreamExt; -use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; -use rustls_acme::{AcmeState, EventOk}; use serde::{Deserialize, Serialize}; +use std::{path::PathBuf, time::Duration}; use url::Url; -/// How long the first issuance may take by default before startup fails. -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5 * 60); - -/// How long a validation connection may take to finish its handshake. -const CHALLENGE_TIMEOUT: Duration = Duration::from_secs(10); - -/// How long each systemd start-timeout extension lasts while issuance is pending. -#[cfg(unix)] -const EXTEND_USEC: u32 = 90_000_000; - -/// How often the start timeout is extended, well inside [`EXTEND_USEC`]. -const EXTEND_INTERVAL: Duration = Duration::from_secs(30); - -/// Challenge handshakes in flight at once; connections past this are closed unanswered, -/// so a flood cannot exhaust descriptors. Far above the handful of vantage points a -/// directory validates from, so idle scanners do not crowd the validator out. -const CHALLENGE_CONCURRENCY: usize = 1024; - -/// Failed validations the first issuance tolerates within [`FAILURE_WINDOW`] before -/// refusing to start, leaving headroom under Let's Encrypt's limit of 5 per hostname per hour. -const FAILURE_LIMIT: usize = 3; - -/// How long a failed validation counts against [`FAILURE_LIMIT`], matching Let's Encrypt's window. -const FAILURE_WINDOW: Duration = Duration::from_secs(60 * 60); +#[cfg(feature = "acme")] +mod client; +#[cfg(feature = "acme")] +pub(crate) use client::Acme; /// Automatic certificates from an ACME directory, enabled by `acme.domains`. #[serde_with::serde_as] @@ -79,10 +43,15 @@ pub struct Config { #[serde_as(as = "serde_with::OneOrMany<_>")] pub contact: Vec, - /// Directory that keeps the account key and certificate across restarts, made owner-only. - #[usage(long = "acme-dir", name = "acme-dir", env = "MOQ_ACME_DIR", setting = "acme.dir")] + /// Cache directory required when domains are configured; keeps the account key and certificate private. + #[usage( + long = "acme-cache", + name = "acme-cache", + env = "MOQ_ACME_CACHE", + setting = "acme.cache" + )] #[usage(value_hint = usage_rs::ValueHint::DirPath)] - pub dir: Option, + pub cache: Option, /// The ACME directory URL, Let's Encrypt production when unset. #[usage( @@ -101,6 +70,12 @@ pub struct Config { #[usage(long = "acme-timeout", env = "MOQ_ACME_TIMEOUT", setting = "acme.timeout")] #[serde(default, rename = "__cli_timeout", skip_serializing_if = "Option::is_none")] timeout_arg: Option, + #[usage(long = "acme-dir", env = "MOQ_ACME_DIR", hide = true)] + #[serde(skip)] + legacy_dir: Option, + #[usage(skip)] + #[serde(rename = "dir", skip_serializing)] + removed_dir: Option, } impl Config { @@ -110,362 +85,27 @@ impl Config { .or(self.timeout) } - /// Validate against the listeners it must serve through and start the client, or `None` when unconfigured. - pub(crate) fn init( - &self, - listen: &moq_tokio::tls::Listen, - https: &crate::web::Https, - ) -> anyhow::Result> { - if self.domains.is_empty() { - anyhow::ensure!( - self.contact.is_empty() && self.dir.is_none() && self.directory.is_none() && self.timeout().is_none(), - "acme settings require acme.domains" - ); - return Ok(None); - } - if cfg!(not(unix)) { - anyhow::bail!("acme is only supported on unix, where acme.dir can be kept owner-only"); - } - // One canonical set, so reordering or recasing the same names reuses the cached certificate. - let mut domains: Vec = self.domains.iter().map(|domain| domain.to_ascii_lowercase()).collect(); - domains.sort(); - domains.dedup(); - for domain in &domains { - anyhow::ensure!( - !domain.contains('*'), - "acme.domains cannot include the wildcard {domain}: TLS-ALPN-01 cannot issue one" - ); - anyhow::ensure!( - matches!(ServerName::try_from(domain.as_str()), Ok(ServerName::DnsName(_))), - "acme.domains entry {domain:?} is not a DNS name" - ); - } - anyhow::ensure!( - listen.cert.is_empty() && listen.key.is_empty() && listen.generate.is_empty(), - "acme cannot be combined with listen.tls.cert, listen.tls.key, or listen.tls.generate" - ); - anyhow::ensure!( - https.listen.is_some(), - "acme requires web.https.listen: the directory validates each domain over TLS-ALPN-01 on port 443" - ); - anyhow::ensure!( - https.cert.is_empty() && https.key.is_empty(), - "acme cannot be combined with web.https.cert or web.https.key" - ); - let dir = self - .dir - .clone() - .context("acme requires acme.dir to keep the account and certificate across restarts")?; - create_private_dir(&dir).with_context(|| format!("failed to prepare acme.dir {}", dir.display()))?; - - let directory = self - .directory - .as_ref() - .map_or(rustls_acme::acme::LETS_ENCRYPT_PRODUCTION_DIRECTORY, Url::as_str); - let cache = Cache { dir }; - let failures = Failures { - path: cache.path("failures", &domains, directory), - }; - let state = rustls_acme::AcmeConfig::new(&domains) - .contact(self.contact.iter().map(Url::as_str)) - .directory(directory) - .cache(cache) - .state(); - - Ok(Some(Acme { - state, - failures, - domains: domains.join(", "), - timeout: self.timeout().unwrap_or(DEFAULT_TIMEOUT), - })) - } -} - -/// The ACME client: the certificate resolver plus the order and renewal loop behind it. -pub(crate) struct Acme { - state: AcmeState, - failures: Failures, - domains: String, - timeout: Duration, -} - -impl Acme { - /// The resolver every TLS listener serves from, which also answers the challenges. - pub(crate) fn resolver(&self) -> Arc { - self.state.resolver() - } - - /// Serve only the TLS-ALPN-01 challenge on `listener` until a certificate is deployed. - /// - /// A valid cached certificate deploys at once with no challenge. Otherwise this - /// orders one, failing with the last ACME error once the startup budget runs out - /// or once [`FAILURE_LIMIT`] validations failed within the hour, counting earlier - /// starts: a supervisor restarting the relay would otherwise spend the rest of - /// the directory's allowance and lock the hostname out for an hour. - pub(crate) async fn issue(&mut self, listener: &std::net::TcpListener) -> anyhow::Result<()> { - let addr = listener.local_addr()?; - let mut failed = self.failures.recent()?; - if failed.len() >= FAILURE_LIMIT { - return Err(self.failures.refuse(&failed, &self.domains, addr, None)); - } - - // A second descriptor for the same socket, so the web server keeps the - // original and takes over the queue once this returns. - let listener = tokio::net::TcpListener::from_std(listener.try_clone()?)?; - let challenge = tokio_rustls::TlsAcceptor::from(self.state.challenge_rustls_config()); - let mut handshakes = tokio::task::JoinSet::new(); - let health = moq_tokio::accept::Health::new("acme"); - // Readiness waits on issuance, so keep extending systemd's start timeout - // rather than letting the unit's default kill the relay inside the budget. - let mut extend = moq_clock::interval(EXTEND_INTERVAL); - let deadline = moq_clock::sleep(self.timeout); - tokio::pin!(deadline); - let mut last = None; - let mut issued = false; - - loop { - tokio::select! { - event = self.state.next() => match event.context("ACME client stopped")? { - Ok(EventOk::DeployedCachedCert) => { - tracing::info!(domains = %self.domains, "serving the cached ACME certificate"); - return self.failures.clear(); - } - // Not ready until it is saved: a certificate lost on restart would be ordered again. - Ok(EventOk::DeployedNewCert) => { - tracing::info!(domains = %self.domains, "issued an ACME certificate"); - issued = true; - } - Ok(EventOk::CertCacheStore) if issued => return self.failures.clear(), - Ok(event) => tracing::debug!(?event, "ACME"), - Err(rustls_acme::EventError::CertCacheStore(err)) => { - return Err(anyhow::Error::new(err).context("failed to save the issued ACME certificate in acme.dir")); - } - // The directory marked an authorization invalid: one failed validation. - Err(err @ rustls_acme::EventError::Order(rustls_acme::OrderError::BadAuth(_))) => { - failed = self.failures.record()?; - tracing::warn!(%err, domains = %self.domains, failed = failed.len(), "ACME validation failed"); - if failed.len() >= FAILURE_LIMIT { - return Err(self.failures.refuse(&failed, &self.domains, addr, Some(&err.to_string()))); - } - last = Some(err.to_string()); - } - Err(err) => { - tracing::warn!(%err, domains = %self.domains, "ACME issuance failed; retrying"); - last = Some(err.to_string()); - } - }, - accepted = listener.accept() => match accepted { - Ok((stream, _)) => { - health.accepted(); - if handshakes.len() < CHALLENGE_CONCURRENCY { - let handshake = challenge.accept(stream); - // The validator only needs the handshake, then closes. - handshakes.spawn(async move { - let _ = moq_clock::timeout(CHALLENGE_TIMEOUT, handshake).await; - }); - } - } - Err(err) => { - if let Some(pause) = health.failed(&err) { - moq_clock::sleep(pause).await; - } - } - }, - Some(_) = handshakes.join_next() => {} - _ = extend.tick() => { - #[cfg(unix)] - let _ = sd_notify::notify(&[sd_notify::NotifyState::ExtendTimeoutUsec(EXTEND_USEC)]); - } - _ = &mut deadline => { - let last = last.as_deref().unwrap_or("no response from the directory"); - anyhow::bail!( - "no ACME certificate for {} within {}: {last}; the directory validates each domain over TLS-ALPN-01 on port 443, and web.https.listen is {addr}", - self.domains, - humantime::format_duration(self.timeout), - ); - } - } - } - } - - /// Renew in the background forever, keeping the current certificate serving through failures. - /// - /// Never returns: the event stream is infinite, so its end panics, which stops - /// the relay through its critical tasks instead of silently ending renewal. - pub(crate) async fn run(mut self) { - loop { - let event = self.state.next().await.expect("the ACME event stream ended"); - match event { - Ok(EventOk::DeployedNewCert) => tracing::info!(domains = %self.domains, "renewed the ACME certificate"), - Ok(event) => tracing::debug!(?event, "ACME"), - Err(err) => tracing::error!( - %err, - domains = %self.domains, - "ACME renewal failed; still serving the previous certificate" - ), - } - } - } -} - -/// Create `dir` if needed and restrict it to the owner, since it holds private keys. -fn create_private_dir(dir: &Path) -> io::Result<()> { - std::fs::create_dir_all(dir)?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; - } - Ok(()) -} - -/// The failed validations recorded in `acme.dir`, so restarts share one allowance. -/// -/// One Unix timestamp per line, keyed like the certificate it was trying to issue. -struct Failures { - path: PathBuf, -} - -impl Failures { - /// The failures within [`FAILURE_WINDOW`], oldest first. - fn recent(&self) -> anyhow::Result> { - let Some(contents) = Cache::read(&self.path)? else { - return Ok(Vec::new()); - }; - let cutoff = SystemTime::from(moq_clock::wall()) - FAILURE_WINDOW; - let mut recent = Vec::new(); - for line in String::from_utf8_lossy(&contents).lines() { - let secs: u64 = line - .parse() - .with_context(|| format!("malformed ACME failure record in {}", self.path.display()))?; - let at = UNIX_EPOCH + Duration::from_secs(secs); - if at > cutoff { - recent.push(at); - } - } - recent.sort(); - Ok(recent) - } - - /// Record a failure now, returning the recent ones including it. - fn record(&self) -> anyhow::Result> { - let mut recent = self.recent()?; - recent.push(SystemTime::from(moq_clock::wall())); - let lines: String = recent - .iter() - .map(|at| format!("{}\n", at.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs())) - .collect(); - Cache::write(&self.path, lines.as_bytes())?; - Ok(recent) - } - - fn clear(&self) -> anyhow::Result<()> { - match std::fs::remove_file(&self.path) { - Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err.into()), - _ => Ok(()), + pub(crate) fn deprecated(&self) -> moq_tokio::cli::Deprecated { + let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy_dir.is_some() { + found.flag("--acme-dir", Some("MOQ_ACME_DIR"), "--acme-cache"); } - } - - /// The startup error once `failed` reaches [`FAILURE_LIMIT`]. - fn refuse(&self, failed: &[SystemTime], domains: &str, addr: SocketAddr, last: Option<&str>) -> anyhow::Error { - let retry = failed - .first() - .map_or(SystemTime::from(moq_clock::wall()), |oldest| *oldest + FAILURE_WINDOW); - let last = last.map(|last| format!("; last error: {last}")).unwrap_or_default(); - anyhow::anyhow!( - "ACME validation for {domains} failed {} times in the last hour; stopping before the directory's failed-validation limit locks the hostname out (Let's Encrypt allows 5 failed validations per hostname per hour). Check that each domain resolves to this relay and that TCP port 443 reaches web.https.listen ({addr}), then restart after {} or delete {}{last}", - failed.len(), - humantime::format_rfc3339_seconds(retry), - self.path.display(), - ) - } -} - -/// Keeps the account key and certificate in `dir`, one owner-only file each. -/// -/// Files are keyed by a hash of the contacts or domains plus the directory URL, so -/// a certificate missing a configured domain or issued by another directory is -/// simply not found and gets reissued. An expired one counts as missing too. -struct Cache { - dir: PathBuf, -} - -impl Cache { - fn path(&self, kind: &str, names: &[String], directory: &str) -> PathBuf { - let mut digest = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); - for name in names { - digest.update(name.as_bytes()); - digest.update(&[0]); + if self.removed_dir.is_some() { + found.toml("[acme] dir", "[acme] cache", None); } - digest.update(directory.as_bytes()); - self.dir.join(format!("{kind}-{}", hex::encode(digest.finish()))) + found } - fn read(path: &Path) -> io::Result>> { - match std::fs::read(path) { - Ok(contents) => Ok(Some(contents)), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } - - /// Replace `path` atomically, so a crash mid-write never leaves a torn key behind. - fn write(path: &Path, contents: &[u8]) -> io::Result<()> { - use std::io::Write; - - let partial = path.with_extension("partial"); - let mut options = std::fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); - let mut file = options.open(&partial)?; - file.write_all(contents)?; - file.sync_all()?; - std::fs::rename(partial, path) - } -} - -/// Whether the leaf in a cached key-and-chain PEM is valid right now. -fn current(pem: &[u8]) -> bool { - let Some(Ok(leaf)) = CertificateDer::pem_slice_iter(pem).next() else { - return false; - }; - x509_parser::parse_x509_certificate(&leaf).is_ok_and(|(_, cert)| cert.validity().is_valid()) -} - -#[async_trait::async_trait] -impl rustls_acme::CertCache for Cache { - type EC = io::Error; - - async fn load_cert(&self, domains: &[String], directory: &str) -> io::Result>> { - let cached = Self::read(&self.path("cert", domains, directory))?; - Ok(cached.filter(|pem| { - let current = current(pem); - if !current { - tracing::info!("the cached ACME certificate is expired or unreadable; reissuing"); - } - current - })) - } - - async fn store_cert(&self, domains: &[String], directory: &str, cert: &[u8]) -> io::Result<()> { - Self::write(&self.path("cert", domains, directory), cert) - } -} - -#[async_trait::async_trait] -impl rustls_acme::AccountCache for Cache { - type EA = io::Error; - - async fn load_account(&self, contact: &[String], directory: &str) -> io::Result>> { - Self::read(&self.path("account", contact, directory)) - } - - async fn store_account(&self, contact: &[String], directory: &str, account: &[u8]) -> io::Result<()> { - Self::write(&self.path("account", contact, directory), account) + #[cfg(not(feature = "acme"))] + pub(crate) fn refuse_unsupported(&self) -> anyhow::Result<()> { + anyhow::ensure!( + self.domains.is_empty() + && self.contact.is_empty() + && self.cache.is_none() + && self.directory.is_none() + && self.timeout().is_none(), + "acme settings require the acme feature" + ); + Ok(()) } } - -#[cfg(all(test, unix))] -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod tests; diff --git a/rs/moq-relay/src/acme/client.rs b/rs/moq-relay/src/acme/client.rs new file mode 100644 index 00000000000..9fcc31bae02 --- /dev/null +++ b/rs/moq-relay/src/acme/client.rs @@ -0,0 +1,409 @@ +//! Automatic certificates from an ACME directory, Let's Encrypt by default. +//! +//! [rustls-acme](rustls_acme) owns the account, order, and renewal loop and +//! answers TLS-ALPN-01 challenges from its certificate resolver. The relay hands +//! that resolver to every TLS listener, so QUIC, HTTPS, and every worker serve +//! the same certificate and pick up a renewal on their next handshake. + +use std::{ + io, + net::SocketAddr, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::Context as _; +use futures::StreamExt; +use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; +use rustls_acme::{AcmeState, EventOk}; +use url::Url; + +/// How long the first issuance may take by default before startup fails. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +/// How long a validation connection may take to finish its handshake. +const CHALLENGE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long each systemd start-timeout extension lasts while issuance is pending. +#[cfg(unix)] +const EXTEND_USEC: u32 = 90_000_000; + +/// How often the start timeout is extended, well inside [`EXTEND_USEC`]. +const EXTEND_INTERVAL: Duration = Duration::from_secs(30); + +/// Challenge handshakes in flight at once; connections past this are closed unanswered, +/// so a flood cannot exhaust descriptors. Far above the handful of vantage points a +/// directory validates from, so idle scanners do not crowd the validator out. +const CHALLENGE_CONCURRENCY: usize = 1024; + +/// Failed validations the first issuance tolerates within [`FAILURE_WINDOW`] before +/// refusing to start, leaving headroom under Let's Encrypt's limit of 5 per hostname per hour. +const FAILURE_LIMIT: usize = 3; + +/// How long a failed validation counts against [`FAILURE_LIMIT`], matching Let's Encrypt's window. +const FAILURE_WINDOW: Duration = Duration::from_secs(60 * 60); + +use super::Config; + +impl Config { + /// Validate against the listeners it must serve through and start the client, or `None` when unconfigured. + pub(crate) fn init( + &self, + listen: &moq_tokio::tls::Listen, + https: &crate::web::Https, + ) -> anyhow::Result> { + if self.domains.is_empty() { + anyhow::ensure!( + self.contact.is_empty() && self.cache.is_none() && self.directory.is_none() && self.timeout().is_none(), + "acme settings require acme.domains" + ); + return Ok(None); + } + if cfg!(not(unix)) { + anyhow::bail!("acme is only supported on unix, where acme.cache can be kept owner-only"); + } + // One canonical set, so reordering or recasing the same names reuses the cached certificate. + let mut domains: Vec = self.domains.iter().map(|domain| domain.to_ascii_lowercase()).collect(); + domains.sort(); + domains.dedup(); + for domain in &domains { + anyhow::ensure!( + !domain.contains('*'), + "acme.domains cannot include the wildcard {domain}: TLS-ALPN-01 cannot issue one" + ); + anyhow::ensure!( + matches!(ServerName::try_from(domain.as_str()), Ok(ServerName::DnsName(_))), + "acme.domains entry {domain:?} is not a DNS name" + ); + } + anyhow::ensure!( + listen.cert.is_empty() && listen.key.is_empty() && listen.generate.is_empty(), + "acme cannot be combined with listen.tls.cert, listen.tls.key, or listen.tls.generate" + ); + anyhow::ensure!( + https.listen.is_some(), + "acme requires web.https.listen: the directory validates each domain over TLS-ALPN-01 on port 443" + ); + anyhow::ensure!( + https.cert.is_empty() && https.key.is_empty(), + "acme cannot be combined with web.https.cert or web.https.key" + ); + let dir = self + .cache + .clone() + .context("acme requires acme.cache to keep the account and certificate across restarts")?; + create_private_dir(&dir).with_context(|| format!("failed to prepare acme.cache {}", dir.display()))?; + + let directory = self + .directory + .as_ref() + .map_or(rustls_acme::acme::LETS_ENCRYPT_PRODUCTION_DIRECTORY, Url::as_str); + let cache = Cache { dir }; + let failures = Failures { + path: cache.path("failures", &domains, directory), + }; + let state = rustls_acme::AcmeConfig::new(&domains) + .contact(self.contact.iter().map(Url::as_str)) + .directory(directory) + .cache(cache) + .state(); + + Ok(Some(Acme { + state, + failures, + domains: domains.join(", "), + timeout: self.timeout().unwrap_or(DEFAULT_TIMEOUT), + })) + } +} + +/// The ACME client: the certificate resolver plus the order and renewal loop behind it. +pub(crate) struct Acme { + state: AcmeState, + failures: Failures, + domains: String, + timeout: Duration, +} + +impl Acme { + /// The resolver every TLS listener serves from, which also answers the challenges. + pub(crate) fn resolver(&self) -> Arc { + self.state.resolver() + } + + /// Serve only the TLS-ALPN-01 challenge on `listener` until a certificate is deployed. + /// + /// A valid cached certificate deploys at once with no challenge. Otherwise this + /// orders one, failing with the last ACME error once the startup budget runs out + /// or once [`FAILURE_LIMIT`] validations failed within the hour, counting earlier + /// starts: a supervisor restarting the relay would otherwise spend the rest of + /// the directory's allowance and lock the hostname out for an hour. + pub(crate) async fn issue(&mut self, listener: &std::net::TcpListener) -> anyhow::Result<()> { + let addr = listener.local_addr()?; + let mut failed = self.failures.recent()?; + if failed.len() >= FAILURE_LIMIT { + return Err(self.failures.refuse(&failed, &self.domains, addr, None)); + } + + // A second descriptor for the same socket, so the web server keeps the + // original and takes over the queue once this returns. + let listener = tokio::net::TcpListener::from_std(listener.try_clone()?)?; + let challenge = tokio_rustls::TlsAcceptor::from(self.state.challenge_rustls_config()); + let mut handshakes = tokio::task::JoinSet::new(); + let health = moq_tokio::accept::Health::new("acme"); + // Readiness waits on issuance, so keep extending systemd's start timeout + // rather than letting the unit's default kill the relay inside the budget. + let mut extend = moq_clock::interval(EXTEND_INTERVAL); + let deadline = moq_clock::sleep(self.timeout); + tokio::pin!(deadline); + let mut last = None; + let mut issued = false; + + loop { + tokio::select! { + event = self.state.next() => match event.context("ACME client stopped")? { + Ok(EventOk::DeployedCachedCert) => { + tracing::info!(domains = %self.domains, "serving the cached ACME certificate"); + return self.failures.clear(); + } + // Not ready until it is saved: a certificate lost on restart would be ordered again. + Ok(EventOk::DeployedNewCert) => { + tracing::info!(domains = %self.domains, "issued an ACME certificate"); + issued = true; + } + Ok(EventOk::CertCacheStore) if issued => return self.failures.clear(), + Ok(event) => tracing::debug!(?event, "ACME"), + Err(rustls_acme::EventError::CertCacheStore(err)) => { + return Err(anyhow::Error::new(err).context("failed to save the issued ACME certificate in acme.cache")); + } + // The directory marked an authorization invalid: one failed validation. + Err(err @ rustls_acme::EventError::Order(rustls_acme::OrderError::BadAuth(_))) => { + failed = self.failures.record()?; + tracing::warn!(%err, domains = %self.domains, failed = failed.len(), "ACME validation failed"); + if failed.len() >= FAILURE_LIMIT { + return Err(self.failures.refuse(&failed, &self.domains, addr, Some(&err.to_string()))); + } + last = Some(err.to_string()); + } + Err(err) => { + tracing::warn!(%err, domains = %self.domains, "ACME issuance failed; retrying"); + last = Some(err.to_string()); + } + }, + accepted = listener.accept() => match accepted { + Ok((stream, _)) => { + health.accepted(); + if handshakes.len() < CHALLENGE_CONCURRENCY { + let handshake = challenge.accept(stream); + // The validator only needs the handshake, then closes. + handshakes.spawn(async move { + let _ = moq_clock::timeout(CHALLENGE_TIMEOUT, handshake).await; + }); + } + } + Err(err) => { + if let Some(pause) = health.failed(&err) { + moq_clock::sleep(pause).await; + } + } + }, + Some(_) = handshakes.join_next() => {} + _ = extend.tick() => { + #[cfg(unix)] + let _ = sd_notify::notify(&[sd_notify::NotifyState::ExtendTimeoutUsec(EXTEND_USEC)]); + } + _ = &mut deadline => { + let last = last.as_deref().unwrap_or("no response from the directory"); + anyhow::bail!( + "no ACME certificate for {} within {}: {last}; the directory validates each domain over TLS-ALPN-01 on port 443, and web.https.listen is {addr}", + self.domains, + humantime::format_duration(self.timeout), + ); + } + } + } + } + + /// Renew in the background forever, keeping the current certificate serving through failures. + /// + /// Never returns: the event stream is infinite, so its end panics, which stops + /// the relay through its critical tasks instead of silently ending renewal. + pub(crate) async fn run(mut self) { + loop { + let event = self.state.next().await.expect("the ACME event stream ended"); + match event { + Ok(EventOk::DeployedNewCert) => tracing::info!(domains = %self.domains, "renewed the ACME certificate"), + Ok(event) => tracing::debug!(?event, "ACME"), + Err(err) => tracing::error!( + %err, + domains = %self.domains, + "ACME renewal failed; still serving the previous certificate" + ), + } + } + } +} + +/// Create `dir` if needed and restrict it to the owner, since it holds private keys. +fn create_private_dir(dir: &Path) -> io::Result<()> { + std::fs::create_dir_all(dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +/// The failed validations recorded in `acme.cache`, so restarts share one allowance. +/// +/// One Unix timestamp per line, keyed like the certificate it was trying to issue. +struct Failures { + path: PathBuf, +} + +impl Failures { + /// The failures within [`FAILURE_WINDOW`], oldest first. + fn recent(&self) -> anyhow::Result> { + let Some(contents) = Cache::read(&self.path)? else { + return Ok(Vec::new()); + }; + let cutoff = SystemTime::from(moq_clock::wall()) - FAILURE_WINDOW; + let mut recent = Vec::new(); + for line in String::from_utf8_lossy(&contents).lines() { + let secs: u64 = line + .parse() + .with_context(|| format!("malformed ACME failure record in {}", self.path.display()))?; + let at = UNIX_EPOCH + Duration::from_secs(secs); + if at > cutoff { + recent.push(at); + } + } + recent.sort(); + Ok(recent) + } + + /// Record a failure now, returning the recent ones including it. + fn record(&self) -> anyhow::Result> { + let mut recent = self.recent()?; + recent.push(SystemTime::from(moq_clock::wall())); + let lines: String = recent + .iter() + .map(|at| format!("{}\n", at.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs())) + .collect(); + Cache::write(&self.path, lines.as_bytes())?; + Ok(recent) + } + + fn clear(&self) -> anyhow::Result<()> { + match std::fs::remove_file(&self.path) { + Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err.into()), + _ => Ok(()), + } + } + + /// The startup error once `failed` reaches [`FAILURE_LIMIT`]. + fn refuse(&self, failed: &[SystemTime], domains: &str, addr: SocketAddr, last: Option<&str>) -> anyhow::Error { + let retry = failed + .first() + .map_or(SystemTime::from(moq_clock::wall()), |oldest| *oldest + FAILURE_WINDOW); + let last = last.map(|last| format!("; last error: {last}")).unwrap_or_default(); + anyhow::anyhow!( + "ACME validation for {domains} failed {} times in the last hour; stopping before the directory's failed-validation limit locks the hostname out (Let's Encrypt allows 5 failed validations per hostname per hour). Check that each domain resolves to this relay and that TCP port 443 reaches web.https.listen ({addr}), then restart after {} or delete {}{last}", + failed.len(), + humantime::format_rfc3339_seconds(retry), + self.path.display(), + ) + } +} + +/// Keeps the account key and certificate in `dir`, one owner-only file each. +/// +/// Files are keyed by a hash of the contacts or domains plus the directory URL, so +/// a certificate missing a configured domain or issued by another directory is +/// simply not found and gets reissued. An expired one counts as missing too. +struct Cache { + dir: PathBuf, +} + +impl Cache { + fn path(&self, kind: &str, names: &[String], directory: &str) -> PathBuf { + let mut digest = aws_lc_rs::digest::Context::new(&aws_lc_rs::digest::SHA256); + for name in names { + digest.update(name.as_bytes()); + digest.update(&[0]); + } + digest.update(directory.as_bytes()); + self.dir.join(format!("{kind}-{}", hex::encode(digest.finish()))) + } + + fn read(path: &Path) -> io::Result>> { + match std::fs::read(path) { + Ok(contents) => Ok(Some(contents)), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + /// Replace `path` atomically, so a crash mid-write never leaves a torn key behind. + fn write(path: &Path, contents: &[u8]) -> io::Result<()> { + use std::io::Write; + + let partial = path.with_extension("partial"); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let mut file = options.open(&partial)?; + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(partial, path) + } +} + +/// Whether the leaf in a cached key-and-chain PEM is valid right now. +fn current(pem: &[u8]) -> bool { + let Some(Ok(leaf)) = CertificateDer::pem_slice_iter(pem).next() else { + return false; + }; + x509_parser::parse_x509_certificate(&leaf).is_ok_and(|(_, cert)| cert.validity().is_valid()) +} + +#[async_trait::async_trait] +impl rustls_acme::CertCache for Cache { + type EC = io::Error; + + async fn load_cert(&self, domains: &[String], directory: &str) -> io::Result>> { + let cached = Self::read(&self.path("cert", domains, directory))?; + Ok(cached.filter(|pem| { + let current = current(pem); + if !current { + tracing::info!("the cached ACME certificate is expired or unreadable; reissuing"); + } + current + })) + } + + async fn store_cert(&self, domains: &[String], directory: &str, cert: &[u8]) -> io::Result<()> { + Self::write(&self.path("cert", domains, directory), cert) + } +} + +#[async_trait::async_trait] +impl rustls_acme::AccountCache for Cache { + type EA = io::Error; + + async fn load_account(&self, contact: &[String], directory: &str) -> io::Result>> { + Self::read(&self.path("account", contact, directory)) + } + + async fn store_account(&self, contact: &[String], directory: &str, account: &[u8]) -> io::Result<()> { + Self::write(&self.path("account", contact, directory), account) + } +} + +#[cfg(all(test, unix))] +#[allow(clippy::disallowed_methods, clippy::disallowed_types)] +#[path = "tests.rs"] +mod tests; diff --git a/rs/moq-relay/src/acme/tests.rs b/rs/moq-relay/src/acme/tests.rs index 4358c8aaab0..8e4b647a47e 100644 --- a/rs/moq-relay/src/acme/tests.rs +++ b/rs/moq-relay/src/acme/tests.rs @@ -18,6 +18,7 @@ use axum::{ response::{IntoResponse, Response}, routing::{get, post}, }; +use serde::Deserialize; use serde_json::{Value, json}; use super::*; @@ -365,7 +366,7 @@ fn relay_config(directory: &Url, dir: &Path, domains: &[&str]) -> RelayConfig { config.web.https.listen = Some("127.0.0.1:0".parse().unwrap()); config.auth.public = vec![moq_auth::Pattern::all()]; config.acme.domains = domains.iter().map(|domain| domain.to_string()).collect(); - config.acme.dir = Some(dir.to_path_buf()); + config.acme.cache = Some(dir.to_path_buf()); config.acme.directory = Some(directory.clone()); config } @@ -715,7 +716,7 @@ fn config_refuses_what_acme_cannot_serve() { let mut config = RelayConfig::default(); config.web.https.listen = Some("127.0.0.1:0".parse().unwrap()); config.acme.domains = vec![DOMAIN.into()]; - config.acme.dir = Some(dir.path().to_path_buf()); + config.acme.cache = Some(dir.path().to_path_buf()); config }; let init = |config: RelayConfig| { @@ -738,7 +739,7 @@ fn config_refuses_what_acme_cannot_serve() { config.web.https.cert = vec!["cert.pem".into()] }), ("wildcard", |config| config.acme.domains = vec!["*.relay.test".into()]), - ("acme.dir", |config| config.acme.dir = None), + ("acme.cache", |config| config.acme.cache = None), ("require acme.domains", |config| config.acme.domains.clear()), ]; for (expected, mutate) in cases { diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index 5258ac7a3a5..0b04eb59e35 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -24,7 +24,7 @@ const ATTACH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); /// Where every session's grant comes from. Exactly one of `url` and the public /// patterns is set; [`validate`](Self::validate) refuses anything else. #[serde_as] -#[derive(usage_rs::Args, Clone, Debug, Serialize, Deserialize)] +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(default)] #[non_exhaustive] @@ -72,16 +72,9 @@ pub struct Config { #[serde_as(as = "OneOrMany<_>")] pub public_publish: Vec, - /// Maximum in-band tokens one session has the auth server verifying at once, - /// counting checks still running past the admission timeout. Later tokens wait - /// for a slot. Defaults to 128. - #[usage( - long = "auth-verifications", - env = "MOQ_AUTH_VERIFICATIONS", - default = "128", - setting = "auth.verifications" - )] - pub verifications: usize, + #[usage(skip)] + #[serde(rename = "verifications", skip_serializing)] + pub(crate) removed_verifications: Option, /// The 0.14 flags and env vars #3688 removed, kept parsing but hidden. Never /// read as settings: [`deprecated`](Self::deprecated) names what replaced each. @@ -111,26 +104,7 @@ pub struct Config { pub(crate) tls: Option, } -impl Default for Config { - fn default() -> Self { - Self { - url: None, - public: Vec::new(), - public_subscribe: Vec::new(), - public_publish: Vec::new(), - verifications: VERIFICATIONS, - legacy: Legacy::default(), - key: None, - key_dir: None, - auth_api: None, - domains: None, - mtls_tier: None, - tls: None, - } - } -} - -/// The default for [`Config::verifications`]. +/// Maximum in-band token verifications one session may hold at its decider. const VERIFICATIONS: usize = 128; /// The `--auth-*` flags 0.14 had and #3688 removed, with their env vars: a relay @@ -138,6 +112,9 @@ const VERIFICATIONS: usize = 128; #[derive(Clone, Debug, Default, usage_rs::Args)] #[usage(unknown_flags = "error", args_override_self = false)] pub(crate) struct Legacy { + #[usage(long = "auth-verifications", env = "MOQ_AUTH_VERIFICATIONS", hide = true)] + verifications: Option, + #[usage(name = "auth-key", long = "auth-key", env = "MOQ_AUTH_KEY", hide = true)] key: Option, #[usage(name = "auth-key-dir", long = "auth-key-dir", env = "MOQ_AUTH_KEY_DIR", hide = true)] @@ -204,6 +181,22 @@ impl Config { /// ignored. pub fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy.verifications.is_some() { + found.changed( + "--auth-verifications", + Some("MOQ_AUTH_VERIFICATIONS"), + "the fixed per-session verification policy", + "the configurable verification window is removed", + ); + } + if self.removed_verifications.is_some() { + found.toml( + "[auth] verifications", + "the fixed per-session verification policy", + None, + ); + } + let legacy = &self.legacy; for (set, flag, env, toml, new) in [ ( @@ -285,6 +278,8 @@ impl Config { /// prefix `anon/`, and a pattern reads it as exactly the broadcast `anon`, so /// either silent reading would mislead someone upgrading. pub fn validate(&self, client_ca: bool) -> anyhow::Result<()> { + let deprecated = self.deprecated(); + anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); let flags = [ ("--auth-public", &self.public), ("--auth-public-publish", &self.public_publish), @@ -311,9 +306,6 @@ impl Config { ), _ => {} } - if self.verifications == 0 { - anyhow::bail!("--auth-verifications must be at least 1, or no in-band token is ever verified"); - } Ok(()) } @@ -340,8 +332,7 @@ impl Config { ), (None, None) => unreachable!("validated above"), }; - let (auth, admissions) = Auth::embedded(node); - let mut auth = auth.with_verifications(self.verifications); + let (mut auth, admissions) = Auth::embedded(node); auth.in_band = matches!(decider, Decider::Server(_)); decider.spawn(admissions); Ok(auth) @@ -758,7 +749,8 @@ pub(crate) fn error(reason: &lease::Reason) -> moq_net::Error { /// Work is dropped when the lease expires, narrows, or is revoked. A completed /// work future ends the lease with zero byte totals. A gateway that tracks /// transport totals can use [`Lease::ended`] and [`Lease::close`] directly. -pub async fn hold(mut lease: Lease, work: impl std::future::Future) -> Result { +#[cfg(test)] +pub(crate) async fn hold(mut lease: Lease, work: impl std::future::Future) -> Result { tokio::select! { biased; reason = lease.ended() => { @@ -851,6 +843,7 @@ pub struct Auth { /// Whether the decider can verify a token presented in band; public rules cannot. in_band: bool, /// How many in-band tokens one session may have the decider verifying at once. + #[cfg(test)] verifications: usize, } @@ -864,20 +857,26 @@ impl Auth { admissions: sender, node: Arc::from(node.into()), in_band: true, + #[cfg(test)] verifications: VERIFICATIONS, }; (auth, Admissions(receiver)) } - /// Verify at most `verifications` in-band tokens per session at once, as - /// `--auth-verifications` does for a configured `Auth`; later ones wait for a slot. - /// Panics on 0, which would leave every in-band token waiting forever. - pub fn with_verifications(mut self, verifications: usize) -> Self { - assert!(verifications > 0, "an Auth must verify at least one token at a time"); + #[cfg(test)] + fn with_verifications(mut self, verifications: usize) -> Self { + assert!(verifications > 0); self.verifications = verifications; self } + fn verifications(&self) -> usize { + #[cfg(test)] + return self.verifications; + #[cfg(not(test))] + VERIFICATIONS + } + /// An `Auth` that refuses every session a server or a public grant would have /// decided, admitting only what the relay decides for itself (a LAN peer). /// Must be called within a Tokio runtime, which drives the refusal decider. @@ -1057,6 +1056,7 @@ pub fn request_for(auth: &Auth, request: &moq_tokio::server::Request) -> Request } /// Build an auth request from the path, credential, and role carried in SETUP. +#[cfg(any(feature = "test-support", all(target_os = "linux", feature = "_uring")))] pub(crate) fn request_for_setup( auth: &Auth, setup: &moq_net::server::Handshake, @@ -1189,18 +1189,15 @@ mod tests { assert!(grant.publish.is_empty()); } - /// The verification concurrency reads from `[auth]` and refuses 0, - /// which would leave every in-band token waiting forever. + /// Removed explicit window values, including zero, are refused by name. #[test] - fn verifications_parse_and_refuse_zero() { - let config = config(Some("http://127.0.0.1:4440/"), &[]); - let parsed: Config = toml::from_str("url = \"http://127.0.0.1:4440/\"\nverifications = 4\n").unwrap(); - assert_eq!(parsed.verifications, 4); - let zero = Config { - verifications: 0, - ..config - }; - assert!(zero.validate(false).is_err()); + fn configurable_verifications_are_refused() { + for value in [0, 4] { + let parsed: Config = + toml::from_str(&format!("url = \"http://127.0.0.1:4440/\"\nverifications = {value}\n")).unwrap(); + let error = parsed.validate(false).unwrap_err(); + assert!(error.to_string().contains("verifications"), "{error}"); + } } /// Public rules grant a certificate what they grant anyone, so a client CA @@ -1575,3 +1572,7 @@ mod outcome_tests { assert_eq!(presence.sessions_ended, 1); } } + +#[cfg(test)] +#[allow(clippy::disallowed_methods, clippy::disallowed_types)] +mod admission_tests; diff --git a/rs/moq-relay/tests/integration/auth_embedder.rs b/rs/moq-relay/src/auth/admission_tests.rs similarity index 98% rename from rs/moq-relay/tests/integration/auth_embedder.rs rename to rs/moq-relay/src/auth/admission_tests.rs index ce2101783a5..500d770223c 100644 --- a/rs/moq-relay/tests/integration/auth_embedder.rs +++ b/rs/moq-relay/src/auth/admission_tests.rs @@ -1,7 +1,7 @@ //! Admission and lease behavior for a gateway embedding the relay. +use crate::{auth, cluster}; use moq_auth::{Grant, Pattern, Patterns, Role, Transport, lease}; -use moq_relay::{auth, cluster}; fn patterns(texts: &[&str]) -> Patterns { texts.iter().map(|text| text.parse::().unwrap()).collect() @@ -184,8 +184,10 @@ async fn gateway_admission_refuses_once_in_the_shared_stats_registry() { async fn embedded_expired_refusals_keep_verified_root_and_tier() { tokio::time::pause(); let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); - let mut config = moq_relay::stats::Config::default(); - config.enabled = true; + let config = crate::stats::Config { + enabled: true, + ..Default::default() + }; let producer = config.build(cluster.origin.clone()); let registry = producer.registry().clone(); let cluster = cluster.with_stats(producer); diff --git a/rs/moq-relay/src/auth/refresh.rs b/rs/moq-relay/src/auth/refresh.rs index 08c8bd10be1..bf268766d23 100644 --- a/rs/moq-relay/src/auth/refresh.rs +++ b/rs/moq-relay/src/auth/refresh.rs @@ -250,7 +250,7 @@ impl Refresh { let Some(source) = self.source.as_ref() else { return; }; - while self.pending.len() + self.settling.len() < source.auth.verifications { + while self.pending.len() + self.settling.len() < source.auth.verifications() { let Some(waiting) = self.waiting.pop_front() else { return; }; diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 4cf3900267f..8055205bef7 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -19,11 +19,14 @@ use url::Url; use crate::auth; +/// Dynamic cluster peer-list configuration. +pub mod api; + /// The request path prefix a LAN mesh dial presents, marking it as a peer rather /// than an ordinary publisher or viewer on the same listener. pub(crate) const CLUSTER_PATH: &str = "/.cluster"; -/// How often the relay re-checks an http(s) `--cluster-connect-api` endpoint. The +/// How often the relay re-checks an http(s) `--cluster-api-url` endpoint. The /// HTTP cache middleware suppresses the actual network round-trip while the cached /// list is still fresh (per the response's `Cache-Control`), so this is the floor /// on responsiveness, not on origin load: a tighter `max-age` means more of these @@ -31,7 +34,7 @@ pub(crate) const CLUSTER_PATH: &str = "/.cluster"; const CONNECT_API_POLL_INTERVAL: Duration = Duration::from_secs(30); /// One cluster peer to dial, as listed in [`Config::connect`] or returned -/// by a `connect_api` endpoint. +/// by an `api.url` endpoint. /// /// Accepts a URL string or an object with `url` plus /// policy: `cost` prices the link, `egress` declares this relay's own price to @@ -365,7 +368,7 @@ fn parse_peer_list(list: Vec, node: Option<&str>) -> anyhow::Result, - /// Fetch the list of peers to dial from an HTTP(S) URL or a local file, - /// reloading at runtime without a restart. The source returns a JSON array - /// of peers: bare URL strings and/or objects with `url`, `cost`, `egress`, - /// `token`, and `upstream`, exactly as [`Self::connect`] accepts, e.g. - /// `["https://a.pop.example/?cost=1", {"url": "https://b.pop.example/", "cost": 2}]`. - /// An http(s) URL is re-checked on a fixed cadence, with caching, conditional revalidation - /// (`ETag` / `Last-Modified`), and stale-if-error handled by the shared HTTP - /// cache client, so the response's `Cache-Control` controls how often a real - /// fetch hits the endpoint; a local path is watched via OS filesystem - /// notifications (with a periodic re-check fallback). This relay's own - /// [`Self::node`] value, when set, is sent as a `?node=` query param so the - /// server can return this node's peers. The relay keeps the last good list if - /// a fetch fails. Composes with [`Self::connect`]. - #[usage( - name = "cluster-connect-api", - long = "cluster-connect-api", - env = "MOQ_CLUSTER_CONNECT_API", - setting = "cluster.connect_api" - )] - pub connect_api: Option, + /// Dynamic peer-list source and its independent TLS trust. + #[usage(flatten)] + pub api: api::Config, - /// Trust these PEM roots for the peer API instead of the system roots. - /// The API hostname comes from its URL; mesh TLS trust is independent. + #[usage(long = "cluster-connect-api", env = "MOQ_CLUSTER_CONNECT_API", hide = true)] + #[serde(skip)] + pub(crate) legacy_api: Option, #[usage( - name = "cluster-connect-api-tls-root", long = "cluster-connect-api-tls-root", env = "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", - setting = "cluster.connect_api_tls_root" + hide = true )] - #[usage(value_hint = usage_rs::ValueHint::FilePath, extensions("pem", "crt", "cer"))] - #[serde_as(as = "serde_with::OneOrMany<_>")] - pub connect_api_tls_root: Vec, + #[serde(skip)] + pub(crate) legacy_api_root: Vec, + #[usage(skip)] + #[serde(rename = "connect_api", skip_serializing)] + pub(crate) removed_api: Option, + #[usage(skip)] + #[serde(rename = "connect_api_tls_root", skip_serializing)] + pub(crate) removed_api_root: Option, /// This relay's own externally-reachable URL (identity). Sent to - /// [`Self::connect_api`] as a `?node=` query param so the endpoint can return + /// [`Self::api`] as a `?node=` query param so the endpoint can return /// this node's peers, and advertised over mDNS when LAN discovery is on. On /// its own it neither opens nor accepts a connection. #[usage( @@ -838,6 +829,27 @@ impl Config { /// Released spellings this config was parsed from, each paired with what replaced it. pub fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut found = moq_tokio::cli::Deprecated::default(); + if self.legacy_api.is_some() { + found.flag( + "--cluster-connect-api", + Some("MOQ_CLUSTER_CONNECT_API"), + "--cluster-api-url", + ); + } + if !self.legacy_api_root.is_empty() { + found.flag( + "--cluster-connect-api-tls-root", + Some("MOQ_CLUSTER_CONNECT_API_TLS_ROOT"), + "--cluster-api-tls-root", + ); + } + if self.removed_api.is_some() { + found.toml("[cluster] connect_api", "[cluster.api] url", None); + } + if self.removed_api_root.is_some() { + found.toml("[cluster] connect_api_tls_root", "[cluster.api.tls] root", None); + } + if self.connect.iter().any(|peer| is_legacy_peer(peer.url())) { found.changed( "--cluster-connect", @@ -968,7 +980,7 @@ struct LanAuth { #[non_exhaustive] pub struct Options { /// Request caps for ordinary client sessions. - pub session: crate::limits::Session, + pub session: crate::limits::Client, /// Request caps and QUIC limits for authenticated peers and outbound cluster links. pub peer: crate::limits::Peer, /// Cluster identity, peers, and discovery. @@ -1119,11 +1131,6 @@ pub struct Cluster { /// view always points at the same session in the logs. connection_ids: Arc, - /// Client TLS config used to build the `--cluster-connect-api` HTTP client, so - /// peer-list fetches present the same cluster cert the QUIC dials do. `Arc` so - /// cloning a `Cluster` per connection stays cheap. - client_tls: Option>, - /// The origin driver, cache governor, and certificate reload: a panic in any /// of them stops [`Relay::run`](crate::Relay::run). pub(crate) critical: crate::critical::Critical, @@ -1149,7 +1156,7 @@ pub struct Cluster { /// A gateway or network session admitted with its lease, scoped origins, and stats. #[non_exhaustive] -pub struct Admitted { +pub(crate) struct Admitted { /// The live authorization that must be held for the session's lifetime. pub lease: auth::Lease, /// Where an admitted publisher writes its broadcasts. @@ -1214,7 +1221,7 @@ impl Cluster { let deprecated = config.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Reject a repeated static identity with conflicting policy up front, - // matching the `--cluster-connect-api` validation, instead of silently + // matching the `--cluster-api-url` validation, instead of silently // keeping the first entry when dials spawn. parse_peer_list(config.connect.clone(), None).context("invalid --cluster-connect peer list")?; peer.quic.validate()?; @@ -1250,7 +1257,6 @@ impl Cluster { nodes, refusals: Default::default(), connection_ids: Arc::default(), - client_tls: None, critical, origin, stats: moq_net::stats::Registry::disabled(), @@ -1262,7 +1268,8 @@ impl Cluster { /// /// Waits on the decider for as long as the caller does; the caller owns the /// deadline, as for [`auth::Auth::admit`]. - pub async fn admit(&self, auth: &auth::Auth, request: moq_auth::Request) -> Result { + #[cfg(test)] + pub(crate) async fn admit(&self, auth: &auth::Auth, request: moq_auth::Request) -> Result { let result = async { if !matches!(request.event, moq_auth::Event::Connect) { return Err(auth::Error::Request("admission requires a connect event".into())); @@ -1391,16 +1398,6 @@ impl Cluster { self } - /// Attach the client TLS config used for `--cluster-connect-api` peer-list - /// fetches. Only its client identity is shared; the API has separate server - /// trust and verifies the hostname in its URL. Required for an HTTP(S) API; pass the TLS config of - /// the [`with_connect`](Self::with_connect) template so the endpoint sees this - /// relay's cluster certificate. - pub fn with_client_tls(mut self, tls: rustls::ClientConfig) -> Self { - self.client_tls = Some(Arc::new(tls)); - self - } - /// Reserve the next `conn` id, the handle a session is logged under. /// /// One counter serves inbound accepts and outbound cluster dials, so an id in @@ -1593,7 +1590,7 @@ impl Cluster { } } - let can_dial = !self.config.connect.is_empty() || self.config.connect_api.is_some() || lan; + let can_dial = !self.config.connect.is_empty() || self.config.api.url.is_some() || lan; if !can_dial { tracing::info!("no cluster peers configured; running standalone"); return Ok(Started { @@ -1603,12 +1600,14 @@ impl Cluster { } // Build here so malformed API roots fail startup, before background work. - let http = if self.config.connect_api.as_deref().is_some_and(connect_api_is_http) { + let http = if self.config.api.url.as_deref().is_some_and(connect_api_is_http) { let tls = self - .client_tls + .connect .as_ref() - .context("cluster.connect_api with an http(s) URL needs client TLS (call Cluster::with_client_tls)")?; - Some(crate::http_client::build(tls, &self.config.connect_api_tls_root)?) + .context("cluster.api.url with an http(s) URL needs a dial template (call Cluster::with_connect)")? + .tls + .build()?; + Some(crate::http_client::build(&tls, &self.config.api.tls.root)?) } else { None }; @@ -1645,7 +1644,7 @@ impl Cluster { // last, so a start that fails earlier never binds a fixed `[connect]` port. // Static and API peers share this client; LAN dials each build their own, // so a LAN-only node builds one only to check the template. - let statics = !self.config.connect.is_empty() || self.config.connect_api.is_some(); + let statics = !self.config.connect.is_empty() || self.config.api.url.is_some(); let client = self .peer_client(!statics, None) .context("cluster peers configured but the dial template is unusable")?; @@ -1672,7 +1671,7 @@ impl Cluster { } /// Runs the cluster event loop against the work [`start`](Self::start) - /// resolved: dials the static peers, then keeps the `connect_api` list and + /// resolved: dials the static peers, then keeps the `api.url` list and /// LAN discovery reconciled into the same dial set. async fn run_work(self, work: Work) -> anyhow::Result<()> { let Work { @@ -1704,7 +1703,7 @@ impl Cluster { dialed.insert(target, handle, DialSource::Static); } - if let Some(source) = self.config.connect_api.clone() { + if let Some(source) = self.config.api.url.clone() { let this = self.clone(); let token = token.clone(); let dialed = dialed.clone(); @@ -1784,7 +1783,7 @@ impl Cluster { anyhow::bail!("LAN discovery stopped unexpectedly"); } - /// Drive `--cluster-connect-api`: an http(s) URL is polled, a local path (or + /// Drive `--cluster-api-url`: an http(s) URL is polled, a local path (or /// `file://` URL) is watched for changes. Either way the source yields a JSON /// array of peers that's reconciled into the shared dial map. async fn run_connect_api( @@ -1802,7 +1801,7 @@ impl Cluster { } Ok(url) if url.scheme() == "file" => match url.to_file_path() { Ok(path) => self.run_connect_api_file(path, node, token, dialed).await, - Err(()) => tracing::error!(%source, "cluster.connect_api file URL is not a valid local path"), + Err(()) => tracing::error!(%source, "cluster.api.url file URL is not a valid local path"), }, // Anything that isn't a URL we recognize is treated as a filesystem path. _ => { @@ -1841,7 +1840,7 @@ impl Cluster { match Self::fetch_peer_list(&http, req_url).await { Ok(list) => self.apply_peer_list(list, &node, &token, &dialed), - Err(err) => tracing::warn!(%err, "cluster.connect_api fetch failed; keeping current peers"), + Err(err) => tracing::warn!(%err, "cluster.api.url fetch failed; keeping current peers"), } } } @@ -1856,7 +1855,7 @@ impl Cluster { let mut watcher = match moq_tokio::watch::Files::new(std::slice::from_ref(&path)) { Ok(watcher) => watcher, Err(err) => { - tracing::error!(%err, ?path, "failed to watch cluster.connect_api file; updates disabled"); + tracing::error!(%err, ?path, "failed to watch cluster.api.url file; updates disabled"); return; } }; @@ -1876,10 +1875,10 @@ impl Cluster { Ok(body) => match serde_json::from_str::>(&body) { Ok(list) => self.apply_peer_list(list, node, token, dialed), Err(err) => { - tracing::warn!(%err, ?path, "cluster.connect_api file is not a JSON array of peers; keeping current peers") + tracing::warn!(%err, ?path, "cluster.api.url file is not a JSON array of peers; keeping current peers") } }, - Err(err) => tracing::warn!(%err, ?path, "failed to read cluster.connect_api file; keeping current peers"), + Err(err) => tracing::warn!(%err, ?path, "failed to read cluster.api.url file; keeping current peers"), } } @@ -1891,14 +1890,14 @@ impl Cluster { .get(url) .send() .await - .context("cluster.connect_api request failed")? + .context("cluster.api.url request failed")? .error_for_status() - .context("cluster.connect_api returned an error status")? + .context("cluster.api.url returned an error status")? .text() .await - .context("failed to read cluster.connect_api body")?; + .context("failed to read cluster.api.url body")?; - serde_json::from_str(&body).context("cluster.connect_api response is not a JSON array of peers") + serde_json::from_str(&body).context("cluster.api.url response is not a JSON array of peers") } /// Reconcile a freshly fetched peer list into the dial map: dial peers that @@ -1911,13 +1910,13 @@ impl Cluster { let desired = match parse_peer_list(list, node.as_deref()) { Ok(desired) => desired, Err(err) => { - tracing::warn!(%err, "invalid cluster.connect_api peer list; keeping current peers"); + tracing::warn!(%err, "invalid cluster.api.url peer list; keeping current peers"); return; } }; dialed.reconcile_api(&desired, |target| { - tracing::info!(peer = %target.key, "cluster.connect_api peer; dialing"); + tracing::info!(peer = %target.key, "cluster.api.url peer; dialing"); let handle = tokio::spawn(self.clone().supervise_remote(target, token.to_string())); handle.abort_handle() }); @@ -2215,7 +2214,7 @@ fn strip_jwt(url: &mut Url) { } } -/// Whether a `--cluster-connect-api` source is an http(s) URL (otherwise it's +/// Whether a `--cluster-api-url` source is an http(s) URL (otherwise it's /// treated as a local file path, which needs no TLS client). fn connect_api_is_http(source: &str) -> bool { Url::parse(source).is_ok_and(|url| matches!(url.scheme(), "http" | "https")) @@ -2225,7 +2224,7 @@ fn connect_api_is_http(source: &str) -> bool { /// /// The modern form is a full URL, e.g. `https://host/?jwt=TOKEN`, which is used /// verbatim. A bare host or `host:port` is wrapped in `https://.../` for -/// `--cluster-connect-api` lists; user-supplied `--cluster-connect` entries +/// `--cluster-api-url` lists; user-supplied `--cluster-connect` entries /// refuse that form through [`Config::deprecated`]. fn peer_url(peer: &str) -> anyhow::Result { // A full URL has a scheme separator; a bare host or `host:port` does not @@ -2339,6 +2338,7 @@ mod tests { } /// The next route and whether it is active. + #[cfg(feature = "cluster-lan")] async fn next_update(announced: &mut moq_net::announce::Consumer) -> Option<(moq_net::announce::Announce, bool)> { match announced.next().await? { moq_net::announce::Event::Start(route) @@ -2984,7 +2984,7 @@ mod tests { } /// Static `--cluster-connect` entries get the same all-or-nothing - /// validation as `--cluster-connect-api` lists: a repeated identity with + /// validation as `--cluster-api-url` lists: a repeated identity with /// conflicting policy fails construction instead of silently keeping the /// first entry, while equivalent entries dedupe. #[tokio::test] @@ -3446,7 +3446,7 @@ mod tests { // that mutate it. let _env = crate::test_env::EnvGuard::lock(); - let toml = "[cluster]\nconnect_api = \"https://api.example.com/cluster/connect\"\nconnect_api_tls_root = [\"api.pem\"]\n"; + let toml = "[cluster.api]\nurl = \"https://api.example.com/cluster/connect\"\ntls.root = [\"api.pem\"]\n"; let dir = std::env::temp_dir().join("moq-relay-cluster-test"); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("cluster-connect-api-toml.toml"); @@ -3455,10 +3455,10 @@ mod tests { let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = RelayConfig::parse_and_merge(args).expect("config load"); assert_eq!( - config.cluster.connect_api.as_deref(), + config.cluster.api.url.as_deref(), Some("https://api.example.com/cluster/connect") ); - assert_eq!(config.cluster.connect_api_tls_root, vec![PathBuf::from("api.pem")]); + assert_eq!(config.cluster.api.tls.root, vec![PathBuf::from("api.pem")]); } /// Two in-process origins share broadcasts both ways over one @@ -3643,7 +3643,10 @@ mod tests { std::fs::write(&peers, format!(r#"["moql://127.0.0.1:{port}/"]"#)).expect("write peers"); let config = Config { - connect_api: Some(peers.display().to_string()), + api: api::Config { + url: Some(peers.display().to_string()), + ..Default::default() + }, ..Default::default() }; let mut quic = moq_tokio::quic::Config::default(); diff --git a/rs/moq-relay/src/cluster/api.rs b/rs/moq-relay/src/cluster/api.rs new file mode 100644 index 00000000000..3714285b10b --- /dev/null +++ b/rs/moq-relay/src/cluster/api.rs @@ -0,0 +1,47 @@ +//! The dynamic peer-list source and its own server trust. + +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Dynamic peer-list source, independent from mesh TLS trust. +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Config { + /// Fetch the list of peers to dial from an HTTP(S) URL or a local file, + /// reloading at runtime without a restart. The source returns a JSON array + /// of peers: bare URL strings and/or objects with `url`, `cost`, `egress`, + /// `token`, and `upstream`, exactly as [`super::Config::connect`] accepts, e.g. + /// `["https://a.pop.example/?cost=1", {"url": "https://b.pop.example/", "cost": 2}]`. + /// An http(s) URL is re-checked on a fixed cadence, with caching, conditional revalidation + /// (`ETag` / `Last-Modified`), and stale-if-error handled by the shared HTTP + /// cache client, so the response's `Cache-Control` controls how often a real + /// fetch hits the endpoint; a local path is watched via OS filesystem + /// notifications (with a periodic re-check fallback). This relay's own + /// [`super::Config::node`] value, when set, is sent as a `?node=` query param so the + /// server can return this node's peers. The relay keeps the last good list if + /// a fetch fails. Composes with [`super::Config::connect`]. + + #[usage(long = "cluster-api-url", env = "MOQ_CLUSTER_API_URL", setting = "cluster.api.url")] + pub url: Option, + /// TLS trust used only for the peer-list API. + #[usage(flatten)] + pub tls: Tls, +} + +/// Server roots for a peer-list API; its client identity comes from the dial template. +#[serde_with::serde_as] +#[derive(usage_rs::Args, Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Tls { + /// PEM roots to use instead of system roots, verifying the API URL's hostname. + #[usage( + long = "cluster-api-tls-root", + env = "MOQ_CLUSTER_API_TLS_ROOT", + setting = "cluster.api.tls.root" + )] + #[usage(value_hint = usage_rs::ValueHint::FilePath, extensions("pem", "crt", "cer"))] + #[serde_as(as = "serde_with::OneOrMany<_>")] + pub root: Vec, +} diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 035c78e7170..8b94e527c15 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -12,8 +12,8 @@ use crate::{auth, cache, cluster, internal, stats, web}; /// name, a version, the completion command) belong to whichever binary owns the /// process, and a `Cli` cannot be `#[usage(flatten)]`ed into another one. Keeping /// them off this type is what lets an embedder put the relay's whole flag surface -/// inside its own CLI -- moq.pro's `edge` does exactly that -- instead of -/// re-declaring it and drifting on every flag added here. This binary wraps it in +/// inside its own CLI instead of re-declaring it and drifting on every flag added +/// here. This binary wraps it in /// a private `Cli` that adds those program-level parts; [`spec`] exposes the /// resulting command line. #[derive(usage_rs::Args, Clone, Debug, Deserialize, Serialize)] @@ -21,13 +21,16 @@ use crate::{auth, cache, cluster, internal, stats, web}; #[serde(deny_unknown_fields, default)] #[non_exhaustive] pub struct Config { - /// Request caps for ordinary client sessions. + /// Request and transport limits grouped by client and cluster-peer role. #[usage(flatten)] - pub session: crate::limits::Session, + pub limits: crate::limits::Config, - /// Request caps for authenticated peers and outgoing cluster links. - #[usage(flatten)] - pub peer: crate::limits::Peer, + #[usage(skip)] + #[serde(rename = "session", skip_serializing)] + legacy_session: Option, + #[usage(skip)] + #[serde(rename = "peer", skip_serializing)] + legacy_peer: Option, /// The QUIC/TLS configuration for the server. #[usage(flatten)] @@ -147,8 +150,9 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { - session: Default::default(), - peer: Default::default(), + limits: Default::default(), + legacy_session: None, + legacy_peer: None, listen: Default::default(), server: None, connect: Default::default(), @@ -268,8 +272,18 @@ impl Config { anyhow::bail!("{}", answer.message()); } }; - let env = usage_rs::config::EnvLayer::from_process(); - let file_path = cli.config.file.clone(); + let mut config = cli.config; + config.merge(&cli_layer)?; + Ok(config) + } + + /// Load this relay's optional file, merge explicit CLI and environment values, and refuse obsolete settings. + /// + /// Call once after the embedding CLI's `parse_from_with_settings`; only this relay + /// fragment is replaced, so the root's CLI-only fields retain their parsed values. + /// [`Self::source`] reports where each merged setting came from. + pub fn merge(&mut self, cli: &usage_rs::config::CliLayer) -> anyhow::Result<()> { + let file_path = self.file.clone(); let file_body = file_path .as_ref() .map(|path| std::fs::read_to_string(path).map(|source| (path.clone(), source))) @@ -285,16 +299,14 @@ impl Config { path: std::path::Path::new(path), value, }); - let mut config = cli.config; - config.merge_into(&cli_layer, &env, file)?; - Ok(config) + self.merge_into(cli, &usage_rs::config::EnvLayer::from_process(), file) } /// Merge CLI, environment, and optional TOML values into this relay fragment. /// /// An embedding binary can parse its own flattened CLI once, then merge only /// its `relay` field. Other CLI-only fields remain untouched. - pub fn merge_into( + fn merge_into( &mut self, cli: &usage_rs::config::CliLayer, env: &usage_rs::config::EnvLayer, @@ -344,10 +356,18 @@ impl Config { /// replaced it. fn deprecated(&self) -> moq_tokio::cli::Deprecated { let mut deprecated = self.quic.deprecated(); + deprecated.extend(self.limits.legacy.deprecated()); + if self.legacy_session.is_some() { + deprecated.toml("[session]", "[limits.client]", None); + } + if self.legacy_peer.is_some() { + deprecated.toml("[peer]", "[limits.peer]", None); + } deprecated.extend(self.listen.deprecated()); deprecated.extend(self.connect.deprecated()); deprecated.extend(self.cluster.deprecated()); deprecated.extend(self.auth.deprecated()); + deprecated.extend(self.acme.deprecated()); if let Some(server) = &self.server { deprecated.toml("[server]", "[listen]", None); deprecated.extend(server.deprecated()); @@ -369,9 +389,11 @@ impl Config { let deprecated = self.deprecated(); anyhow::ensure!(deprecated.is_empty(), "{deprecated}"); // Here rather than at an endpoint, since io_uring workers and lazily built - // dial clients would otherwise check `[quic]` late or never. `[peer.quic]` + // dial clients would otherwise check `[quic]` late or never. `[limits.peer.quic]` // is checked by `Cluster::new`, which embedders call directly. self.quic.validate().context("invalid [quic]")?; + #[cfg(not(feature = "acme"))] + self.acme.refuse_unsupported()?; self.quic .max_streams @@ -387,6 +409,95 @@ mod tests { use super::*; use crate::test_env::EnvGuard; + /// Renames refuse an explicit source even when its value equals the old default. + #[test] + fn role_config_refuses_obsolete_sources() { + let cases = [ + ( + "--session-announces", + "MOQ_SESSION_ANNOUNCES", + "1000", + "--limits-client-announces", + ), + ("--peer-fetches", "MOQ_PEER_FETCHES", "10000", "--limits-peer-fetches"), + ("--acme-dir", "MOQ_ACME_DIR", "cache", "--acme-cache"), + ( + "--cluster-connect-api", + "MOQ_CLUSTER_CONNECT_API", + "peers.json", + "--cluster-api-url", + ), + ( + "--cluster-connect-api-tls-root", + "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", + "api.pem", + "--cluster-api-tls-root", + ), + ( + "--auth-verifications", + "MOQ_AUTH_VERIFICATIONS", + "128", + "fixed per-session verification policy", + ), + ]; + let _env = EnvGuard::clear(&cases.map(|(_, env, _, _)| env)); + for (flag, env, value, replacement) in cases { + let error = Config::parse_and_merge(["moq-relay", flag, value]) + .unwrap_err() + .to_string(); + assert!(error.contains(flag) && error.contains(replacement), "{error}"); + // SAFETY: the guard serializes mutation and restores every case's variable. + unsafe { + std::env::set_var(env, value); + } + let error = Config::parse_and_merge(["moq-relay"]).unwrap_err().to_string(); + unsafe { + std::env::remove_var(env); + } + assert!(error.contains(env) && error.contains(replacement), "{error}"); + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("obsolete.toml"); + for (body, replacement) in [ + ("[session]\nannounces = 1000", "[limits.client]"), + ("[peer]\nfetches = 10000", "[limits.peer]"), + ("[acme]\ndir = 'cache'", "[acme] cache"), + ("[cluster]\nconnect_api = 'peers.json'", "[cluster.api] url"), + ("[cluster]\nconnect_api_tls_root = []", "[cluster.api.tls] root"), + ("[auth]\nverifications = 128", "fixed per-session verification policy"), + ] { + std::fs::write(&path, body).unwrap(); + let error = + Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap_err(); + assert!(error.to_string().contains(replacement), "{body}: {error}"); + } + } + + #[cfg(not(feature = "acme"))] + #[test] + fn an_acme_free_build_refuses_configured_issuance() { + let mut config = Config::default(); + config.resolve().unwrap(); + config.acme.domains.push("relay.example.com".into()); + let error = config.resolve().unwrap_err(); + assert!(error.to_string().contains("acme feature"), "{error}"); + } + + /// Both explicit values of a removed TOML knob must be refused by the common loader. + #[test] + fn loader_refuses_toml_extension_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + for side in ["listen", "connect"] { + for value in ["true", "false"] { + std::fs::write(&path, format!("[{side}.extensions]\nietf = {value}\n")).unwrap(); + let error = + Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap_err(); + assert!(error.to_string().contains("extensions"), "{side} {value}: {error}"); + } + } + } + #[test] fn stun_binding_merges_file_environment_and_cli() { let _env = EnvGuard::clear(&["MOQ_STUN_BIND"]); @@ -414,92 +525,92 @@ mod tests { #[test] fn session_limits_merge_without_crossing_peer_policy() { let _env = EnvGuard::clear(&[ - "MOQ_SESSION_ANNOUNCES", - "MOQ_SESSION_SUBSCRIPTIONS", - "MOQ_SESSION_FETCHES", - "MOQ_PEER_FETCHES", - "MOQ_PEER_ANNOUNCES", - "MOQ_PEER_SUBSCRIPTIONS", + "MOQ_LIMITS_CLIENT_ANNOUNCES", + "MOQ_LIMITS_CLIENT_SUBSCRIPTIONS", + "MOQ_LIMITS_CLIENT_FETCHES", + "MOQ_LIMITS_PEER_FETCHES", + "MOQ_LIMITS_PEER_ANNOUNCES", + "MOQ_LIMITS_PEER_SUBSCRIPTIONS", ]); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("limits.toml"); std::fs::write( &path, - "[session]\nannounces = 7\nsubscriptions = 8\nfetches = 9\n[peer]\nannounces = 90\nsubscriptions = 91\nfetches = 92\n", + "[limits.client]\nannounces = 7\nsubscriptions = 8\nfetches = 9\n[limits.peer]\nannounces = 90\nsubscriptions = 91\nfetches = 92\n", ) .unwrap(); let config = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap(); - assert_eq!(config.session.resolve().announces, 7); - assert_eq!(config.session.resolve().subscriptions, 8); - assert_eq!(config.peer.resolve().announces, 90); - assert_eq!(config.peer.resolve().subscriptions, 91); - assert_eq!(config.session.resolve().fetches, 9); - assert_eq!(config.peer.resolve().fetches, 92); + assert_eq!(config.limits.client.resolve().announces, 7); + assert_eq!(config.limits.client.resolve().subscriptions, 8); + assert_eq!(config.limits.peer.resolve().announces, 90); + assert_eq!(config.limits.peer.resolve().subscriptions, 91); + assert_eq!(config.limits.client.resolve().fetches, 9); + assert_eq!(config.limits.peer.resolve().fetches, 92); let config = Config::parse_and_merge([ std::ffi::OsString::from("moq-relay"), path.into(), - "--session-announces=0".into(), - "--peer-subscriptions=1000".into(), - "--session-fetches=0".into(), - "--peer-fetches=2000".into(), + "--limits-client-announces=0".into(), + "--limits-peer-subscriptions=1000".into(), + "--limits-client-fetches=0".into(), + "--limits-peer-fetches=2000".into(), ]) .unwrap(); - assert_eq!(config.session.resolve().announces, 0); - assert_eq!(config.session.resolve().subscriptions, 8); - assert_eq!(config.peer.resolve().announces, 90); - assert_eq!(config.peer.resolve().subscriptions, 1000); - assert_eq!(config.session.resolve().fetches, 0); - assert_eq!(config.peer.resolve().fetches, 2000); + assert_eq!(config.limits.client.resolve().announces, 0); + assert_eq!(config.limits.client.resolve().subscriptions, 8); + assert_eq!(config.limits.peer.resolve().announces, 90); + assert_eq!(config.limits.peer.resolve().subscriptions, 1000); + assert_eq!(config.limits.client.resolve().fetches, 0); + assert_eq!(config.limits.peer.resolve().fetches, 2000); } - /// `[peer.quic]` mirrors `[quic]`'s keys without touching `[quic]` itself. + /// `[limits.peer.quic]` mirrors `[quic]`'s keys without touching `[quic]` itself. #[test] fn peer_quic_table_merges_with_flags() { let _env = EnvGuard::clear(&[ "MOQ_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_RECEIVE_WINDOW", - "MOQ_PEER_QUIC_SEND_WINDOW", + "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", + "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", ]); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("peer-quic.toml"); std::fs::write( &path, - "[quic]\nmax_streams = 4\n[peer.quic]\nmax_streams = 64\nsend_window = 1000\n", + "[quic]\nmax_streams = 4\n[limits.peer.quic]\nmax_streams = 64\nsend_window = 1000\n", ) .unwrap(); let config = Config::parse_and_merge([ std::ffi::OsString::from("moq-relay"), path.into(), - "--peer-quic-receive-window=2000".into(), + "--limits-peer-quic-receive-window=2000".into(), ]) .unwrap(); assert_eq!(config.quic.max_streams, Some(4)); - assert_eq!(config.peer.quic.max_streams.get(), 64); - assert_eq!(config.peer.quic.receive_window.get(), 2000); - assert_eq!(config.peer.quic.send_window.get(), 1000); + assert_eq!(config.limits.peer.quic.max_streams.get(), 64); + assert_eq!(config.limits.peer.quic.receive_window.get(), 2000); + assert_eq!(config.limits.peer.quic.send_window.get(), 1000); - assert!(toml::from_str::("[peer]\nmax_streams = 64\n").is_err()); + assert!(toml::from_str::("[limits.peer]\nmax_streams = 64\n").is_err()); } /// A zero peer stream limit or window would wedge the connection, so it never parses. #[test] fn zero_peer_quic_limits_are_refused() { let _env = EnvGuard::clear(&[ - "MOQ_PEER_QUIC_MAX_STREAMS", - "MOQ_PEER_QUIC_RECEIVE_WINDOW", - "MOQ_PEER_QUIC_SEND_WINDOW", + "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", + "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", ]); for key in ["max_streams", "receive_window", "send_window"] { assert!( - toml::from_str::(&format!("[peer.quic]\n{key} = 0\n")).is_err(), + toml::from_str::(&format!("[limits.peer.quic]\n{key} = 0\n")).is_err(), "{key}" ); } for flag in [ - "--peer-quic-max-streams=0", - "--peer-quic-receive-window=0", - "--peer-quic-send-window=0", + "--limits-peer-quic-max-streams=0", + "--limits-peer-quic-receive-window=0", + "--limits-peer-quic-send-window=0", ] { assert!( Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), flag.into()]).is_err(), @@ -720,26 +831,41 @@ root = ["ca.pem"] assert_eq!(config.quic.max_streams, Some(128)); } - /// Every config under `demo/relay/` still parses. - /// - /// These are the configs a reader copies, and a rename that lands in the code - /// but not in them is invisible until someone's relay refuses to boot. A move - /// across tables (`[server.quic]` to a top-level `[quic]`) is the case serde - /// aliases cannot cover, which is exactly why this reads the real files. + /// Real relay examples, packaging and test fixtures use the same loader as an embedding CLI. #[test] - fn demo_configs_parse() { - let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../demo/relay"); - let mut checked = 0; - for entry in std::fs::read_dir(&dir).expect("demo/relay") { - let path = entry.expect("dir entry").path(); - if path.extension().is_none_or(|ext| ext != "toml") { - continue; + fn repository_configs_load() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + // Only repository inputs belong here; installed clients and build output below + // test/ can contain unrelated TOML. Git's tracked list also includes new fixtures. + let tracked = std::process::Command::new("git") + .args(["ls-files", "-z", "--", "demo/relay", "packaging", "test"]) + .current_dir(&root) + .output() + .expect("list repository fixtures"); + assert!(tracked.status.success()); + let tracked = String::from_utf8(tracked.stdout).unwrap(); + for area in ["demo/relay", "packaging", "test"] { + let mut checked = 0; + for name in tracked.split('\0').filter(|name| name.starts_with(area)) { + let path = root.join(name); + if path.extension().is_none_or(|ext| ext != "toml") + || matches!( + path.file_name().and_then(|name| name.to_str()), + Some("Cargo.toml" | "rust-toolchain.toml" | ".rustfmt.toml") + ) { + continue; + } + let result = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]); + if path.file_name().unwrap() == "smoke-legacy.toml" { + let error = result.expect_err("the published legacy fixture must be refused"); + assert!(error.to_string().contains("[server]"), "{}: {error}", path.display()); + } else { + result.unwrap_or_else(|error| panic!("{}: {error}", path.display())); + } + checked += 1; } - let toml = std::fs::read_to_string(&path).expect("read config"); - toml::from_str::(&toml).unwrap_or_else(|err| panic!("{}: {err}", path.display())); - checked += 1; + assert!(checked > 0, "no relay fixtures found in {area}"); } - assert!(checked > 0, "no demo configs found in {}", dir.display()); } /// Bare defaults loaded from TOML survive when the CLI does not mention them. @@ -807,7 +933,7 @@ ws = false let dir = std::env::temp_dir().join("moq-relay-config-test"); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("runtime-toml-wins.toml"); - std::fs::write(&path, &toml).unwrap(); + std::fs::write(&path, toml.as_bytes()).unwrap(); let args = vec![std::ffi::OsString::from("moq-relay"), std::ffi::OsString::from(&path)]; let config = Config::parse_and_merge(args).expect("config load"); @@ -881,7 +1007,7 @@ ws = false /// This is the reason [`Config`] derives `Args` rather than `Cli`: a `Cli` /// cannot be flattened, so declaring the program-level parts on it would force /// every embedder to re-declare the relay's flags and drift on each one added - /// here. moq.pro's `edge` is the embedder this exists for. + /// here. #[test] fn the_config_can_be_embedded() { /// A stand-in for an embedder's command line: the relay's flags plus one of @@ -938,7 +1064,7 @@ ws = false "Top-level relay configuration, loadable from CLI arguments, environment variables, or a TOML file." ); // The tell that a developer-facing doc comment reached this surface. - for leak in ["[`", "moq.pro", "implementation detail"] { + for leak in ["[`", "implementation detail"] { assert!( !shown.contains(leak), "{leak:?} leaked into the help description: {shown}" diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index e9c44913af2..1cf6cf4209a 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -30,7 +30,7 @@ impl From for StatusError { /// publish/subscribe origins, and serve the session until it closes. /// /// The request is a moq-tokio [`Request`](moq_tokio::server::Request), or an -/// [`Accepted`](crate::Accepted) session from any moq-net transport. +/// in-memory accepted session when the `test-support` feature is enabled. pub struct Connection { request: R, lifecycle: Lifecycle, diff --git a/rs/moq-relay/src/incoming.rs b/rs/moq-relay/src/incoming.rs index 86ea3e3d492..bc6a754d8f3 100644 --- a/rs/moq-relay/src/incoming.rs +++ b/rs/moq-relay/src/incoming.rs @@ -2,26 +2,30 @@ use crate::auth; use axum::http; +#[cfg(feature = "test-support")] use moq_net::transport::poll; /// A session whose SETUP awaits the relay's reply: what a [`Connection`](crate::Connection) serves. /// -/// Sealed: implemented by [`moq_tokio::server::Request`] and [`Accepted`]. +/// Sealed: native requests, plus in-memory accepted sessions with `test-support`. pub trait Incoming: Sealed {} impl Incoming for moq_tokio::server::Request {} +#[cfg(feature = "test-support")] impl Incoming for Accepted {} /// A session accepted on any moq-net transport, its SETUP awaiting the relay's reply. /// /// The auth decider sees the path, query, token, and role the SETUP carried, so the /// client must send its path in-band. +#[cfg(feature = "test-support")] pub struct Accepted { handshake: moq_net::server::Handshake, transport: moq_auth::Transport, } +#[cfg(feature = "test-support")] impl Accepted { /// Wrap `handshake`, reported to the auth decider as arriving over `transport`. pub fn new(handshake: moq_net::server::Handshake, transport: moq_auth::Transport) -> Self { @@ -145,6 +149,7 @@ impl Sealed for moq_tokio::server::Request { } } +#[cfg(feature = "test-support")] impl Sealed for Accepted { /// Polled on the connection's own task, so it needs no runtime to spawn on. type Driver = moq_net::Driver; diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index b3ed94f3000..c612e5f6abf 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -64,6 +64,7 @@ fn configured_tier(label: Option) -> moq_net::stats::Tier { pub use config::*; pub use connection::*; pub use fetch::*; +#[cfg(feature = "test-support")] pub use incoming::{Accepted, Incoming}; pub use relay::*; #[cfg(feature = "test-support")] diff --git a/rs/moq-relay/src/limits.rs b/rs/moq-relay/src/limits.rs index 2ae1946be96..fcdf4bae41f 100644 --- a/rs/moq-relay/src/limits.rs +++ b/rs/moq-relay/src/limits.rs @@ -3,38 +3,54 @@ use anyhow::Context; use serde::{Deserialize, Serialize}; +/// Request and transport limits grouped by the authorized connection's role. +#[derive(usage_rs::Args, Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct Config { + /// Request caps for ordinary clients. + #[usage(flatten)] + pub client: Client, + /// Request and QUIC caps for authenticated cluster peers. + #[usage(flatten)] + pub peer: Peer, + #[usage(flatten)] + #[serde(skip)] + pub(crate) legacy: Legacy, +} + /// Request caps for ordinary client sessions. #[derive(usage_rs::Args, Clone, Debug, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] #[non_exhaustive] -pub struct Session { +pub struct Client { /// Maximum simultaneously announced broadcasts or namespaces. #[usage( - long = "session-announces", - env = "MOQ_SESSION_ANNOUNCES", + long = "limits-client-announces", + env = "MOQ_LIMITS_CLIENT_ANNOUNCES", default = "1000", - setting = "session.announces" + setting = "limits.client.announces" )] pub announces: usize, /// Maximum simultaneously held subscriptions. #[usage( - long = "session-subscriptions", - env = "MOQ_SESSION_SUBSCRIPTIONS", + long = "limits-client-subscriptions", + env = "MOQ_LIMITS_CLIENT_SUBSCRIPTIONS", default = "1000", - setting = "session.subscriptions" + setting = "limits.client.subscriptions" )] pub subscriptions: usize, /// Maximum simultaneously held FETCH requests. #[usage( - long = "session-fetches", - env = "MOQ_SESSION_FETCHES", + long = "limits-client-fetches", + env = "MOQ_LIMITS_CLIENT_FETCHES", default = "10000", - setting = "session.fetches" + setting = "limits.client.fetches" )] pub fetches: usize, } -impl Default for Session { +impl Default for Client { fn default() -> Self { Self { announces: 1000, @@ -44,7 +60,7 @@ impl Default for Session { } } -impl Session { +impl Client { pub(crate) fn resolve(&self) -> moq_net::session::Limits { resolve(self.announces, self.subscriptions, self.fetches) } @@ -57,29 +73,29 @@ impl Session { pub struct Peer { /// Maximum simultaneously announced broadcasts or namespaces. #[usage( - long = "peer-announces", - env = "MOQ_PEER_ANNOUNCES", + long = "limits-peer-announces", + env = "MOQ_LIMITS_PEER_ANNOUNCES", default = "100000", - setting = "peer.announces" + setting = "limits.peer.announces" )] pub announces: usize, /// Maximum simultaneously held subscriptions. #[usage( - long = "peer-subscriptions", - env = "MOQ_PEER_SUBSCRIPTIONS", + long = "limits-peer-subscriptions", + env = "MOQ_LIMITS_PEER_SUBSCRIPTIONS", default = "10000", - setting = "peer.subscriptions" + setting = "limits.peer.subscriptions" )] pub subscriptions: usize, /// Maximum simultaneously held FETCH requests. #[usage( - long = "peer-fetches", - env = "MOQ_PEER_FETCHES", + long = "limits-peer-fetches", + env = "MOQ_LIMITS_PEER_FETCHES", default = "10000", - setting = "peer.fetches" + setting = "limits.peer.fetches" )] pub fetches: usize, - /// QUIC limits a peer session widens to (`[peer.quic]`). + /// QUIC limits a peer session widens to (`[limits.peer.quic]`). #[usage(flatten)] pub quic: Quic, } @@ -108,26 +124,26 @@ impl Peer { pub struct Quic { /// Maximum concurrent QUIC streams the peer may open, per direction. #[usage( - long = "peer-quic-max-streams", - env = "MOQ_PEER_QUIC_MAX_STREAMS", + long = "limits-peer-quic-max-streams", + env = "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", default = "100000", - setting = "peer.quic.max_streams" + setting = "limits.peer.quic.max_streams" )] pub max_streams: std::num::NonZeroU64, /// Connection-wide QUIC receive window, in bytes. #[usage( - long = "peer-quic-receive-window", - env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", + long = "limits-peer-quic-receive-window", + env = "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", default = "671088640", - setting = "peer.quic.receive_window" + setting = "limits.peer.quic.receive_window" )] pub receive_window: std::num::NonZeroU64, /// Cap on unacknowledged outgoing QUIC data, in bytes. #[usage( - long = "peer-quic-send-window", - env = "MOQ_PEER_QUIC_SEND_WINDOW", + long = "limits-peer-quic-send-window", + env = "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", default = "671088640", - setting = "peer.quic.send_window" + setting = "limits.peer.quic.send_window" )] pub send_window: std::num::NonZeroU64, } @@ -151,7 +167,7 @@ impl Quic { pub(crate) fn validate(&self) -> anyhow::Result<()> { let mut quic = moq_tokio::quic::Config::default(); self.dial(&mut quic); - quic.validate().context("invalid [peer.quic]") + quic.validate().context("invalid [limits.peer.quic]") } /// Apply these limits to the `[quic]` section a peer link is dialed with. @@ -179,3 +195,91 @@ fn resolve(announces: usize, subscriptions: usize, fetches: usize) -> moq_net::s limits.fetches = fetches; limits } + +#[derive(usage_rs::Args, Clone, Debug, Default)] +pub(crate) struct Legacy { + #[usage(long = "session-announces", env = "MOQ_SESSION_ANNOUNCES", hide = true)] + session_announces: Option, + #[usage(long = "session-subscriptions", env = "MOQ_SESSION_SUBSCRIPTIONS", hide = true)] + session_subscriptions: Option, + #[usage(long = "session-fetches", env = "MOQ_SESSION_FETCHES", hide = true)] + session_fetches: Option, + #[usage(long = "peer-announces", env = "MOQ_PEER_ANNOUNCES", hide = true)] + peer_announces: Option, + #[usage(long = "peer-subscriptions", env = "MOQ_PEER_SUBSCRIPTIONS", hide = true)] + peer_subscriptions: Option, + #[usage(long = "peer-fetches", env = "MOQ_PEER_FETCHES", hide = true)] + peer_fetches: Option, + #[usage(long = "peer-quic-max-streams", env = "MOQ_PEER_QUIC_MAX_STREAMS", hide = true)] + peer_quic_max_streams: Option, + #[usage(long = "peer-quic-receive-window", env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", hide = true)] + peer_quic_receive_window: Option, + #[usage(long = "peer-quic-send-window", env = "MOQ_PEER_QUIC_SEND_WINDOW", hide = true)] + peer_quic_send_window: Option, +} + +impl Legacy { + pub(crate) fn deprecated(&self) -> moq_tokio::cli::Deprecated { + let mut found = moq_tokio::cli::Deprecated::default(); + if self.session_announces.is_some() { + found.flag( + "--session-announces", + Some("MOQ_SESSION_ANNOUNCES"), + "--limits-client-announces", + ); + } + if self.session_subscriptions.is_some() { + found.flag( + "--session-subscriptions", + Some("MOQ_SESSION_SUBSCRIPTIONS"), + "--limits-client-subscriptions", + ); + } + if self.session_fetches.is_some() { + found.flag( + "--session-fetches", + Some("MOQ_SESSION_FETCHES"), + "--limits-client-fetches", + ); + } + if self.peer_announces.is_some() { + found.flag( + "--peer-announces", + Some("MOQ_PEER_ANNOUNCES"), + "--limits-peer-announces", + ); + } + if self.peer_subscriptions.is_some() { + found.flag( + "--peer-subscriptions", + Some("MOQ_PEER_SUBSCRIPTIONS"), + "--limits-peer-subscriptions", + ); + } + if self.peer_fetches.is_some() { + found.flag("--peer-fetches", Some("MOQ_PEER_FETCHES"), "--limits-peer-fetches"); + } + if self.peer_quic_max_streams.is_some() { + found.flag( + "--peer-quic-max-streams", + Some("MOQ_PEER_QUIC_MAX_STREAMS"), + "--limits-peer-quic-max-streams", + ); + } + if self.peer_quic_receive_window.is_some() { + found.flag( + "--peer-quic-receive-window", + Some("MOQ_PEER_QUIC_RECEIVE_WINDOW"), + "--limits-peer-quic-receive-window", + ); + } + if self.peer_quic_send_window.is_some() { + found.flag( + "--peer-quic-send-window", + Some("MOQ_PEER_QUIC_SEND_WINDOW"), + "--limits-peer-quic-send-window", + ); + } + found + } +} diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 41f0ca1af36..f424d260658 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -81,6 +81,7 @@ pub struct Relay { internal: internal::Internal, web: web::Web, /// The ACME client, issuing before [`Self::run`] serves and renewing after. + #[cfg(feature = "acme")] acme: Option, stun: Option, addr: Option, @@ -101,7 +102,7 @@ pub struct Relay { #[cfg(feature = "_quic")] workers: Option, /// The io_uring QUIC workers, already bound and waiting for - /// [`serve`](crate::uring::Workers::serve). `None` unless both + /// [`serve`](crate::uring::Spawned::serve). `None` unless both /// `runtime.workers` and `runtime.io_uring` are configured, in which case /// they own the QUIC listen address instead of `workers`. #[cfg(all(target_os = "linux", feature = "_uring"))] @@ -128,7 +129,9 @@ impl Relay { config.listen.extensions.ietf = true; config.connect.extensions.ietf = true; // Every TLS listener serves the ACME certificate, so install it before any is built. + #[cfg(feature = "acme")] let acme = config.acme.init(&config.listen.tls, &config.web.https)?; + #[cfg(feature = "acme")] if let Some(acme) = &acme { config.listen.tls.resolver = Some(acme.resolver()); } @@ -289,10 +292,9 @@ impl Relay { advertise = advertise.with_fingerprint(fingerprint); } let mut cluster_options = cluster::Options::new(config.cluster).with_cache(cache); - cluster_options.session = config.session; - cluster_options.peer = config.peer; + cluster_options.session = config.limits.client; + cluster_options.peer = config.limits.peer; let cluster = cluster::Cluster::new(cluster_options)? - .with_client_tls(config.connect.tls.build()?) .with_connect(config.connect.clone(), config.quic.clone()) .with_advertise(advertise); #[cfg(feature = "iroh")] @@ -308,13 +310,14 @@ impl Relay { let sessions = crate::session::Registry::new(); let (ready, _) = tokio::sync::watch::channel(false); let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web) - .with_acme(acme.as_ref().map(crate::acme::Acme::resolver)) .with_shutdown(shutdown.clone()) .with_versions(server_versions) .with_extensions(server_extensions) .with_timeout(handshake_timeout) - .with_sessions(sessions.clone()) - .bind()?; + .with_sessions(sessions.clone()); + #[cfg(feature = "acme")] + let web = web.with_acme(acme.as_ref().map(crate::acme::Acme::resolver)); + let web = web.bind()?; // `bind`, not `listen`: the TCP/Unix accept loops handshake as soon as // they run, and `load` is not yet willing to take a session. `run` // starts them on its first accept. @@ -362,6 +365,7 @@ impl Relay { stats, internal, web, + #[cfg(feature = "acme")] acme, stun, addr, @@ -538,6 +542,7 @@ impl Relay { cluster, internal, web, + #[cfg(feature = "acme")] acme, stun, shutdown, @@ -567,6 +572,7 @@ impl Relay { // Nothing else serves or reports ready until the certificate exists: a // self-signed placeholder would only fail every browser that connects. + #[cfg(feature = "acme")] if let Some(mut acme) = acme { let listener = web.https_listener().context("ACME needs the HTTPS listener")?; // Issuance can take minutes, and as a container's PID 1 the relay @@ -594,13 +600,13 @@ impl Relay { // identity rustls will not load fails here, and reporting ready first would // release the units depending on a relay that is about to exit. #[cfg(all(target_os = "linux", feature = "_uring"))] - let mut uring = uring; - #[cfg(all(target_os = "linux", feature = "_uring"))] - if let Some(uring) = uring.as_mut() { - uring - .spawn(cluster.clone(), auth.clone(), shutdown.clone(), sessions.clone()) - .context("failed to start the io_uring QUIC workers")?; - } + let mut uring = uring + .map(|uring| { + uring + .spawn(cluster.clone(), auth.clone(), shutdown.clone(), sessions.clone()) + .context("failed to start the io_uring QUIC workers") + }) + .transpose()?; // Without ACME, after the last step that can fail, since a handler outlives a // failed `run`, and before anything accepts a session a signal would kill diff --git a/rs/moq-relay/src/settings.rs b/rs/moq-relay/src/settings.rs index 25ad57508b0..cc4c26f2c3b 100644 --- a/rs/moq-relay/src/settings.rs +++ b/rs/moq-relay/src/settings.rs @@ -10,9 +10,7 @@ #[derive(usage_rs::Config)] pub struct Settings { #[usage(flatten)] - session: Session, - #[usage(flatten)] - peer: Peer, + limits: Limits, #[usage(flatten)] listen: moq_tokio::settings::Listen, @@ -82,11 +80,8 @@ struct Cluster { #[usage(env = "MOQ_CLUSTER_CONNECT", cli("--cluster-connect"), parse = "list_by_comma")] connect: Option>, - #[usage(env = "MOQ_CLUSTER_CONNECT_API", cli("--cluster-connect-api"))] - connect_api: Option, - - #[usage(env = "MOQ_CLUSTER_CONNECT_API_TLS_ROOT", cli("--cluster-connect-api-tls-root"))] - connect_api_tls_root: Option>, + #[usage(flatten)] + api: ClusterApi, #[usage(env = "MOQ_CLUSTER_NODE", cli("--cluster-node"))] node: Option, @@ -102,6 +97,22 @@ struct Cluster { lan: ClusterLan, } +#[derive(usage_rs::Config)] +#[usage(prefix = "cluster.api")] +struct ClusterApi { + #[usage(env = "MOQ_CLUSTER_API_URL", cli("--cluster-api-url"))] + url: Option, + #[usage(flatten)] + tls: ClusterApiTls, +} + +#[derive(usage_rs::Config)] +#[usage(prefix = "cluster.api.tls")] +struct ClusterApiTls { + #[usage(env = "MOQ_CLUSTER_API_TLS_ROOT", cli("--cluster-api-tls-root"))] + root: Option>, +} + #[cfg(feature = "cluster-lan")] #[derive(usage_rs::Config)] #[usage(prefix = "cluster.lan")] @@ -138,9 +149,6 @@ struct Auth { parse = "list_by_comma" )] public_publish: Option>, - - #[usage(env = "MOQ_AUTH_VERIFICATIONS", cli("--auth-verifications"))] - verifications: Option, } #[derive(usage_rs::Config)] @@ -188,8 +196,8 @@ struct Acme { #[usage(env = "MOQ_ACME_CONTACT", cli("--acme-contact"), parse = "list_by_comma")] contact: Option>, - #[usage(env = "MOQ_ACME_DIR", cli("--acme-dir"))] - dir: Option, + #[usage(env = "MOQ_ACME_CACHE", cli("--acme-cache"))] + cache: Option, #[usage(env = "MOQ_ACME_DIRECTORY", cli("--acme-directory"))] directory: Option, @@ -241,37 +249,48 @@ struct Internal { } #[derive(usage_rs::Config)] -#[usage(prefix = "session")] -struct Session { - #[usage(env = "MOQ_SESSION_ANNOUNCES", cli("--session-announces"))] +struct Limits { + #[usage(flatten)] + client: Client, + #[usage(flatten)] + peer: Peer, +} + +#[derive(usage_rs::Config)] +#[usage(prefix = "limits.client")] +struct Client { + #[usage(env = "MOQ_LIMITS_CLIENT_ANNOUNCES", cli("--limits-client-announces"))] announces: Option, - #[usage(env = "MOQ_SESSION_SUBSCRIPTIONS", cli("--session-subscriptions"))] + #[usage(env = "MOQ_LIMITS_CLIENT_SUBSCRIPTIONS", cli("--limits-client-subscriptions"))] subscriptions: Option, - #[usage(env = "MOQ_SESSION_FETCHES", cli("--session-fetches"))] + #[usage(env = "MOQ_LIMITS_CLIENT_FETCHES", cli("--limits-client-fetches"))] fetches: Option, } #[derive(usage_rs::Config)] -#[usage(prefix = "peer")] +#[usage(prefix = "limits.peer")] struct Peer { - #[usage(env = "MOQ_PEER_ANNOUNCES", cli("--peer-announces"))] + #[usage(env = "MOQ_LIMITS_PEER_ANNOUNCES", cli("--limits-peer-announces"))] announces: Option, - #[usage(env = "MOQ_PEER_SUBSCRIPTIONS", cli("--peer-subscriptions"))] + #[usage(env = "MOQ_LIMITS_PEER_SUBSCRIPTIONS", cli("--limits-peer-subscriptions"))] subscriptions: Option, - #[usage(env = "MOQ_PEER_FETCHES", cli("--peer-fetches"))] + #[usage(env = "MOQ_LIMITS_PEER_FETCHES", cli("--limits-peer-fetches"))] fetches: Option, #[usage(flatten)] quic: PeerQuic, } #[derive(usage_rs::Config)] -#[usage(prefix = "peer.quic")] +#[usage(prefix = "limits.peer.quic")] struct PeerQuic { - #[usage(env = "MOQ_PEER_QUIC_MAX_STREAMS", cli("--peer-quic-max-streams"))] + #[usage(env = "MOQ_LIMITS_PEER_QUIC_MAX_STREAMS", cli("--limits-peer-quic-max-streams"))] max_streams: Option, - #[usage(env = "MOQ_PEER_QUIC_RECEIVE_WINDOW", cli("--peer-quic-receive-window"))] + #[usage( + env = "MOQ_LIMITS_PEER_QUIC_RECEIVE_WINDOW", + cli("--limits-peer-quic-receive-window") + )] receive_window: Option, - #[usage(env = "MOQ_PEER_QUIC_SEND_WINDOW", cli("--peer-quic-send-window"))] + #[usage(env = "MOQ_LIMITS_PEER_QUIC_SEND_WINDOW", cli("--limits-peer-quic-send-window"))] send_window: Option, } diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 371bea3f427..dc21eadba90 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -87,7 +87,13 @@ struct Serve { /// [`bind`](Self::bind) opens every socket, so a returned value already owns /// the listen address and a port conflict is an error here rather than a /// worker that quietly died. [`spawn`](Self::spawn) starts every thread, and -/// nothing is served until [`serve`](Self::serve). +/// nothing is served until [`Spawned::serve`]. +/// +/// ```compile_fail +/// fn serve_before_spawn(mut bound: moq_relay::uring::Workers) { +/// bound.serve(); +/// } +/// ``` pub struct Workers { members: Vec, addr: SocketAddr, @@ -111,7 +117,7 @@ pub struct Workers { stops: Vec>, /// Releases each spawned worker to accept; dropped unsent, it exits instead. starts: Vec>, - /// A worker's fatal error, reported to whoever polls [`failed`](Self::failed). + /// A worker's fatal error, reported to whoever polls [`Spawned::failed`]. failures: tokio::sync::mpsc::UnboundedReceiver, failures_tx: tokio::sync::mpsc::UnboundedSender, @@ -280,14 +286,14 @@ impl Workers { /// authentication and session supervision. Returns once every worker is /// set up; a worker that cannot start (an old kernel, a ring failure) is /// an error here rather than a thread that quietly died. Nothing accepts - /// until [`serve`](Self::serve), so a failure here has served nobody. + /// until [`Spawned::serve`], so a failure here has served nobody. pub fn spawn( - &mut self, + mut self, cluster: cluster::Cluster, auth: auth::Auth, shutdown: shutdown::Observer, sessions: crate::session::Registry, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { let serve = Serve { cluster, auth, @@ -349,11 +355,11 @@ impl Workers { self.starts.push(start_tx); } - Ok(()) + Ok(Spawned { workers: self }) } - /// Start every [`spawn`](Self::spawn)ed worker accepting. - pub fn serve(&mut self) { + /// Start every [`Workers::spawn`]ed worker accepting. + fn serve(&mut self) { for start in self.starts.drain(..) { // A worker that died since setup is reported through `failed`. let _ = start.send(()); @@ -362,7 +368,7 @@ impl Workers { /// The first worker's fatal error. Pends until one dies, which a healthy /// group never does, so this composes into a `select!` loop. - pub async fn failed(&mut self) -> anyhow::Error { + async fn failed(&mut self) -> anyhow::Error { match self.failures.recv().await { Some(err) => err, // Senders live in `self`, so this is unreachable; pend regardless. @@ -371,7 +377,7 @@ impl Workers { } /// Stop every worker and wait for its thread, off the caller's runtime. - pub async fn shutdown(mut self) { + async fn shutdown(mut self) { let threads = self.signal(); if threads.is_empty() { return; @@ -389,6 +395,26 @@ impl Workers { } } +/// Workers that have all initialized and can now begin accepting sessions. +pub struct Spawned { + workers: Workers, +} + +impl Spawned { + /// Release every initialized worker to accept sessions. + pub fn serve(&mut self) { + self.workers.serve(); + } + /// Wait for the first worker's fatal error. + pub async fn failed(&mut self) -> anyhow::Error { + self.workers.failed().await + } + /// Stop every worker and wait until its thread has ended. + pub async fn shutdown(self) { + self.workers.shutdown().await; + } +} + impl Drop for Workers { fn drop(&mut self) { join(self.signal()); @@ -490,7 +516,7 @@ struct Spawn { /// Reports setup back to [`Workers::spawn`], which blocks on it so a /// worker that cannot start is an error there rather than a dead thread. ready: std::sync::mpsc::Sender>, - /// Fires on [`Workers::serve`]; closes when the group stops first. + /// Fires on [`Spawned::serve`]; closes when the group stops first. start: std::sync::mpsc::Receiver<()>, /// Where a fatal error goes once the worker is running. failures: tokio::sync::mpsc::UnboundedSender, @@ -1012,9 +1038,9 @@ mod tests { let mut server = moq_tokio::server::Config::default(); server.listen.bind = Some("127.0.0.1:0".parse().unwrap()); server.listen.tls.generate = vec!["localhost".into()]; - let mut workers = Workers::bind(server, moq_tokio::worker::Config::new(2)).unwrap(); + let workers = Workers::bind(server, moq_tokio::worker::Config::new(2)).unwrap(); let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); - workers + let workers = workers .spawn( cluster, auth::Auth::refuse("test"), diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index de58da2f33e..ee2b633a37e 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -273,12 +273,14 @@ impl Web { } /// Serve HTTPS from the ACME resolver, which also answers TLS-ALPN-01 challenges. + #[cfg(feature = "acme")] pub(crate) fn with_acme(mut self, resolver: Option>) -> Self { self.acme = resolver; self } /// The bound HTTPS socket, where ACME answers challenges before serving starts. + #[cfg(feature = "acme")] pub(crate) fn https_listener(&self) -> Option<&net::TcpListener> { self.https_listener.as_ref() } @@ -301,7 +303,7 @@ impl Web { } /// Choose which moq-transport extensions WebSocket sessions offer in SETUP. - pub fn with_extensions(mut self, extensions: moq_net::setup::Extensions) -> Self { + pub(crate) fn with_extensions(mut self, extensions: moq_net::setup::Extensions) -> Self { self.extensions = extensions; self } @@ -563,11 +565,7 @@ fn build_acme_config( tls.resolver = Some(resolver); tls.root = root.to_vec(); - let alpn = vec![ - b"h2".to_vec(), - b"http/1.1".to_vec(), - rustls_acme::acme::ACME_TLS_ALPN_NAME.to_vec(), - ]; + let alpn = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"acme-tls/1".to_vec()]; tls.server_config(alpn).context("failed to build https TLS config") } @@ -718,7 +716,7 @@ impl HttpStream for TlsStream { self.get_ref().1.alpn_protocol() == Some(b"h2") } fn acme_challenge(&self) -> bool { - self.get_ref().1.alpn_protocol() == Some(rustls_acme::acme::ACME_TLS_ALPN_NAME) + self.get_ref().1.alpn_protocol() == Some(b"acme-tls/1") } fn mtls_peer(&self) -> Option { self.get_ref() diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index cd5784fdae3..aa66052b55a 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1200,7 +1200,7 @@ async fn an_mtls_client_ingests_here() { } /// The source the relay records for a broadcast announced over a certificate -/// session, admitted by `moq auth serve` with `mtls_peer` set as given. +/// session, admitted by `moq auth serve` with the peer link role when requested. async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let dir = tempfile::tempdir().expect("tempdir"); @@ -1208,7 +1208,7 @@ async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { let mut policy = moq_auth::serve::Policy::default(); policy.mtls = moq_auth::Permissions::new(all(), all()); - policy.mtls_peer = mtls_peer; + policy.mtls_link = mtls_peer.then_some(moq_auth::serve::Link::Peer); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let auth_url: url::Url = format!("http://{}/", listener.local_addr().unwrap()).parse().unwrap(); let server = moq_auth::serve::Server::new(policy).unwrap(); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 749c0c5fa0d..1fbd0758454 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -466,24 +466,43 @@ struct EmbeddedConfig { /// A flattened relay merges its settings without resetting the embedder's flags. #[test] fn embedded_cli_merges_only_relay_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write(&path, "[cluster]\nid = 7\n[stats]\nenabled = false\n").unwrap(); let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[ + path.as_os_str(), std::ffi::OsStr::new("--worker-name"), std::ffi::OsStr::new("recorder"), std::ffi::OsStr::new("--cluster-id"), std::ffi::OsStr::new("9"), ]) .expect("parse embedding CLI"); - let file = toml::from_str::("[cluster]\nid = 7\n").unwrap(); - let source = moq_tokio::cli::FileSource { - path: std::path::Path::new("relay.toml"), - value: &file, - }; - parsed - .relay - .merge_into(&cli, &usage_rs::config::EnvLayer::from_process(), Some(source)) - .unwrap(); + parsed.relay.merge(&cli).unwrap(); assert_eq!(parsed.worker_name.as_deref(), Some("recorder")); assert_eq!(parsed.relay.cluster.id, Some(9)); + assert_eq!(parsed.relay.source("cluster.id"), Some("--cluster-id")); + assert!(!parsed.relay.stats.enabled); + assert!(parsed.relay.source("stats.enabled").unwrap().contains("enabled")); + // The explicit file value equals the runtime default but still has file provenance. + let (mut defaults, cli) = EmbeddedConfig::parse_from_with_settings(&[]).unwrap(); + defaults.relay.merge(&cli).unwrap(); + assert_eq!(defaults.relay.stats.enabled, parsed.relay.stats.enabled); + assert_ne!( + defaults.relay.source("stats.enabled"), + parsed.relay.source("stats.enabled") + ); +} + +/// A composed CLI refuses obsolete file keys before constructing a relay or initializing logging. +#[test] +fn embedded_loader_refuses_obsolete_file_settings() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("relay.toml"); + std::fs::write(&path, "[server]\nbind = \"127.0.0.1:0\"\n").unwrap(); + let (mut parsed, cli) = EmbeddedConfig::parse_from_with_settings(&[path.as_os_str()]).unwrap(); + let error = parsed.relay.merge(&cli).unwrap_err(); + assert!(error.to_string().contains("[server]"), "{error}"); + assert!(error.to_string().contains("[listen]"), "{error}"); } /// The next route and whether it is active. diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 267afaf80fc..4a59d02fe1c 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -6,12 +6,13 @@ //! the drills. #![allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod auth_embedder; mod auth_lifetime; mod cluster_close; mod cluster_idle; mod cluster_unknown; +#[cfg(feature = "test-support")] mod connection; +#[cfg(feature = "test-support")] mod drain; mod embed; mod goaway_cluster; diff --git a/rs/moq-relay/tests/integration/memory.rs b/rs/moq-relay/tests/integration/memory.rs index c5c7c2367f1..f48689565bd 100644 --- a/rs/moq-relay/tests/integration/memory.rs +++ b/rs/moq-relay/tests/integration/memory.rs @@ -3,7 +3,9 @@ //! No sockets, so a test on Tokio's paused clock never waits on the host: the clock //! only advances once every message in flight has landed. +#[cfg(feature = "test-support")] use moq_net::transport::memory; +#[cfg(feature = "test-support")] use moq_relay::{Accepted, Connection, auth, cluster}; /// The version a test speaks unless it pins one: the newest the relay offers. @@ -17,6 +19,7 @@ pub fn newest() -> moq_net::Version { /// The relay's half runs as [`Connection::run`] on its own task, reported to the /// decider as QUIC. Lite 05+ answers SETUP before the verdict, so a refused session /// may connect and then close. +#[cfg(feature = "test-support")] pub async fn connect( client: moq_net::Client, version: moq_net::Version, @@ -47,6 +50,7 @@ pub async fn connect( /// An `Auth` granting everything under `public`, `public_publish`, and /// `public_subscribe` to anyone, as the `[auth]` config does. +#[cfg(feature = "test-support")] pub fn public(public: &[&str], publish: &[&str], subscribe: &[&str]) -> auth::Auth { let patterns = |texts: &[&str]| texts.iter().map(|text| text.parse().expect("pattern")).collect(); let mut config = auth::Config::default(); @@ -59,6 +63,7 @@ pub fn public(public: &[&str], publish: &[&str], subscribe: &[&str]) -> auth::Au } /// A standalone cluster with default options. +#[cfg(feature = "test-support")] pub fn cluster() -> cluster::Cluster { cluster::Cluster::new(cluster::Options::default()).expect("cluster init") } diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index 04edfbcd013..f6115559ea8 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -37,7 +37,7 @@ pub(crate) fn config(dir: &std::path::Path) -> Config { config.listen.tls.cert = vec![cert]; config.listen.tls.key = vec![key]; config.quic.max_streams = Some(VIEWER_STREAMS); - config.peer.quic.max_streams = std::num::NonZeroU64::new(4 * PROBE as u64).unwrap(); + config.limits.peer.quic.max_streams = std::num::NonZeroU64::new(4 * PROBE as u64).unwrap(); config } @@ -165,30 +165,39 @@ async fn refused_at_load(set: impl FnOnce(&mut Config)) -> String { } } -/// A limit QUIC cannot advertise or honor is refused at load, in `[peer.quic]` +/// A limit QUIC cannot advertise or honor is refused at load, in `[limits.peer.quic]` /// and `[quic]` alike, rather than wedging the peer or the handshake later. #[tokio::test] async fn unadvertisable_quic_limits_are_refused_at_load() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let err = refused_at_load(|config| { - config.peer.quic.receive_window = std::num::NonZeroU64::new(1 << 62).unwrap(); + config.limits.peer.quic.receive_window = std::num::NonZeroU64::new(1 << 62).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("receive_window"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("receive_window"), + "{err}" + ); // Incomplete messages hold connection credit, so the window has to fit the largest. let err = refused_at_load(|config| { let window = moq_net::transport::MIN_RECEIVE_WINDOW - 1; - config.peer.quic.receive_window = std::num::NonZeroU64::new(window).unwrap(); + config.limits.peer.quic.receive_window = std::num::NonZeroU64::new(window).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("largest message"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("largest message"), + "{err}" + ); let err = refused_at_load(|config| { - config.peer.quic.max_streams = std::num::NonZeroU64::new((1 << 60) + 1).unwrap(); + config.limits.peer.quic.max_streams = std::num::NonZeroU64::new((1 << 60) + 1).unwrap(); }) .await; - assert!(err.contains("[peer.quic]") && err.contains("max_streams"), "{err}"); + assert!( + err.contains("[limits.peer.quic]") && err.contains("max_streams"), + "{err}" + ); // Zero lets the peer open nothing, so it is refused too rather than wedging the connection. for max_streams in [0, (1 << 60) + 1] { diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index 4f4a034a986..a4eef2dec09 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,6 +4,14 @@ How to update your code for each breaking change, newest first. ## Unreleased +### Extension choices are Rust-only (#6535) + +Remove `[listen.extensions]` and `[connect.extensions]` from TOML. Explicit +values, including `false`, are refused. Native embedders can still set the typed +`listen::Config.extensions` / `connect::Config.extensions` fields before setup; +endpoint APIs and negotiated wire behavior are unchanged. + + ### The connection receive window is 128 MiB, and has a floor (#5878) `quic::DEFAULT_RECEIVE_WINDOW` is 128 MiB instead of 64 MiB, since incomplete messages now hold connection credit until complete. diff --git a/rs/moq-tokio/src/connect.rs b/rs/moq-tokio/src/connect.rs index e1767221ab8..9b3e351593b 100644 --- a/rs/moq-tokio/src/connect.rs +++ b/rs/moq-tokio/src/connect.rs @@ -364,16 +364,14 @@ mod tests { /// The dial config offers every extension unless told otherwise, and leaves the /// default out when serialized. #[test] - fn extensions_default_on_and_parse_off() { - let config: Config = toml::from_str("[extensions]\nsolicit = false\n").expect("parse"); + fn extensions_are_rust_only() { + for value in ["false", "true"] { + assert!(toml::from_str::(&format!("[extensions]\nsolicit = {value}\n")).is_err()); + } + let mut config = Config::default(); + config.extensions.solicit = false; assert!(!config.extensions.solicit); - assert!(config.extensions.auth); - assert!(toml::to_string(&config).expect("serialize").contains("solicit = false")); - assert!( - !toml::to_string(&Config::default()) - .expect("serialize") - .contains("extensions") - ); + assert!(!toml::to_string(&config).unwrap().contains("extensions")); } #[test] @@ -479,11 +477,6 @@ failover_delay = "1s" } } -/// Whether `extensions` is the default a config leaves out. -pub(crate) fn all_extensions(extensions: &moq_net::setup::Extensions) -> bool { - *extensions == moq_net::setup::Extensions::default() -} - /// The dial side of an endpoint: where to connect and how to get there. /// /// Derives [`usage_rs::Args`], so flatten it into a binary's own parser with @@ -641,8 +634,8 @@ pub struct Config { )] pub version: Vec, - /// The extensions to offer in SETUP, every moq-transport one by default. - #[serde(default, skip_serializing_if = "crate::connect::all_extensions")] + /// Extensions offered by a programmatic endpoint; unavailable in TOML. + #[serde(skip)] #[usage(skip)] pub extensions: moq_net::setup::Extensions, diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 3c14ca6fa3e..4d2086d95cc 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -146,8 +146,8 @@ pub struct Config { )] pub version: Vec, - /// The extensions to offer in SETUP, every moq-transport one by default. - #[serde(default, skip_serializing_if = "crate::connect::all_extensions")] + /// Extensions offered by a programmatic endpoint; unavailable in TOML. + #[serde(skip)] #[usage(skip)] pub extensions: moq_net::setup::Extensions, @@ -581,16 +581,14 @@ mod tests { /// The listen config offers every extension unless told otherwise, and leaves the /// default out when serialized. #[test] - fn extensions_default_on_and_parse_off() { - let config: Config = toml::from_str("[extensions]\nauth = false\n").expect("parse"); + fn extensions_are_rust_only() { + for value in ["false", "true"] { + assert!(toml::from_str::(&format!("[extensions]\nauth = {value}\n")).is_err()); + } + let mut config = Config::default(); + config.extensions.auth = false; assert!(!config.extensions.auth); - assert!(config.extensions.solicit); - assert!(toml::to_string(&config).expect("serialize").contains("auth = false")); - assert!( - !toml::to_string(&Config::default()) - .expect("serialize") - .contains("extensions") - ); + assert!(!toml::to_string(&config).unwrap().contains("extensions")); } /// A parser wrapping the config, since it derives `Args` (see the note in /// [`crate::connect`]). From 86456a35d97cd14452025a3d6052820627f4d5b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 03:30:03 -0700 Subject: [PATCH 3/8] docs: format relay configuration migrations Co-Authored-By: GPT-6 --- rs/moq-auth/MIGRATE.md | 3 +-- rs/moq-cli/MIGRATE.md | 3 +-- rs/moq-relay/MIGRATE.md | 3 +-- rs/moq-tokio/MIGRATE.md | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rs/moq-auth/MIGRATE.md b/rs/moq-auth/MIGRATE.md index bffb5bb646c..70c15a99585 100644 --- a/rs/moq-auth/MIGRATE.md +++ b/rs/moq-auth/MIGRATE.md @@ -4,14 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased -### A certificate has one cluster-link role (#6535) +### A certificate has one cluster-link role ([moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535)) Replace `serve::Policy.mtls_peer` and `.mtls_upstream` with `.mtls_link`: use `None` for ordinary certificate grants, `Some(serve::Link::Peer)` for peers, and `Some(serve::Link::Upstream)` for upstream peers. `Grant.peer` and `Grant.upstream` retain their existing JSON boolean contract. - ### `Client::connect` has no timeout of its own (#5765) `Client::connect` waits on the auth server for as long as its caller does, and dropping the future drops the request. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 748a3223db6..282ce3c4e3f 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,7 +4,7 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased -### Relay config and certificate roles (#6535) +### Relay config and certificate roles ([moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535)) Replace `moq auth serve --mtls-peer` with `--mtls-link peer`, and the combined peer/upstream flags with `--mtls-link upstream`. Omit the flag for ordinary @@ -15,7 +15,6 @@ Cluster peer-list flags are now `--cluster-api-url` and `--cluster-api-tls-root` the matching environment variables are `MOQ_CLUSTER_API_URL` and `MOQ_CLUSTER_API_TLS_ROOT`. TOML uses `[cluster.api] url` and `tls.root`. - ### `import fmp4` and `import hls` unwrap to Legacy (#6250) Both now publish one `legacy` frame per sample instead of one `cmaf` frame per fragment. diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 273ad07bc9f..bfc664f971b 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,7 +5,7 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased -### One loader and role-scoped config (#6535) +### One loader and role-scoped config ([moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535)) After an embedded CLI's `parse_from_with_settings`, call `config.relay.merge(&cli_layer)?`. It reads `relay.file`, applies CLI > environment > file > defaults, refuses obsolete @@ -39,7 +39,6 @@ and consuming `shutdown` on that value. A merely bound worker group cannot serve `Cluster::admit`, `cluster::Admitted`, and `auth::hold` are private; real admissions use the existing `Auth::embedded` / `Admissions` and `Lease` lifecycle. - ### `receive_window` has a floor (#5878) The `[quic]` and `[limits.peer.quic]` `receive_window` must be at least 67108880 bytes (64 MiB plus 16), or the relay refuses to start; the `[quic]` default is now 128 MiB. diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index a4eef2dec09..fec21f4cc8b 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,14 +4,13 @@ How to update your code for each breaking change, newest first. ## Unreleased -### Extension choices are Rust-only (#6535) +### Extension choices are Rust-only ([moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535)) Remove `[listen.extensions]` and `[connect.extensions]` from TOML. Explicit values, including `false`, are refused. Native embedders can still set the typed `listen::Config.extensions` / `connect::Config.extensions` fields before setup; endpoint APIs and negotiated wire behavior are unchanged. - ### The connection receive window is 128 MiB, and has a floor (#5878) `quic::DEFAULT_RECEIVE_WINDOW` is 128 MiB instead of 64 MiB, since incomplete messages now hold connection credit until complete. From 98aa4fb5972d40c5285fd914a005d0dc2c286aad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 09:15:34 -0700 Subject: [PATCH 4/8] quest: finish relay config, split embedder adoption The upstream loader lands here; embedding consumers migrate once their pins carry it, tracked by the new release-epic quest. Co-Authored-By: Claude Opus 5.5 --- quest/m1/api-audit/README.md | 1 - quest/m1/api-audit/relay-config.md | 47 ----------------------- quest/m2/release/README.md | 1 + quest/m2/release/relay-loader-adoption.md | 35 +++++++++++++++++ 4 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 quest/m1/api-audit/relay-config.md create mode 100644 quest/m2/release/relay-loader-adoption.md diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index 250bb6bc3fa..3c51022bc4b 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -88,7 +88,6 @@ Rejected in planning: - [CLI and gst flags](/quest/m1/api-audit/cli-flags.md) - one flag name per meaning across CLI subcommands and gst elements, and `linger` means one thing - [qmux shape](/quest/m1/api-audit/qmux-shape.md) - qmux binds framing at connect, shares flow-control names with moq-tokio, and JS drops the legacy wire - [Transport config](/quest/m1/api-audit/transport-config.md) - moq-tokio and moq-quic configs have one meaning for unset, enums for modes, no order-dependent setters, and no leftover features -- [Relay config](/quest/m1/api-audit/relay-config.md) - moq-relay and its embedders load config through one path that refuses old spellings, under names that do not collide - [Auth and clock names](/quest/m1/api-audit/auth-clock-names.md) - refusal and reason types have distinct names, and the Rust and JS clocks mirror each other ## Related diff --git a/quest/m1/api-audit/relay-config.md b/quest/m1/api-audit/relay-config.md deleted file mode 100644 index 50c4db09e37..00000000000 --- a/quest/m1/api-audit/relay-config.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] Relay config - -## Goal - -moq-relay and every embedder load configuration through one call that -refuses old spellings, and relay config names do not collide with module -names or each other. - -## Plan - -Decided 2026-10-22 (API audit). The public implementation is checkpointed in -[moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535): - -- `Config::merge(&CliLayer)` reads the optional file and environment, reuses - the existing Merge/Resolved provenance, and retains embedded CLI-only fields. -- Limits live under `[limits.client]` / `[limits.peer]`; ACME uses `cache` and - a default-enabled `acme` dependency feature; dynamic peers use - `[cluster.api] url` and `tls.root`. -- Verification concurrency is a private production constant, with a private - test override; the benchmark sweeps offered concurrency and token payload. -- Extension fields were not unused: the relay enables IETF and forwards its - offers to Web. Preserve the typed Rust fields and native setup API, refuse - their TOML surface including explicit false, and keep Web setup crate-private. -- Worker binding and spawning use `Workers` → `Spawned`; only the spawned - owner serves. Cluster API TLS comes from the existing dial template. -- mTLS policy uses one optional `Link::{Peer, Upstream}` role, preserving - the `Grant` JSON booleans. Test-only admission helpers are private; - in-memory `Incoming` / `Accepted` exports require `test-support`. -- Tracked demo, packaging, and test relay fixtures use the shared loader; - the intentional published legacy fixture is checked for refusal. - -Remaining completion prerequisite, not an unresolved public API policy: - -1. Land the upstream implementation. -2. Advance each pinned embedding consumer's dependency through its normal - upstream-first process. -3. Replace that consumer's direct TOML/update setup with the shared loader, - migrate renamed settings and native types, and run its own required checks. - Verify explicit obsolete file/environment spellings refuse startup and - embedded CLI-only fields survive, without default-value comparisons. -4. Remove this quest once every embedder uses the one-call loader. - -Do not pre-migrate consumers while their dependency pin lacks this API, add -compatibility aliases, or combine unrelated pending transport changes here. - -Docs: `doc/bin/relay/`. Public API: moq-relay/moq-auth native configuration and -moq CLI breaking, with package migrations. Wire: unchanged. diff --git a/quest/m2/release/README.md b/quest/m2/release/README.md index 7c2623de556..6409d8c61b5 100644 --- a/quest/m2/release/README.md +++ b/quest/m2/release/README.md @@ -42,3 +42,4 @@ production stability proven merely because upstream artifacts published. - [Retire the web-transport JS qmux](/quest/m2/release/qmux-js-retire.md) - once `@moq/qmux` publishes from `js/qmux`, deprecate `@moq/web-socket-stream` and stop the second publisher - [moq.sh deployment](/quest/m2/release/moq-sh-deploy.md) - verify its release-triggered publication - [Runner client bump](/quest/m2/release/runner-client-bump.md) - the upstream interop runner's Rust client moves to the published moq-tokio, so it stops stamping every object +- [Relay loader adoption](/quest/m2/release/relay-loader-adoption.md) - once their pins carry it, embedding consumers load relay config through `Config::merge` and refuse obsolete spellings diff --git a/quest/m2/release/relay-loader-adoption.md b/quest/m2/release/relay-loader-adoption.md new file mode 100644 index 00000000000..360259ab629 --- /dev/null +++ b/quest/m2/release/relay-loader-adoption.md @@ -0,0 +1,35 @@ +# [M] Embedders load relay config through the shared loader + +## Goal + +Condition: an embedding consumer's moq-relay pin includes +[moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535), which adds +`Config::merge(&CliLayer)`. Check each consumer's lockfile for a moq-relay +version or revision that carries it, and advance a pin that lacks it through +that consumer's normal upstream-first process. + +Then every embedding consumer loads its relay config through +`Config::merge`, an explicit obsolete file or environment spelling refuses +its startup, and its embedded CLI-only fields survive the merge. Delete this +quest once every embedder is migrated. + +## Plan + +Split from the relay config API audit (maintainer, 2026-10-23): the upstream +loader landed first, because migrating a consumer before its pin carries the +API would break it. + +For each embedding consumer: + +1. Replace its direct TOML parsing plus `update_from` (or hand-built + `merge_into`) setup with `config.relay.merge(&cli_layer)?`. +2. Migrate the renamed settings and native types listed under + [moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535) in + `rs/moq-relay/MIGRATE.md`, in its config files, environment, and code. +3. Verify with that consumer's own checks that an explicit obsolete file or + environment spelling refuses startup, and that its root CLI-only fields + survive, reading `Config::source` provenance rather than comparing against + default values. + +Do not add compatibility aliases upstream, and do not combine unrelated +pending transport configuration changes with a consumer's migration. From 3d60ad909b5b72aecdd743abf768cefacd38ee8a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 11:30:56 -0700 Subject: [PATCH 5/8] docs(relay): finish renaming acme.cache, cluster.api.url, and --mtls-link The production guide, packaging template, demo config, and crate README still used the removed spellings, which the loader now refuses. Co-Authored-By: Claude Opus 5.5 --- demo/relay/prod.toml | 2 +- doc/setup/prod.md | 2 +- packaging/moq-relay/relay.toml | 2 +- rs/moq-cli/src/main.rs | 2 +- rs/moq-relay/README.md | 2 +- rs/moq-relay/tests/integration/auth_lifetime.rs | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/demo/relay/prod.toml b/demo/relay/prod.toml index d30593368f0..fe4ca26be31 100644 --- a/demo/relay/prod.toml +++ b/demo/relay/prod.toml @@ -30,7 +30,7 @@ key = "key.pem" # validates over web.https, which must be reachable on TCP port 443. # [acme] # domains = ["relay.example.com"] -# dir = "/var/lib/moq-relay/acme" +# cache = "/var/lib/moq-relay/acme" # See docs/auth.md for more information. [auth] diff --git a/doc/setup/prod.md b/doc/setup/prod.md index 370cbb60d66..43ac32268e0 100644 --- a/doc/setup/prod.md +++ b/doc/setup/prod.md @@ -26,7 +26,7 @@ listen = "[::]:443" [acme] domains = ["relay.example.com"] contact = ["mailto:ops@example.com"] -dir = "/var/lib/moq-relay/acme" +cache = "/var/lib/moq-relay/acme" [auth] url = "http://127.0.0.1:4440/" diff --git a/packaging/moq-relay/relay.toml b/packaging/moq-relay/relay.toml index 249e6e2685d..2fe700f5b21 100644 --- a/packaging/moq-relay/relay.toml +++ b/packaging/moq-relay/relay.toml @@ -30,7 +30,7 @@ key = "/var/lib/moq-relay/key.pem" # [acme] # domains = ["relay.example.com"] # contact = ["mailto:ops@example.com"] -# dir = "/var/lib/moq-relay/acme" +# cache = "/var/lib/moq-relay/acme" # Uncomment to also serve plain HTTP/WebSocket on port 80. # [web.http] diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 5becb67f2a4..068fafc1c11 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -1424,7 +1424,7 @@ mod tests { /// An HTTP `--cluster-api-url` is a MoQ side, so start-up must attach /// client TLS the way the relay does rather than refuse after validate. #[tokio::test] - async fn cluster_connect_api_http_attaches_client_tls() { + async fn cluster_api_url_http_attaches_client_tls() { let _ = moq_tokio::crypto::install_default(); let invocation = Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) diff --git a/rs/moq-relay/README.md b/rs/moq-relay/README.md index df6d155bfcf..258ef3a41ec 100644 --- a/rs/moq-relay/README.md +++ b/rs/moq-relay/README.md @@ -62,7 +62,7 @@ HTTPS is currently not supported. Relays can be joined together to proxy announcements and subscriptions. A viewer talks to whichever relay is closest; if their broadcast lives elsewhere in the cluster, the local relay fetches it from a neighbor and caches it. Hop tracking on every broadcast keeps loops out and picks the shortest path when there's more than one. - `--cluster-connect ` lists the peers this relay dials. Repeatable; defines the topology by hand. A simple chain like `eu-west <- us-east <- us-west` lets `us-east` cache and dedup the transatlantic fetches that fan out to many `us-west` viewers. -- `--cluster-connect-api ` fetches that list from an endpoint or file instead, reloading it without a restart. +- `--cluster-api-url ` fetches that list from an endpoint or file instead, reloading it without a restart. - `--cluster-lan` finds peers on the local network over mDNS. A relay only dials peers listed this way or found on the LAN. diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index b38837b59b4..1e9a6ebc909 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -1183,7 +1183,7 @@ async fn a_certificate_admits_only_what_the_server_grants() { relay.abort(); } -/// A relay dialing in with a certificate, admitted by `moq auth serve --mtls-peer`, +/// A relay dialing in with a certificate, admitted by `moq auth serve --mtls-link peer`, /// is a cluster peer: what it announces entered the cluster elsewhere. #[tokio::test] async fn an_mtls_peer_is_a_cluster_peer() { @@ -1194,7 +1194,7 @@ async fn an_mtls_peer_is_a_cluster_peer() { ); } -/// Without `--mtls-peer`, a certificate identifies a client ingesting here. +/// Without `--mtls-link peer`, a certificate identifies a client ingesting here. #[tokio::test] async fn an_mtls_client_ingests_here() { assert_eq!(mtls_route_source(false).await, moq_net::origin::Source::Local); From b3103f1c95b19aa75b7c3293d86fce6f753ec698 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 13:52:05 -0700 Subject: [PATCH 6/8] fix(relay): keep the moq-transport extensions TOML The maintainer reversed the drop in #6581: [listen|connect.extensions] is the operator's interop knob for moq-transport extensions. Restore the moq-tokio serde surface and docs, and test that the shared loader carries it through. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/config.md | 3 +++ rs/moq-relay/MIGRATE.md | 5 ++--- rs/moq-relay/src/config.rs | 18 ++++++++---------- rs/moq-tokio/MIGRATE.md | 7 ------- rs/moq-tokio/src/connect.rs | 25 ++++++++++++++++--------- rs/moq-tokio/src/listen.rs | 20 +++++++++++--------- 6 files changed, 40 insertions(+), 38 deletions(-) diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 17903ee18aa..94a247c0a14 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -306,6 +306,9 @@ tls.cert = "relay.pem" # Present a client certificate (mTLS to pee tls.key = "relay.key" goaway.redirect = "same-host" # How far to trust a draining peer's redirect URI. goaway.handover = "10s" # Cap on how long the drained upstream keeps serving. +extensions.auth = false # Decline MoQ Auth on outbound moq-transport sessions. +extensions.solicit = false # Decline MoQ Solicit. +extensions.epoch = false # Decline MoQ Epoch. All default to on, as in [listen.extensions]. ``` A draining upstream may name a replacement URI. `same-host` follows it only diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index af800de8c0f..2cb72f0b118 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -29,9 +29,8 @@ client identity, with its own `api.tls.root` server trust. Remove `auth.verifications` / `--auth-verifications` / `MOQ_AUTH_VERIFICATIONS` and `Auth::with_verifications`; the verification window is a private production policy. -Remove `[listen.extensions]` and `[connect.extensions]` from TOML. The typed Rust -fields and native endpoint setup APIs remain; the relay still offers IETF support. -`Web::with_extensions` is private to the relay's own setup. +`Web::with_extensions` is private to the relay's own setup; set +`[listen.extensions]` instead. `Workers::spawn(self, …)` now returns `uring::Spawned`; call `serve`, `failed`, and consuming `shutdown` on that value. A merely bound worker group cannot serve. diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index ae87143ca65..22b6eef1eba 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -493,19 +493,17 @@ mod tests { assert!(error.to_string().contains("acme feature"), "{error}"); } - /// Both explicit values of a removed TOML knob must be refused by the common loader. + /// The moq-transport extension choices are file-only, so the loader must carry them through. #[test] - fn loader_refuses_toml_extension_settings() { + fn loader_keeps_toml_extension_settings() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("relay.toml"); - for side in ["listen", "connect"] { - for value in ["true", "false"] { - std::fs::write(&path, format!("[{side}.extensions]\nietf = {value}\n")).unwrap(); - let error = - Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]).unwrap_err(); - assert!(error.to_string().contains("extensions"), "{side} {value}: {error}"); - } - } + std::fs::write(&path, "[listen.extensions]\nauth = false\n\n[connect.extensions]\nsolicit = false\n").unwrap(); + let config = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.into()]).unwrap(); + assert!(!config.listen.extensions.auth); + assert!(config.listen.extensions.solicit); + assert!(!config.connect.extensions.solicit); + assert!(config.connect.extensions.auth); } #[test] diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index fec21f4cc8b..4f4a034a986 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,13 +4,6 @@ How to update your code for each breaking change, newest first. ## Unreleased -### Extension choices are Rust-only ([moq-dev/moq#6535](https://github.com/moq-dev/moq/pull/6535)) - -Remove `[listen.extensions]` and `[connect.extensions]` from TOML. Explicit -values, including `false`, are refused. Native embedders can still set the typed -`listen::Config.extensions` / `connect::Config.extensions` fields before setup; -endpoint APIs and negotiated wire behavior are unchanged. - ### The connection receive window is 128 MiB, and has a floor (#5878) `quic::DEFAULT_RECEIVE_WINDOW` is 128 MiB instead of 64 MiB, since incomplete messages now hold connection credit until complete. diff --git a/rs/moq-tokio/src/connect.rs b/rs/moq-tokio/src/connect.rs index 9b3e351593b..e1767221ab8 100644 --- a/rs/moq-tokio/src/connect.rs +++ b/rs/moq-tokio/src/connect.rs @@ -364,14 +364,16 @@ mod tests { /// The dial config offers every extension unless told otherwise, and leaves the /// default out when serialized. #[test] - fn extensions_are_rust_only() { - for value in ["false", "true"] { - assert!(toml::from_str::(&format!("[extensions]\nsolicit = {value}\n")).is_err()); - } - let mut config = Config::default(); - config.extensions.solicit = false; + fn extensions_default_on_and_parse_off() { + let config: Config = toml::from_str("[extensions]\nsolicit = false\n").expect("parse"); assert!(!config.extensions.solicit); - assert!(!toml::to_string(&config).unwrap().contains("extensions")); + assert!(config.extensions.auth); + assert!(toml::to_string(&config).expect("serialize").contains("solicit = false")); + assert!( + !toml::to_string(&Config::default()) + .expect("serialize") + .contains("extensions") + ); } #[test] @@ -477,6 +479,11 @@ failover_delay = "1s" } } +/// Whether `extensions` is the default a config leaves out. +pub(crate) fn all_extensions(extensions: &moq_net::setup::Extensions) -> bool { + *extensions == moq_net::setup::Extensions::default() +} + /// The dial side of an endpoint: where to connect and how to get there. /// /// Derives [`usage_rs::Args`], so flatten it into a binary's own parser with @@ -634,8 +641,8 @@ pub struct Config { )] pub version: Vec, - /// Extensions offered by a programmatic endpoint; unavailable in TOML. - #[serde(skip)] + /// The extensions to offer in SETUP, every moq-transport one by default. + #[serde(default, skip_serializing_if = "crate::connect::all_extensions")] #[usage(skip)] pub extensions: moq_net::setup::Extensions, diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 4d2086d95cc..3c14ca6fa3e 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -146,8 +146,8 @@ pub struct Config { )] pub version: Vec, - /// Extensions offered by a programmatic endpoint; unavailable in TOML. - #[serde(skip)] + /// The extensions to offer in SETUP, every moq-transport one by default. + #[serde(default, skip_serializing_if = "crate::connect::all_extensions")] #[usage(skip)] pub extensions: moq_net::setup::Extensions, @@ -581,14 +581,16 @@ mod tests { /// The listen config offers every extension unless told otherwise, and leaves the /// default out when serialized. #[test] - fn extensions_are_rust_only() { - for value in ["false", "true"] { - assert!(toml::from_str::(&format!("[extensions]\nauth = {value}\n")).is_err()); - } - let mut config = Config::default(); - config.extensions.auth = false; + fn extensions_default_on_and_parse_off() { + let config: Config = toml::from_str("[extensions]\nauth = false\n").expect("parse"); assert!(!config.extensions.auth); - assert!(!toml::to_string(&config).unwrap().contains("extensions")); + assert!(config.extensions.solicit); + assert!(toml::to_string(&config).expect("serialize").contains("auth = false")); + assert!( + !toml::to_string(&Config::default()) + .expect("serialize") + .contains("extensions") + ); } /// A parser wrapping the config, since it derives `Args` (see the note in /// [`crate::connect`]). From cd503216d5f648295275aa8d5528036a5d06fffa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 13:57:29 -0700 Subject: [PATCH 7/8] style(relay): format the extensions loader test Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/config.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 22b6eef1eba..7e7f8eb53a8 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -498,7 +498,11 @@ mod tests { fn loader_keeps_toml_extension_settings() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("relay.toml"); - std::fs::write(&path, "[listen.extensions]\nauth = false\n\n[connect.extensions]\nsolicit = false\n").unwrap(); + std::fs::write( + &path, + "[listen.extensions]\nauth = false\n\n[connect.extensions]\nsolicit = false\n", + ) + .unwrap(); let config = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.into()]).unwrap(); assert!(!config.listen.extensions.auth); assert!(config.listen.extensions.solicit); From 0b96c75a98ccda5fd030bb02814362b90ceac698 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:49:01 -0700 Subject: [PATCH 8/8] quest: cite the relay config PR from the uring extensions quest Co-Authored-By: Claude Opus 5.5 --- quest/m1/uring-listen-extensions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/uring-listen-extensions.md b/quest/m1/uring-listen-extensions.md index 840cf5499f0..65e7a1f5b7a 100644 --- a/quest/m1/uring-listen-extensions.md +++ b/quest/m1/uring-listen-extensions.md @@ -11,7 +11,7 @@ configured, as the tokio listeners already do. and never applies `listen.extensions`; the tokio path (`rs/moq-relay/src/relay.rs`, `websocket.rs`) does. Apply them and add a uring test that a configured extension is negotiated. -[Relay config](/quest/m1/api-audit/relay-config.md) keeps extensions as the +The relay config audit ([#6535](https://github.com/moq-dev/moq/pull/6535)) keeps extensions as the operator's interop knob, so every listener must honor it. Public API: none. Wire: none.