Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions demo/relay/leaf0.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ tls.key = ["leaf0.key"]
# To authenticate peers by certificate, trust client certificates signed by
# this CA and switch `[auth]` below to the auth server, which grants them
# as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and
# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a
# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a
# client CA alongside them.
# tls.root = ["ca.pem"]

Expand Down Expand Up @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/"]
public = "**"

# To exercise JWT + public-pattern auth instead, drop `public` above, run
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`,
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`,
# point the relay at it, and uncomment `listen.tls.root` above:
# url = "http://127.0.0.1:4440/"

Expand Down
4 changes: 2 additions & 2 deletions demo/relay/leaf1.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ tls.key = ["leaf1.key"]
# To authenticate peers by certificate, trust client certificates signed by
# this CA and switch `[auth]` below to the auth server, which grants them
# as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and
# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a
# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a
# client CA alongside them.
# tls.root = ["ca.pem"]

Expand Down Expand Up @@ -45,7 +45,7 @@ connect = ["https://localhost:4443/", "https://localhost:4444/"]
public = "**"

# To exercise JWT + public-pattern auth instead, drop `public` above, run
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`,
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`,
# point the relay at it, and uncomment `listen.tls.root` above:
# url = "http://127.0.0.1:4440/"

Expand Down
4 changes: 2 additions & 2 deletions demo/relay/prod.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ key = "key.pem"
# validates over web.https, which must be reachable on TCP port 443.
# [acme]
# domains = ["relay.example.com"]
# dir = "/var/lib/moq-relay/acme"
# cache = "/var/lib/moq-relay/acme"

# See docs/auth.md for more information.
[auth]
Expand All @@ -39,7 +39,7 @@ key = "key.pem"
# moq auth serve --listen 127.0.0.1:4440 --key root.jwk \
# --public-subscribe 'anon/**' --public-subscribe 'demo/**' \
# --public-publish 'anon/**' --public-publish 'demo/viewer/**' \
# --mtls-publish '**' --mtls-subscribe '**' --mtls-peer
# --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer
#
# For multiple keys with rotation, give it `--key-dir /path/to/keys/` instead:
# keys are named {kid}.jwk and resolved by the kid in the JWT header.
Expand Down
4 changes: 2 additions & 2 deletions demo/relay/root.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ tls.key = ["root.key"]
# To authenticate peers by certificate, trust client certificates signed by
# this CA and switch `[auth]` below to the auth server, which grants them
# as cluster peers through `--mtls-publish`, `--mtls-subscribe`, and
# `--mtls-peer`. Public rules ignore certificates, so the relay refuses a
# `--mtls-link peer`. Public rules ignore certificates, so the relay refuses a
# client CA alongside them.
# `just ca` generates the CA.
# tls.root = ["ca.pem"]
Expand All @@ -32,6 +32,6 @@ listen = "[::]:4443"
public = "**"

# To exercise JWT + public-pattern auth instead, drop `public` above, run
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`,
# `moq auth serve --key root.jwk --public-subscribe 'anon/**' --public-subscribe 'demo/**' --public-publish 'anon/**' --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`,
# point the relay at it, and uncomment `listen.tls.root` above:
# url = "http://127.0.0.1:4440/"
2 changes: 1 addition & 1 deletion doc/bin/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ MoQ process and `moq-relay` on the LAN, with no other configuration. Without
`--cluster-lan-secret`, anyone who can reach the listener joins, so leave it
unset only on networks you trust. `--cluster-connect` is a MoQ side on its own,
so it needs no `--connect`. WAN cluster dials (`--cluster-connect`,
`--cluster-connect-api`) share `--connect-bind` with `--connect`, and each binds
`--cluster-api-url`) share `--connect-bind` with `--connect`, and each binds
it, while LAN dials take an ephemeral port. So a fixed `--connect-bind` port with
both `--connect` and a WAN cluster peer is refused at startup: use port 0, or drop
`--connect` (unset `MOQ_CONNECT` if it supplies the URL).
Expand Down
17 changes: 8 additions & 9 deletions doc/bin/relay/auth.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ cannot verify a presented token; both refuse it as unsupported. A moq-transport
same way on any draft, but its grant authorizes only that request and ends with
it; it never joins the union. A `PUBLISH_NAMESPACE` grant must cover the whole
namespace: `room/**`, not just `room`. A session verifies at most
`verifications` tokens at once (`--auth-verifications`, default 128), counting
128 tokens at once, counting
checks still running past the admission timeout. Later tokens wait their turn
rather than being refused. Each HTTP request is admitted on its own, so an
HTTP/2 connection carries at most as many requests at once, and the client
Expand Down Expand Up @@ -267,14 +267,14 @@ verifies client certificates, so a relay without `listen.bind` refuses
`listen.tls.root` too.

Cluster peers are admitted the same way, so a mesh runs
`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer` (or a
`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` (or a
server granting its cluster CA everything with `peer: true`); see
[Clustering](/bin/relay/cluster). `--mtls-peer` marks every certificate as
[Clustering](/bin/relay/cluster). `--mtls-link peer` marks every certificate as
another relay; leave it off when certificates identify clients. Without
`peer`, a certificate is a client's: it routes like one and sees
[hidden](/concept/moq-lite#hidden-broadcasts) routes only by asking.
`--mtls-upstream`, which needs `--mtls-peer`, also marks them
[upstream](/bin/relay/cluster#upstream-links). Changing either flag ends each
`--mtls-link upstream` also marks them
[upstream](/bin/relay/cluster#upstream-links). Changing the link role ends each
live mTLS session at its next re-check (the `--revalidate` cadence or a
`moq auth revalidate` push), so the mesh redials once.

Expand All @@ -300,7 +300,7 @@ map future transport names to `unknown`, which your policy should handle explici
moq auth serve --listen 127.0.0.1:4440 \
--key-dir /etc/moq/keys \
--public-subscribe 'anon/**' --public-publish 'anon/**' \
--mtls-publish '**' --mtls-subscribe '**' --mtls-peer \
--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer \
--tier edge --expires 1d --revalidate 1m --limit-remote 64
```

Expand Down Expand Up @@ -350,7 +350,7 @@ patterns: `--auth-public 'PREFIX/**'`.
| `--auth-public-publish` / `--auth-public-subscribe` | `--public-publish` / `--public-subscribe`, as patterns |
| `--auth-public-api URL` | your own server answering the contract |
| `--auth-mtls-tier LABEL` | `--tier LABEL` (one tier per server) |
| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-peer` |
| `listen.tls.root` alone admitting a peer unscoped | `--mtls-publish '**' --mtls-subscribe '**' --mtls-link peer` |
| `--auth-api` (token or proxy mode), `Cache-Control` | `--auth-url` pointed at any server answering the contract; `revalidate` and `expires` in the grant |
| `--auth-domain` | your server reads `server_name` and decides |

Expand All @@ -375,8 +375,7 @@ session at the grant's `expires`, and a connection whose admission is still
unanswered at its `listen.timeout` handshake deadline is closed; a token
presented in band is refused after ten seconds. `Admission::attached()` marks a token presented
[in band](#in-band-renewal) for a session already admitted under the same id,
so count sessions only on the others. Gateway accept loops can call `Cluster::admit` for the same
scoped session as a native connection. See
so count sessions only on the others. See
[docs.rs/moq-relay](https://docs.rs/moq-relay) and
[moq-auth](/lib/rs/moq-auth).

Expand Down
20 changes: 11 additions & 9 deletions doc/bin/relay/cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ that is another source, so it restarts.
## Topology

List the peers each relay dials. That's the whole topology: a relay dials only
peers from `connect`, [`connect_api`](#dynamic-peer-lists), or
peers from `connect`, [`api.url`](#dynamic-peer-lists), or
[LAN discovery](#lan-discovery), never a URL learned from an announcement.

```toml
Expand All @@ -79,7 +79,7 @@ connect = ["https://us-east.example.com/"]
A chain (`eu-west <- us-east <- us-west`) dedupes fetches through the middle;
a full mesh trades that for one fewer hop. Mix shapes as your traffic demands.

For a full mesh, list every other relay, or serve the list from `connect_api`.
For a full mesh, list every other relay, or serve the list from `api.url`.
A session carries both directions, so one dial per pair is enough; listing a
pair on both sides opens a redundant second session.

Expand Down Expand Up @@ -124,7 +124,7 @@ connect = [
The mark belongs to the link, whichever side dialed: a dialed peer is upstream
when its `connect` entry says so, and a peer that dials in when its
[grant](/bin/relay/auth#the-contract) sets `"upstream": true` beside
`"peer": true`. `moq auth serve --mtls-peer --mtls-upstream` grants that to
`"peer": true`. `moq auth serve --mtls-link upstream` grants that to
every certificate, so use it only where cores are the only relays dialing in
with mTLS: on a hub that leaf relays dial into, it marks every leaf upstream
and the hub stops forwarding between them. A relay that predates the mark treats every link as
Expand Down Expand Up @@ -221,7 +221,7 @@ different `app` never discover each other.

## Dynamic peer lists

Point `connect_api` at an HTTP(S) endpoint or local file returning a JSON
Point `api.url` at an HTTP(S) endpoint or local file returning a JSON
array of peers: bare URL strings and/or the same objects `connect` accepts.
The relay re-checks it (honoring `Cache-Control`, or
watching the file) and reconciles: new peers are dialed, missing ones dropped,
Expand All @@ -233,15 +233,17 @@ changed URLs redialed. A bad fetch keeps the last good list.

```toml
[cluster]
connect_api = "https://api.example.com/cluster/peers"
node = "https://us-west.example.com/"

[cluster.api]
url = "https://api.example.com/cluster/peers"
```

HTTPS peer-list fetches present the same client certificate as cluster dials,
but authenticate the API with system roots and the hostname in its URL.
Mesh roots, fingerprints, insecure mode, and the mesh hostname override do not
apply to the API. For a private API, set `cluster.connect_api_tls_root` to PEM
paths (or repeat `--cluster-connect-api-tls-root`); these replace system trust
apply to the API. For a private API, set `cluster.api.tls.root` to PEM
paths (or repeat `--cluster-api-tls-root`); these replace system trust
for API fetches only. Malformed roots fail startup, and files reload for new
connections when rotated.

Expand Down Expand Up @@ -274,9 +276,9 @@ Peers dial with **mTLS** (recommended: `listen.tls.root` on the listener,
URL, `token` on a peer object, or a shared `cluster.token` file for every
listed peer). The accepting relay admits a peer like any client, so a mesh
needs an auth server that grants the cluster CA, such as
`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-peer`. An
`moq auth serve --mtls-publish '**' --mtls-subscribe '**' --mtls-link peer`. An
accepted peer counts as a cluster peer only when its grant sets `peer: true`,
as `--mtls-peer` does; otherwise what it announces counts as ingest here, like
as `--mtls-link peer` does; otherwise what it announces counts as ingest here, like
a client's, and it lists [hidden](/concept/moq-lite#hidden-broadcasts) routes
only by asking, which a relay below moq-lite-07-wip, or on moq-transport
without the hidden extension, cannot. Dials retry forever with jittered backoff
Expand Down
35 changes: 18 additions & 17 deletions doc/bin/relay/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,46 +99,46 @@ Raise the receive windows when a fat, long path idles below the link rate. The
128 MiB `receive_window` default carries several Gbps at a 100 ms RTT and bounds
how much unread data and incomplete messages one peer can make the relay
buffer. It has to fit the largest message, just over 64 MiB, so a smaller
value, here or in `[peer.quic]`, is refused at startup. When incomplete
value, here or in `[limits.peer.quic]`, is refused at startup. When incomplete
messages fill the window, the relay cancels the least urgent stream: the
lowest subscription priority, then the group its subscription would deliver
last. Over iroh the window bounds only
unread data, and over WebSocket or TCP nothing bounds incomplete messages.
`stream_receive_window` only paces one stream: its credit returns as bytes are
read, whether or not their message is complete.

## \[session] and \[peer]
## \[limits.client] and \[limits.peer]

Session limits apply separately to each session on every transport. The relay
uses `[peer]` for an admitted grant marked `peer` or `upstream`, including
uses `[limits.peer]` for an admitted grant marked `peer` or `upstream`, including
verified LAN membership, and for links it dials through the cluster. A client
certificate, route hop, or cluster-looking URL alone does not select peer limits.

```toml
[session]
[limits.client]
announces = 1000 # Simultaneous broadcasts or namespaces.
requests = 10000 # Other simultaneous moq-transport requests.

[peer]
[limits.peer]
announces = 100000
requests = 20000

[peer.quic]
[limits.peer.quic]
max_streams = 100000 # Concurrent QUIC streams per direction.
receive_window = 671088640 # Connection-wide receive window (640 MiB).
send_window = 671088640 # Unacknowledged outgoing data (640 MiB).
```

`[peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`;
`[limits.peer.quic]` takes only `max_streams`, `receive_window`, and `send_window`;
any other `[quic]` key is refused. They replace the `[quic]` values for a
peer's QUIC connection once its grant is known, before the SETUP reply. Links
the relay dials get them from the handshake on. A link carrying a whole cluster is then
not throttled by `[quic]` limits sized for one viewer, for example
`--peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep
`--limits-peer-quic-max-streams=200000`. Peers over WebSocket, TCP, or iroh keep
their transport's own limits.

These are the defaults. Each subscription and FETCH holds a stream on moq-lite
and on moq-transport draft 17 and later, so `[quic]` or `[peer.quic]`
and on moq-transport draft 17 and later, so `[quic]` or `[limits.peer.quic]`
`max_streams` bounds them there, and a peer past it waits rather than fails.
On drafts 14 to 16, `announces` and `requests` together size the request
window instead, which every request counts against, so very small values can
Expand Down Expand Up @@ -204,13 +204,14 @@ See [HTTP endpoints](/bin/relay/http).
## \[acme]

The relay provisions and renews a publicly trusted certificate itself, for QUIC
and HTTPS alike (Unix only, where the key can be kept owner-only):
and HTTPS alike (Unix only, where the key can be kept owner-only). This needs
the default `acme` Cargo feature; builds without it refuse configured issuance:

```toml
[acme]
domains = ["relay.example.com"] # Enables ACME. No wildcards.
contact = ["mailto:ops@example.com"] # Optional account contact.
dir = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only.
cache = "/var/lib/moq-relay/acme" # Required: account key and certificate, kept owner-only.
Comment thread
kixelated marked this conversation as resolved.
Resolved
# directory = "https://acme-staging-v02.api.letsencrypt.org/directory" # Default: Let's Encrypt production.
# timeout = "5m" # How long the first issuance may take before startup fails.
```
Expand All @@ -220,11 +221,11 @@ domain, so `web.https.listen` is required and must be reachable there, directly
or forwarded. `[acme]` cannot be combined with `listen.tls.cert`, `key`, or
`generate`, or with `web.https.cert` and `key`.

With no usable certificate in `dir` (missing, expired, missing a domain, or
With no usable certificate in `cache` (missing, expired, missing a domain, or
from another directory), startup answers only the challenge until one is
issued; QUIC, the other routes, and readiness wait for it, and startup fails
with the ACME error once `timeout` runs out. It also fails after 3 failed
validations within an hour, counting earlier starts recorded in `dir`, so a
validations within an hour, counting earlier starts recorded in `cache`, so a
restart loop stops short of Let's Encrypt's limit of 5 per hostname per hour. Renewal runs in the background
well before expiry and reaches every listener on its next handshake. A failed
renewal keeps the current certificate serving and retries.
Expand Down Expand Up @@ -267,8 +268,6 @@ url = "http://127.0.0.1:4440/" # An auth server asked once per session eve
# public = "anon/**" # Or a static anonymous grant rooted at /, publish and subscribe alike.
# public_subscribe = ["anon/**", "demo/**"] # Or split them; patterns, `foo/**` for a subtree.
# public_publish = ["anon/**"]
verifications = 128 # In-band tokens one session, or requests one HTTP/2 connection,
# has verifying at once; later ones wait.
```

See [Authentication](/bin/relay/auth).
Expand All @@ -279,12 +278,14 @@ See [Authentication](/bin/relay/auth).
[cluster]
connect = ["https://us-east.example.com/?cost=10"] # Peers to dial. ?cost prices the link, or use {url, cost, egress, token, upstream} objects.
node = "https://us-west.example.com/" # This relay's own URL.
# connect_api_tls_root = ["api-ca.pem"] # Private API roots, independent from connect.tls.
connect_api = "https://api.example.com/peers" # Or fetch the peer list (JSON array of URLs and/or objects) live.
token = "cluster.jwt" # JWT for dials without an inline ?jwt=.
id = 12345 # Stable Hop ID across restarts.
max_hops = 32 # Longest accepted/forwarded hop path (1 to 32).

[cluster.api]
url = "https://api.example.com/peers" # Live JSON peer list; a local file also works.
# tls.root = ["api-ca.pem"] # API server trust, independent from connect.tls.

[cluster.lan] # Find peers on the LAN over mDNS.
enabled = true
# secret = "/etc/moq/cluster.key" # Optional: a file holding 64 hex chars.
Expand Down
7 changes: 5 additions & 2 deletions doc/bin/relay/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,10 @@ between releases are listed in
`moq-relay` is also a library, for an application that wants extra HTTP
routes or in-process workers against the cluster origin. Load a `Relay`, take
what you need, and call `run`, which keeps the listeners, QUIC workers, and
shutdown inside the relay.
shutdown inside the relay. An embedded CLI calls
`config.relay.merge(&cli_layer)?` after its own `parse_from_with_settings` to
load the relay file and environment, retain provenance, and refuse obsolete
spellings. CLI-only fields on the embedding root stay intact.

```rust
use axum::routing::get;
Expand Down Expand Up @@ -89,7 +92,7 @@ relay.with_web(web).run().await?;
from `relay.shutdown().activity()` and pass `relay.shutdown().clone()` to
`Connection::with_shutdown`, so the drain waits for its handshakes too.
- To serve a session from any other moq-net transport, such as the in-memory
one in tests, read its SETUP with `moq_net::Server::accept_request` and pass
one in tests, enable the `test-support` feature, read its SETUP with `moq_net::Server::accept_request` and pass
`Accepted::new(handshake, transport)` to `Connection::new`. The client must
send its path in the SETUP.
- To decide admissions yourself, leave `[auth]` empty and answer
Expand Down
Loading
Loading