diff --git a/Cargo.lock b/Cargo.lock index 11e02128..cfe657f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1181,6 +1181,7 @@ dependencies = [ "chrono", "clap", "ed25519-dalek", + "flate2", "git2", "hex", "http-body-util", diff --git a/crates/auths-witness-node/Cargo.toml b/crates/auths-witness-node/Cargo.toml index 9203a606..a53bf1b6 100644 --- a/crates/auths-witness-node/Cargo.toml +++ b/crates/auths-witness-node/Cargo.toml @@ -53,7 +53,10 @@ axum = { version = "0.8" } clap = { version = "4", features = ["derive", "env"] } tower = { version = "0.5", features = ["limit"] } tower-http = { version = "0.6", features = ["timeout"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync", "process", "io-util"] } +# Decompress gzip-encoded git-upload-pack request bodies (the `registry` role's +# git smart-HTTP surface); standard git clients gzip the fetch negotiation. +flate2 = "1" # Durable per-seed anchor state — the workspace's single `links = sqlite3` owner. sqlite = { version = "0.32", features = ["bundled"] } # The cosign role's signed-note signing primitive (as in auths-transparency). diff --git a/crates/auths-witness-node/src/bin/witness_node.rs b/crates/auths-witness-node/src/bin/witness_node.rs index 41865dc1..047b1c16 100644 --- a/crates/auths-witness-node/src/bin/witness_node.rs +++ b/crates/auths-witness-node/src/bin/witness_node.rs @@ -99,8 +99,11 @@ fn probe_health(url: &str) -> Result<(), String> { #[derive(Parser)] struct ServeArgs { /// Roles to serve (comma-separated): `anchor` (spend anchors), `kel` - /// (KERI receipt witnessing), `cosign` (checkpoint cosigning). All three - /// share one identity seed, one data dir, and one hardening envelope. + /// (KERI receipt witnessing), `cosign` (checkpoint cosigning), `registry` + /// (serve the held `refs/auths/*` read-only over git smart-HTTP, so the node + /// is its own resolution surface). anchor/kel/cosign share one identity seed, + /// one data dir, and one tight hardening envelope; `registry` carries its own + /// (larger bodies, longer timeout) for git transfers. #[arg(long, value_delimiter = ',', default_value = "anchor,kel,cosign")] roles: Vec, @@ -242,7 +245,7 @@ async fn main() -> std::process::ExitCode { Command::Serve(args) => { for role in &args.roles { match role.as_str() { - "anchor" | "kel" | "cosign" => {} + "anchor" | "kel" | "cosign" | "registry" => {} other => { eprintln!("witness-node: unknown role `{other}`"); return std::process::ExitCode::FAILURE; @@ -250,6 +253,18 @@ async fn main() -> std::process::ExitCode { } } let has = |role: &str| args.roles.iter().any(|r| r == role); + + // Roles that read (anchor: key resolution) or serve (registry) the + // party registry need it to be an openable repo. Ensure it exists — + // an empty registry is a valid new-witness state, and this lets the + // first node in a network (no peer to sync from) bootstrap cleanly. + if (has("anchor") || has("registry")) + && let Err(e) = auths_witness_node::sync::ensure_registry(&args.registry) + { + eprintln!("witness-node: registry init: {e:#}"); + return std::process::ExitCode::FAILURE; + } + let mut app = Router::new(); if has("anchor") { @@ -285,9 +300,9 @@ async fn main() -> std::process::ExitCode { } } - // One hardening envelope over every role — the same posture the - // standalone KEL witness shipped with. - let app = app + // The anchor/kel/cosign roles share one tight hardening envelope — + // the posture the standalone KEL witness shipped with. + let core = app .layer(DefaultBodyLimit::max(MAX_BODY_BYTES)) .layer(ConcurrencyLimitLayer::new( auths_witness_node::MAX_CONCURRENT_REQUESTS, @@ -296,6 +311,41 @@ async fn main() -> std::process::ExitCode { axum::http::StatusCode::REQUEST_TIMEOUT, auths_witness_node::REQUEST_TIMEOUT, )); + + // The registry role serves git transfers — legitimately larger + // bodies over longer timeouts than an anchor POST — so it gets its + // own envelope, merged after `core` is sealed so the two never mix. + // Fail closed (I-DEPLOY-6) if its repo or the `git` binary is absent. + let app = if has("registry") { + if let Err(e) = registry_ready(&args.registry) { + eprintln!( + "witness-node: registry role: {e} (registry path: {})", + args.registry.display() + ); + return std::process::ExitCode::FAILURE; + } + if !auths_witness_node::serve_registry::git_available().await { + eprintln!( + "witness-node: registry role: the `git` binary is required to serve \ + refs/auths/* but was not found on PATH" + ); + return std::process::ExitCode::FAILURE; + } + let registry = auths_witness_node::serve_registry::registry_router(&args.registry) + .layer(DefaultBodyLimit::max( + auths_witness_node::serve_registry::REGISTRY_MAX_BODY_BYTES, + )) + .layer(ConcurrencyLimitLayer::new( + auths_witness_node::serve_registry::REGISTRY_MAX_CONCURRENT_REQUESTS, + )) + .layer(TimeoutLayer::with_status_code( + axum::http::StatusCode::REQUEST_TIMEOUT, + auths_witness_node::serve_registry::REGISTRY_TIMEOUT, + )); + core.merge(registry) + } else { + core + }; let listener = match tokio::net::TcpListener::bind(&args.bind).await { Ok(listener) => listener, Err(e) => { diff --git a/crates/auths-witness-node/src/lib.rs b/crates/auths-witness-node/src/lib.rs index ded192e2..8283905c 100644 --- a/crates/auths-witness-node/src/lib.rs +++ b/crates/auths-witness-node/src/lib.rs @@ -43,6 +43,7 @@ pub mod engine; pub mod hardening; pub mod receipt; pub mod registry; +pub mod serve_registry; pub mod signer; pub mod sqlite_store; pub mod standup; diff --git a/crates/auths-witness-node/src/serve_registry.rs b/crates/auths-witness-node/src/serve_registry.rs new file mode 100644 index 00000000..d412d423 --- /dev/null +++ b/crates/auths-witness-node/src/serve_registry.rs @@ -0,0 +1,266 @@ +//! Read-only git smart-HTTP over the witness's held KEL store. +//! +//! A witness already holds the party KELs it receipts (under `refs/auths/*` in +//! the `--registry` repo). Exposing them read-only over git smart-HTTP makes the +//! node its own resolution surface — a verifier or a peer witness runs a plain +//! `git fetch +refs/auths/*:refs/auths/*` against it (exactly what +//! [`crate::sync`] does), with no separate registry server anywhere. This is the +//! "serve what you witness" leg of `docs/plans/network/network-auths-dev.md`. +//! +//! Only `git-upload-pack` (fetch) is ever invoked. There is no `receive-pack` +//! code path, so the surface is **read-only by construction** — not by config +//! that could be flipped. Writes are a separate, signed ingest (the KEL +//! receipting role), never an anonymous push. +//! +//! The heavy lifting — pkt-line framing, protocol negotiation, packfile +//! generation — is delegated to the `git` binary's own `upload-pack`, the +//! reference implementation, rather than reimplemented. The node passes the +//! client's `Git-Protocol` through so protocol v2 (which avoids advertising +//! every ref) works, and decompresses gzip request bodies, so any standard git +//! client interoperates. + +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Bytes; +use axum::extract::{RawQuery, State}; +use axum::http::{HeaderMap, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use tokio::io::AsyncWriteExt; +use tokio::process::Command; + +/// The only git service this node speaks. `receive-pack` is deliberately absent. +const SERVICE: &str = "git-upload-pack"; + +/// Smart-HTTP media types for the upload-pack service (RFC-less, but stable git +/// protocol strings). Kept as `&'static str` so a response's header pair is one +/// concrete type. +const ADVERTISEMENT_CONTENT_TYPE: &str = "application/x-git-upload-pack-advertisement"; +const RESULT_CONTENT_TYPE: &str = "application/x-git-upload-pack-result"; + +/// Each response is a per-fetch computation — never a cacheable fixed body. +const NO_CACHE: &str = "no-cache, max-age=0, must-revalidate"; + +/// Request-body cap for the fetch negotiation. A client's want/have list is +/// small (KB) even for large repos, but generous here so a deep fetch of a +/// many-ref registry is never truncated. Far above the anchor role's cap; the +/// registry routes carry their own envelope for exactly this reason. +pub const REGISTRY_MAX_BODY_BYTES: usize = 32 * 1024 * 1024; + +/// Concurrent `upload-pack` subprocesses to allow. Each fetch forks `git`, so +/// this bounds fork pressure independently of the anchor role's limit. +pub const REGISTRY_MAX_CONCURRENT_REQUESTS: usize = 32; + +/// Per-request timeout for the registry routes. A pack transfer legitimately +/// takes longer than an anchor POST, so it gets its own, larger budget. +pub const REGISTRY_TIMEOUT: Duration = Duration::from_secs(120); + +/// The read-only git-smart-HTTP router over `registry` — the repo whose +/// `refs/auths/*` this node serves. Mounted at the node root, so the git URL is +/// simply the node's base (`git fetch +refs/auths/*:refs/auths/*`). +/// +/// Args: +/// * `registry`: path to the local registry repo (the `--registry` dir). +/// +/// Usage: +/// ```ignore +/// let app = app.merge(serve_registry::registry_router(&args.registry)); +/// ``` +pub fn registry_router(registry: &Path) -> Router { + let repo = Arc::new(registry.to_path_buf()); + Router::new() + .route("/info/refs", get(info_refs)) + .route("/git-upload-pack", post(upload_pack)) + .with_state(repo) +} + +/// Whether the `git` binary is invocable — the registry role's hard dependency. +/// The node calls this at startup and refuses the role (I-DEPLOY-6, fail closed) +/// rather than 500-ing every fetch later. +pub async fn git_available() -> bool { + Command::new("git") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .await + .map(|s| s.success()) + .unwrap_or(false) +} + +/// `GET /info/refs?service=git-upload-pack` — the smart-HTTP ref advertisement. +async fn info_refs( + State(repo): State>, + headers: HeaderMap, + RawQuery(query): RawQuery, +) -> Response { + // Read-only: advertise nothing but upload-pack. A client asking for + // receive-pack (a push) is refused here — there is no write surface. + if query.as_deref() != Some(&format!("service={SERVICE}")) { + return ( + StatusCode::FORBIDDEN, + format!("only service={SERVICE} is served (read-only registry)\n"), + ) + .into_response(); + } + + let mut cmd = Command::new("git"); + cmd.args(["upload-pack", "--stateless-rpc", "--advertise-refs"]) + .arg(repo.as_path()); + apply_git_protocol(&mut cmd, &headers); + + let out = match cmd.output().await { + Ok(out) if out.status.success() => out.stdout, + Ok(out) => return git_failed("upload-pack --advertise-refs", &out.stderr), + Err(e) => return spawn_failed(e), + }; + + // Smart-HTTP wraps the advertisement in the service pkt-line + a flush. + let mut body = pkt_line(format!("# service={SERVICE}\n").as_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&out); + + ( + [ + (header::CONTENT_TYPE, ADVERTISEMENT_CONTENT_TYPE), + (header::CACHE_CONTROL, NO_CACHE), + ], + body, + ) + .into_response() +} + +/// `POST /git-upload-pack` — the fetch negotiation + packfile response. +async fn upload_pack( + State(repo): State>, + headers: HeaderMap, + body: Bytes, +) -> Response { + // git clients may gzip the request body; upload-pack expects it raw. + let input = if is_gzip(&headers) { + match gunzip(&body) { + Ok(decoded) => decoded, + Err(e) => { + return (StatusCode::BAD_REQUEST, format!("gzip decode: {e}\n")).into_response(); + } + } + } else { + body.to_vec() + }; + + let mut cmd = Command::new("git"); + cmd.args(["upload-pack", "--stateless-rpc"]) + .arg(repo.as_path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + apply_git_protocol(&mut cmd, &headers); + + let mut child = match cmd.spawn() { + Ok(child) => child, + Err(e) => return spawn_failed(e), + }; + + // Feed the negotiation to git on its own task while we drain stdout, so a + // large request can never deadlock against a filling stdout pipe. + if let Some(mut stdin) = child.stdin.take() { + tokio::spawn(async move { + let _ = stdin.write_all(&input).await; + // Dropping `stdin` closes it, signalling end-of-input to git. + }); + } + + let out = match child.wait_with_output().await { + Ok(out) if out.status.success() => out.stdout, + Ok(out) => return git_failed("upload-pack", &out.stderr), + Err(e) => return spawn_failed(e), + }; + + ( + [ + (header::CONTENT_TYPE, RESULT_CONTENT_TYPE), + (header::CACHE_CONTROL, NO_CACHE), + ], + out, + ) + .into_response() +} + +/// Frame `payload` as a single git pkt-line (4-hex length prefix + payload). +fn pkt_line(payload: &[u8]) -> Vec { + let mut line = format!("{:04x}", payload.len() + 4).into_bytes(); + line.extend_from_slice(payload); + line +} + +/// Pass the client's negotiated protocol version through to `git` so protocol +/// v2 (ref-filtered, essential at scale) is honoured; absent, git uses v0. +fn apply_git_protocol(cmd: &mut Command, headers: &HeaderMap) { + if let Some(proto) = headers.get("git-protocol").and_then(|v| v.to_str().ok()) { + cmd.env("GIT_PROTOCOL", proto); + } +} + +fn is_gzip(headers: &HeaderMap) -> bool { + headers + .get(header::CONTENT_ENCODING) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.eq_ignore_ascii_case("gzip")) +} + +fn gunzip(data: &[u8]) -> std::io::Result> { + use std::io::Read; + let mut decoder = flate2::read::GzDecoder::new(data); + let mut out = Vec::new(); + decoder.read_to_end(&mut out)?; + Ok(out) +} + +fn git_failed(what: &str, stderr: &[u8]) -> Response { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("git {what} failed: {}\n", String::from_utf8_lossy(stderr)), + ) + .into_response() +} + +fn spawn_failed(e: std::io::Error) -> Response { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("could not run git: {e}\n"), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pkt_line_prefixes_hex_length() { + // "# service=git-upload-pack\n" is 26 bytes → 26 + 4 = 30 = 0x1e. + let line = pkt_line(b"# service=git-upload-pack\n"); + assert_eq!(&line[..4], b"001e"); + assert_eq!(&line[4..], b"# service=git-upload-pack\n"); + } + + #[test] + fn gunzip_roundtrips() { + use std::io::Write; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(b"0011command=ls-refs\n0000").unwrap(); + let compressed = encoder.finish().unwrap(); + assert_eq!(gunzip(&compressed).unwrap(), b"0011command=ls-refs\n0000"); + } + + #[test] + fn only_upload_pack_service_is_accepted() { + // A defensive check that the constant is the fetch service, never + // receive-pack — the read-only guarantee depends on it. + assert_eq!(SERVICE, "git-upload-pack"); + } +} diff --git a/crates/auths-witness-node/src/sync.rs b/crates/auths-witness-node/src/sync.rs index 226d7798..9dbf1bec 100644 --- a/crates/auths-witness-node/src/sync.rs +++ b/crates/auths-witness-node/src/sync.rs @@ -13,6 +13,24 @@ use std::path::Path; use crate::registry::registry_ready; +/// Ensure `registry` is an initialized git repo (idempotent). +/// +/// An empty registry is a valid state — a witness with no members yet — so this +/// is what lets a fresh node (the first in a network, with no peer to sync from) +/// bootstrap cleanly: it starts serving/resolving an empty `refs/auths/*` that +/// receipts, a later peer sync, or a maintainer push then populate. `git init` +/// on an existing repo is a no-op, so this is safe to call on every start. +/// +/// Args: +/// * `registry`: the local dir the node reads/serves with `--registry`. +pub fn ensure_registry(registry: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(registry) + .map_err(|e| anyhow::anyhow!("create registry dir {}: {e}", registry.display()))?; + git2::Repository::init(registry) + .map_err(|e| anyhow::anyhow!("init registry dir {}: {e}", registry.display()))?; + Ok(()) +} + /// Fetch or refresh the parties' public registry at `registry` from `url`. /// /// Idempotent: creates and initializes the repo if absent, then force-fetches @@ -30,10 +48,9 @@ use crate::registry::registry_ready; /// sync_registry("https://github.com/auths-dev/registry", Path::new("/data/registry"))?; /// ``` pub fn sync_registry(url: &str, registry: &Path) -> anyhow::Result<()> { - std::fs::create_dir_all(registry) - .map_err(|e| anyhow::anyhow!("create registry dir {}: {e}", registry.display()))?; - let repo = git2::Repository::init(registry) - .map_err(|e| anyhow::anyhow!("init registry dir {}: {e}", registry.display()))?; + ensure_registry(registry)?; + let repo = git2::Repository::open(registry) + .map_err(|e| anyhow::anyhow!("open registry dir {}: {e}", registry.display()))?; let mut remote = repo .remote_anonymous(url) .map_err(|e| anyhow::anyhow!("open remote {url}: {e}"))?; diff --git a/crates/auths-witness-node/tests/cases/mod.rs b/crates/auths-witness-node/tests/cases/mod.rs index 4ab0cee2..0f85bd3f 100644 --- a/crates/auths-witness-node/tests/cases/mod.rs +++ b/crates/auths-witness-node/tests/cases/mod.rs @@ -1,4 +1,5 @@ mod anchor; mod config; mod cosign; +mod serve_registry; pub mod support; diff --git a/crates/auths-witness-node/tests/cases/serve_registry.rs b/crates/auths-witness-node/tests/cases/serve_registry.rs new file mode 100644 index 00000000..83ad3aa4 --- /dev/null +++ b/crates/auths-witness-node/tests/cases/serve_registry.rs @@ -0,0 +1,97 @@ +//! The `registry` role's git smart-HTTP surface. The fetch path is driven +//! end-to-end with the real `git` binary — the same `+refs/auths/*:refs/auths/*` +//! fetch the witness sync (`crate::sync`) performs — since a subprocess- and +//! protocol-sensitive path can't be covered by unit assertions alone. + +use std::path::Path; +use std::process::Command; + +use auths_witness_node::serve_registry::registry_router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tempfile::TempDir; +use tower::ServiceExt; + +/// Run a git command in `dir`, isolated from any host/global git config (so the +/// auths repo's own commit-signing never touches these throwaway repos), and +/// return stdout. Panics with stderr on failure. +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .unwrap_or_else(|e| panic!("spawn git {args:?}: {e}")); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn serves_refs_auths_over_git_smart_http() { + // A source repo holding a party KEL under refs/auths/* — the shape the + // anchor role resolves keys from and the registry role serves. + let src = TempDir::new().unwrap(); + git(src.path(), &["init", "-q"]); + std::fs::write(src.path().join("kel.cesr"), b"party-a kel\n").unwrap(); + git(src.path(), &["add", "."]); + git( + src.path(), + &["-c", "commit.gpgsign=false", "commit", "-q", "-m", "kel"], + ); + let head = git(src.path(), &["rev-parse", "HEAD"]).trim().to_string(); + git( + src.path(), + &["update-ref", "refs/auths/registry/party-a", &head], + ); + + // Serve it read-only on an ephemeral port. + let app = registry_router(src.path()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + // Fetch the custom namespace exactly as `witness-node sync-registry` does. + let dst = TempDir::new().unwrap(); + git(dst.path(), &["init", "-q"]); + let url = format!("http://{addr}"); + git( + dst.path(), + &["fetch", "-q", &url, "+refs/auths/*:refs/auths/*"], + ); + + // The party ref arrived and points at the same object we served. + let fetched = git(dst.path(), &["rev-parse", "refs/auths/registry/party-a"]) + .trim() + .to_string(); + assert_eq!(fetched, head, "fetched ref must match the served ref"); + + server.abort(); +} + +#[tokio::test] +async fn refuses_receive_pack_advertisement() { + // Read-only by construction: asking for the push service is refused before + // git is ever invoked, so no client can even begin to negotiate a write. + let dir = TempDir::new().unwrap(); + let response = registry_router(dir.path()) + .oneshot( + Request::builder() + .uri("/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} diff --git a/deploy/witness/Dockerfile b/deploy/witness/Dockerfile index 28ebf307..5ae323a6 100644 --- a/deploy/witness/Dockerfile +++ b/deploy/witness/Dockerfile @@ -8,6 +8,12 @@ COPY . . RUN cargo build --release -p auths-witness-node --bin witness-node FROM debian:bookworm-slim +# `git` is the reference implementation the `registry` role shells out to for +# git smart-HTTP (upload-pack). `ca-certificates` lets the in-process git2 +# HTTPS sync validate remotes. Both kept minimal; apt lists dropped. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* RUN useradd --system --home /data witness COPY --from=build /src/target/release/witness-node /usr/local/bin/witness-node USER witness diff --git a/deploy/witness/fly/README.md b/deploy/witness/fly/README.md new file mode 100644 index 00000000..0e5b3cb0 --- /dev/null +++ b/deploy/witness/fly/README.md @@ -0,0 +1,78 @@ +# deploy/witness/fly — network.auths.dev on Fly + +The first-party Auths witness, `network.auths.dev`, is a Fly deployment of the +**open** `auths-witness-node` image built by [`../Dockerfile`](../Dockerfile) — +the exact artifact anyone else runs. This directory is *only config*; there is +no `network.auths.dev`-specific code (see +[`docs/plans/network/network-auths-dev.md`](../../../docs/plans/network/network-auths-dev.md)). + +## What it runs + +`serve --roles anchor,kel,cosign,registry` — one node that both **witnesses +members** (KEL receipting, spend anchoring, checkpoint cosigning) and **serves +what it holds**: the party `refs/auths/*` read-only over git smart-HTTP, so the +node is its own resolution surface. + +| Surface | URL | +|---|---| +| Resolve KELs (git) | `git fetch https://network.auths.dev +refs/auths/*:refs/auths/*` | +| Submit a spend anchor | `POST https://network.auths.dev/v1/anchor` | +| Latest anchor / withholding view | `GET https://network.auths.dev/v1/anchor/{seed_id}` | +| Health | `GET https://network.auths.dev/health` | + +Writes are never anonymous: the registry is **read-only over git** (no +`receive-pack` path exists). Members' KELs enter via signed KEL receipting, a +peer sync, or a maintainer push — not a push to this endpoint. + +## First-time setup + +Run everything from the **repo root** (the Dockerfile's build context is the +workspace): + +| Step | Command | +|---|---| +| Create the app (don't deploy yet) | `fly launch --no-deploy --copy-config --config deploy/witness/fly/fly.toml --name auths-network` | +| Create the durable volume | `fly volumes create witness_data --region lhr --size 1 --app auths-network` | +| Set the witness seed (stable, secret) | `fly secrets set WITNESS_SEED="$(openssl rand -hex 32)" --app auths-network` | +| Deploy | `fly deploy --config deploy/witness/fly/fly.toml` | +| Add the domain | `fly certs add network.auths.dev --app auths-network` (then point DNS at the app) | + +The `WITNESS_SEED` is the node's identity — **keep it stable across restarts**; a +changed seed is a *new* witness. Store it only in Fly secrets. + +## Bootstrapping the registry + +The node **auto-initializes** an empty `/data/registry` on first boot — an empty +registry (no members yet) is a valid state, and it serves an empty +`refs/auths/*` until populated. Populate it by: + +- **members receipting** their KELs to this witness (the KEL role), or +- **a peer sync** from another witness — `fly ssh console -a auths-network -C "witness-node sync-registry --from --registry /data/registry"`, or +- a maintainer push of vetted KELs. + +## After deploy — verify it is a real witness + +```bash +# resolution surface is live (empty until members exist): +git ls-remote https://network.auths.dev + +# it fail-closes, it doesn't fake success: +curl -fsS https://network.auths.dev/health + +# publish its member key + endpoint in the /network directory (auths-site) so +# members can name it in their witness set. +``` + +## Operations + +| Task | Command | +|---|---| +| Logs | `fly logs --app auths-network` | +| Shell (has the volume) | `fly ssh console --app auths-network` | +| Rotate to a new seed (⚠ new identity) | `fly secrets set WITNESS_SEED=… ` then redeploy | +| Scale memory | edit `[[vm]] memory` in `fly.toml`, redeploy | + +Never run more than one machine for a given seed: the anchor store is a +single-writer, and two writers on one identity is exactly the equivocation the +witness exists to prevent (`auto_stop_machines = 'off'`, `min_machines_running = +1`, single machine). diff --git a/deploy/witness/fly/fly.toml b/deploy/witness/fly/fly.toml new file mode 100644 index 00000000..19f85caa --- /dev/null +++ b/deploy/witness/fly/fly.toml @@ -0,0 +1,66 @@ +# network.auths.dev — Auths's first-party witness, on Fly. +# +# This runs the SAME open `auths-witness-node` image as the rest of +# deploy/witness (../Dockerfile). Nothing here is network.auths.dev-specific +# code — only config. A stranger stands up their own witness by copying this +# file and changing `app`, the region, and the domain. +# +# The Dockerfile's build context is the workspace root, so deploy FROM THE +# REPO ROOT: +# +# # one-time: +# fly launch --no-deploy --copy-config --config deploy/witness/fly/fly.toml --name auths-network +# fly volumes create witness_data --region lhr --size 1 --app auths-network +# fly secrets set WITNESS_SEED="$(openssl rand -hex 32)" --app auths-network +# +# # each deploy (from the repo root): +# fly deploy --config deploy/witness/fly/fly.toml +# +# # custom domain (then point DNS at the app): +# fly certs add network.auths.dev --app auths-network + +app = 'auths-network' +primary_region = 'lhr' + +[build] + # Relative to the build context (the repo root — run `fly deploy` from there). + dockerfile = 'deploy/witness/Dockerfile' + +# One witness, four roles: witness members (kel receipting, spend anchoring, +# checkpoint cosigning) AND serve the held refs/auths/* read-only (registry) so +# the node is its own resolution surface. This overrides the image's default +# CMD to add the `registry` role and our witness name. +[processes] + app = 'serve --roles anchor,kel,cosign,registry --bind 0.0.0.0:3333 --data-dir /data --registry /data/registry --witness-name auths-network' + +[[mounts]] + # DURABLE, never tmpfs: the per-seed anchor store is the node's + # anti-equivocation memory — an amnesiac witness co-signs a fork after a + # restart. The served registry (refs/auths/*) lives here too. + source = 'witness_data' + destination = '/data' + +[http_service] + internal_port = 3333 + # The single process group defined above; required once [processes] is named. + processes = ['app'] + force_https = true + # A witness must not cold-start away its duty. Keep one machine always up and + # never auto-stop — auto-stopping drops the live anti-equivocation window. + auto_stop_machines = 'off' + auto_start_machines = false + min_machines_running = 1 + + [[http_service.checks]] + interval = '15s' + timeout = '2s' + grace_period = '10s' + method = 'get' + path = '/health' + +[[vm]] + # 1 GiB gives git pack operations headroom; a witness's reliability is worth + # more than the shared-CPU cost. + memory = '1gb' + cpu_kind = 'shared' + cpus = 1