diff --git a/.dockerignore b/.dockerignore index c78ea133..dd77991d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,3 +25,12 @@ target/ **/*.profraw coverage/ dist/ + +# Large non-source trees that a `cargo build` of a binary never needs — keeping +# them out drops the build context from ~940 MB to a fraction of that. +# (.dockerignore has no inline comments; crate-level tests under crates/*/tests +# are unaffected — only the top-level suite is excluded.) +tests/ +.recurve/ +.chunkhound/ +site/ diff --git a/crates/auths-witness-node/src/bin/witness_node.rs b/crates/auths-witness-node/src/bin/witness_node.rs index 047b1c16..30989ef8 100644 --- a/crates/auths-witness-node/src/bin/witness_node.rs +++ b/crates/auths-witness-node/src/bin/witness_node.rs @@ -218,6 +218,16 @@ fn build_state(args: &ServeArgs) -> Result, String> { #[tokio::main] async fn main() -> std::process::ExitCode { let cli = Cli::parse(); + + // libgit2 refuses to touch a repo dir owned by a different user (the + // "dubious ownership" mitigation, CVE-2022-24765). In a single-tenant + // container the data volume's owner legitimately differs from the process + // — the node owns its data and the container/VM is the boundary — so opt + // out here, exactly as `safe.directory = *` does for the git CLI. + unsafe { + git2::opts::set_verify_owner_validation(false).ok(); + } + match cli.command { Command::Healthcheck(args) => match probe_health(&args.url) { Ok(()) => std::process::ExitCode::SUCCESS, @@ -315,15 +325,12 @@ async fn main() -> std::process::ExitCode { // The registry role serves git transfers — legitimately larger // bodies over longer timeouts than an anchor POST — so it gets its // own envelope, merged after `core` is sealed so the two never mix. - // Fail closed (I-DEPLOY-6) if its repo or the `git` binary is absent. let app = if has("registry") { - if let Err(e) = registry_ready(&args.registry) { - eprintln!( - "witness-node: registry role: {e} (registry path: {})", - args.registry.display() - ); - return std::process::ExitCode::FAILURE; - } + // The serve surface only needs an initialized repo (guaranteed by + // ensure_registry, above) and the `git` binary — NOT a populated + // party registry. It serves an empty refs/auths/* until members + // exist, which is the correct first-witness state (unlike the + // anchor role, which fails closed without parties to resolve). if !auths_witness_node::serve_registry::git_available().await { eprintln!( "witness-node: registry role: the `git` binary is required to serve \ diff --git a/deploy/witness/Dockerfile b/deploy/witness/Dockerfile index 5ae323a6..61c8a1d8 100644 --- a/deploy/witness/Dockerfile +++ b/deploy/witness/Dockerfile @@ -14,9 +14,11 @@ FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends git ca-certificates \ && rm -rf /var/lib/apt/lists/* -RUN useradd --system --home /data witness COPY --from=build /src/target/release/witness-node /usr/local/bin/witness-node -USER witness +# Runs as root: the node's data dir IS the mounted volume (anchor store + +# registry), and a freshly attached volume mounts root-owned on fly, compose, +# and k8s alike — a non-root process cannot create its state there. The node +# exposes only the HTTP surface; the VM/container is the isolation boundary. VOLUME ["/data"] EXPOSE 3333 ENTRYPOINT ["/usr/local/bin/witness-node"] diff --git a/deploy/witness/fly/README.md b/deploy/witness/fly/README.md index 0e5b3cb0..953cbc4c 100644 --- a/deploy/witness/fly/README.md +++ b/deploy/witness/fly/README.md @@ -76,3 +76,67 @@ Never run more than one machine for a given seed: the anchor store is a single-writer, and two writers on one identity is exactly the equivocation the witness exists to prevent (`auto_stop_machines = 'off'`, `min_machines_running = 1`, single machine). + +## Troubleshooting + +Everything below was learned bringing the first witness up. The build itself is +reliable — nearly every "deploy failure" here was the **node rejecting its +runtime environment on boot**, visible only after the image was already built. + +### First rule: get the node's *real* error (Fly hides a fast crash) + +A witness that rejects its environment exits in a few seconds. Fly's log +shipping usually misses that window, so `fly logs` shows nothing and the deploy +just reports `timeout reached waiting for health checks`. **Do not guess from +that** — read the stderr directly: + +```bash +# 1. Confirm it's crash-looping and see the exit code (1 = the node refused something). +fly machine status --app auths-network # read the Event Logs block +# stopped │ exit │ … exit_code=1,oom_killed=false,requested_stop=false ← a real crash + +# 2. Free the volume, then launch a one-off that just sleeps (so you can poke at it). +fly machine destroy --app auths-network --force +IMG=$(fly image show --app auths-network --json | jq -r '.Ref') # or the image: line from a deploy log +fly machine run "$IMG" 900 --app auths-network \ + --volume witness_data:/data --entrypoint /bin/sleep + +# 3. ssh in and run the EXACT command with a timeout — stderr comes back to YOUR terminal. +SEED=0000000000000000000000000000000000000000000000000000000000000000 # any 64 hex chars +fly ssh console --app auths-network --machine -C "sh -c ' + id; git --version; + WITNESS_SEED=$SEED timeout 6 /usr/local/bin/witness-node \ + serve --roles kel,cosign,registry --data-dir /data --registry /data/registry \ + --bind 127.0.0.1:3399 --witness-name diag 2>&1 | head -40'" + +# 4. Clean up (and free the volume for the real deploy). +fly machine destroy --app auths-network --force +``` + +The one-off mounts the *real* volume and runs the *real* binary, so it +reproduces the failure exactly — and `fly ssh console -C` hands you the output +that `fly logs` swallowed. This one technique found both bugs below. + +### Known failure modes + +| Symptom | Cause | Fix | +|---|---|---| +| `dockerfile '…/fly/deploy/witness/Dockerfile' not found` (doubled path) | flyctl resolves `[build] dockerfile` relative to the **config file's directory**, not the build context | Make the path relative to the config: `dockerfile = '../Dockerfile'`. Keep running `fly deploy` from the **repo root** (that's the build context `COPY . .` needs). | +| `WARN Build context is ~940 MB` and slow uploads | `COPY . .` ships the whole working tree | `.dockerignore` the non-source trees (`tests/`, `.recurve/`, `.chunkhound/`, `site/`, `target/`, venvs). | +| `registry init: … '/data/registry' is not owned by current user; class=Config (7); code=Owner (-36)` | libgit2's dubious-ownership check (CVE-2022-24765): the repo dir's owner ≠ the process user (a container volume, or a dir a *previous* run created as a different user) | The node opts out at startup (`git2::opts::set_verify_owner_validation(false)`). Debugging by hand: `git config --global --add safe.directory '*'`, or `chown -R /data`. | +| Machine `exit_code=1` immediately; anything under `/data` fails to create | a freshly attached Fly volume mounts **root-owned**, so a non-root container user cannot create its state there | Run the node as **root** — its data dir *is* the volume, and the VM/container is the isolation boundary (the node exposes only HTTP). | +| `anchor role: this witness has no synced registry` | intentional fail-closed (RT-2): the anchor role refuses to start without parties to resolve against | A *fresh* witness has no members yet — boot with `--roles kel,cosign,registry` (the registry role serves an empty namespace fine). Add `anchor` once the registry has parties. | +| deploy ends `timeout reached waiting for health checks` + Fly API `request canceled` | almost always the machine is **crash-looping** (health never comes up), not Fly flakiness | Don't retry blindly — run the "First rule" recipe; `fly machine status` will show the `exit_code`. | +| A previous crash left junk / wrong-owned files on the volume | half-initialized state from failed boots | Wipe it from a one-off shell: `fly ssh console -C "sh -c 'rm -rf /data/registry /data/*.db /data/log /data/duplicity'"`, then redeploy (the node re-initializes on boot). | + +### Redeploying after a code fix + +`fly deploy --config deploy/witness/fly/fly.toml` from the repo root. A source +change re-runs the full release compile (~3–4 min); only pure config/`fly.toml` +edits are fast. After it reports success, always confirm the node actually +serves — a green build is not a healthy witness: + +```bash +curl -fsS https://.fly.dev/health # {"status":"ok","witness_did":"did:key:…"} +git ls-remote https://.fly.dev # registry smart-HTTP responds (empty is fine) +``` diff --git a/deploy/witness/fly/fly.toml b/deploy/witness/fly/fly.toml index 19f85caa..6c55f89c 100644 --- a/deploy/witness/fly/fly.toml +++ b/deploy/witness/fly/fly.toml @@ -2,7 +2,7 @@ # # This runs the SAME open `auths-witness-node` image as the rest of # deploy/witness (../Dockerfile). Nothing here is network.auths.dev-specific -# code — only config. A stranger stands up their own witness by copying this +# code — only config. Your stand up your own witness by copying this # file and changing `app`, the region, and the domain. # # The Dockerfile's build context is the workspace root, so deploy FROM THE @@ -23,15 +23,18 @@ app = 'auths-network' primary_region = 'lhr' [build] - # Relative to the build context (the repo root — run `fly deploy` from there). - dockerfile = 'deploy/witness/Dockerfile' + # Relative to THIS config file's dir (flyctl resolves the dockerfile path + # against the config, not the build context). The context is still the repo + # root — run `fly deploy` from there. + dockerfile = '../Dockerfile' -# One witness, four roles: witness members (kel receipting, spend anchoring, -# checkpoint cosigning) AND serve the held refs/auths/* read-only (registry) so -# the node is its own resolution surface. This overrides the image's default -# CMD to add the `registry` role and our witness name. +# Bootstrap roles: KEL receipting, checkpoint cosigning, and serving the held +# refs/auths/* read-only (registry). The `anchor` role is intentionally omitted +# until members exist — it fails closed without a populated party registry to +# resolve against (the RT-2 discipline), whereas the registry role happily +# serves an empty namespace. Add `anchor` once the registry has parties. [processes] - app = 'serve --roles anchor,kel,cosign,registry --bind 0.0.0.0:3333 --data-dir /data --registry /data/registry --witness-name auths-network' + app = 'serve --roles kel,cosign,registry --bind 0.0.0.0:3333 --data-dir /data --registry /data/registry --witness-name auths-network' [[mounts]] # DURABLE, never tmpfs: the per-seed anchor store is the node's