deploy(witness): fixes from bringing network.auths.dev up on Fly

.dockerignore

@@ -25,3 +25,12 @@ target/
 **/*.profraw
 coverage/
 dist/
+
+# Large non-source trees that a `cargo build` of a binary never needs — keeping
+# them out drops the build context from ~940 MB to a fraction of that.
+# (.dockerignore has no inline comments; crate-level tests under crates/*/tests
+# are unaffected — only the top-level suite is excluded.)
+tests/
+.recurve/
+.chunkhound/
+site/

crates/auths-witness-node/src/bin/witness_node.rs

@@ -218,6 +218,16 @@ fn build_state(args: &ServeArgs) -> Result<Arc<AppState>, String> {
 #[tokio::main]
 async fn main() -> std::process::ExitCode {
     let cli = Cli::parse();
+
+    // libgit2 refuses to touch a repo dir owned by a different user (the
+    // "dubious ownership" mitigation, CVE-2022-24765). In a single-tenant
+    // container the data volume's owner legitimately differs from the process
+    // — the node owns its data and the container/VM is the boundary — so opt
+    // out here, exactly as `safe.directory = *` does for the git CLI.
+    unsafe {
+        git2::opts::set_verify_owner_validation(false).ok();
+    }
+
     match cli.command {
         Command::Healthcheck(args) => match probe_health(&args.url) {
             Ok(()) => std::process::ExitCode::SUCCESS,
@@ -315,15 +325,12 @@ async fn main() -> std::process::ExitCode {
             // The registry role serves git transfers — legitimately larger
             // bodies over longer timeouts than an anchor POST — so it gets its
             // own envelope, merged after `core` is sealed so the two never mix.
-            // Fail closed (I-DEPLOY-6) if its repo or the `git` binary is absent.
             let app = if has("registry") {
-                if let Err(e) = registry_ready(&args.registry) {
-                    eprintln!(
-                        "witness-node: registry role: {e} (registry path: {})",
-                        args.registry.display()
-                    );
-                    return std::process::ExitCode::FAILURE;
-                }
+                // The serve surface only needs an initialized repo (guaranteed by
+                // ensure_registry, above) and the `git` binary — NOT a populated
+                // party registry. It serves an empty refs/auths/* until members
+                // exist, which is the correct first-witness state (unlike the
+                // anchor role, which fails closed without parties to resolve).
                 if !auths_witness_node::serve_registry::git_available().await {
                     eprintln!(
                         "witness-node: registry role: the `git` binary is required to serve \

deploy/witness/Dockerfile

@@ -14,9 +14,11 @@ FROM debian:bookworm-slim
 RUN apt-get update \
     && apt-get install -y --no-install-recommends git ca-certificates \
     && rm -rf /var/lib/apt/lists/*
-RUN useradd --system --home /data witness
 COPY --from=build /src/target/release/witness-node /usr/local/bin/witness-node
-USER witness
+# Runs as root: the node's data dir IS the mounted volume (anchor store +
+# registry), and a freshly attached volume mounts root-owned on fly, compose,
+# and k8s alike — a non-root process cannot create its state there. The node
+# exposes only the HTTP surface; the VM/container is the isolation boundary.
 VOLUME ["/data"]
 EXPOSE 3333
 ENTRYPOINT ["/usr/local/bin/witness-node"]

deploy/witness/fly/README.md

@@ -76,3 +76,67 @@ Never run more than one machine for a given seed: the anchor store is a
 single-writer, and two writers on one identity is exactly the equivocation the
 witness exists to prevent (`auto_stop_machines = 'off'`, `min_machines_running =
 1`, single machine).
+
+## Troubleshooting
+
+Everything below was learned bringing the first witness up. The build itself is
+reliable — nearly every "deploy failure" here was the **node rejecting its
+runtime environment on boot**, visible only after the image was already built.
+
+### First rule: get the node's *real* error (Fly hides a fast crash)
+
+A witness that rejects its environment exits in a few seconds. Fly's log
+shipping usually misses that window, so `fly logs` shows nothing and the deploy
+just reports `timeout reached waiting for health checks`. **Do not guess from
+that** — read the stderr directly:
+
+```bash
+# 1. Confirm it's crash-looping and see the exit code (1 = the node refused something).
+fly machine status <machine-id> --app auths-network      # read the Event Logs block
+#   stopped │ exit │ … exit_code=1,oom_killed=false,requested_stop=false   ← a real crash
+
+# 2. Free the volume, then launch a one-off that just sleeps (so you can poke at it).
+fly machine destroy <machine-id> --app auths-network --force
+IMG=$(fly image show --app auths-network --json | jq -r '.Ref')   # or the image: line from a deploy log
+fly machine run "$IMG" 900 --app auths-network \
+  --volume witness_data:/data --entrypoint /bin/sleep
+
+# 3. ssh in and run the EXACT command with a timeout — stderr comes back to YOUR terminal.
+SEED=0000000000000000000000000000000000000000000000000000000000000000   # any 64 hex chars
+fly ssh console --app auths-network --machine <diag-id> -C "sh -c '
+  id; git --version;
+  WITNESS_SEED=$SEED timeout 6 /usr/local/bin/witness-node \
+    serve --roles kel,cosign,registry --data-dir /data --registry /data/registry \
+    --bind 127.0.0.1:3399 --witness-name diag 2>&1 | head -40'"
+
+# 4. Clean up (and free the volume for the real deploy).
+fly machine destroy <diag-id> --app auths-network --force
+```
+
+The one-off mounts the *real* volume and runs the *real* binary, so it
+reproduces the failure exactly — and `fly ssh console -C` hands you the output
+that `fly logs` swallowed. This one technique found both bugs below.
+
+### Known failure modes
+
+| Symptom | Cause | Fix |
+|---|---|---|
+| `dockerfile '…/fly/deploy/witness/Dockerfile' not found` (doubled path) | flyctl resolves `[build] dockerfile` relative to the **config file's directory**, not the build context | Make the path relative to the config: `dockerfile = '../Dockerfile'`. Keep running `fly deploy` from the **repo root** (that's the build context `COPY . .` needs). |
+| `WARN Build context is ~940 MB` and slow uploads | `COPY . .` ships the whole working tree | `.dockerignore` the non-source trees (`tests/`, `.recurve/`, `.chunkhound/`, `site/`, `target/`, venvs). |
+| `registry init: … '/data/registry' is not owned by current user; class=Config (7); code=Owner (-36)` | libgit2's dubious-ownership check (CVE-2022-24765): the repo dir's owner ≠ the process user (a container volume, or a dir a *previous* run created as a different user) | The node opts out at startup (`git2::opts::set_verify_owner_validation(false)`). Debugging by hand: `git config --global --add safe.directory '*'`, or `chown -R <user> /data`. |
+| Machine `exit_code=1` immediately; anything under `/data` fails to create | a freshly attached Fly volume mounts **root-owned**, so a non-root container user cannot create its state there | Run the node as **root** — its data dir *is* the volume, and the VM/container is the isolation boundary (the node exposes only HTTP). |
+| `anchor role: this witness has no synced registry` | intentional fail-closed (RT-2): the anchor role refuses to start without parties to resolve against | A *fresh* witness has no members yet — boot with `--roles kel,cosign,registry` (the registry role serves an empty namespace fine). Add `anchor` once the registry has parties. |
+| deploy ends `timeout reached waiting for health checks` + Fly API `request canceled` | almost always the machine is **crash-looping** (health never comes up), not Fly flakiness | Don't retry blindly — run the "First rule" recipe; `fly machine status` will show the `exit_code`. |
+| A previous crash left junk / wrong-owned files on the volume | half-initialized state from failed boots | Wipe it from a one-off shell: `fly ssh console -C "sh -c 'rm -rf /data/registry /data/*.db /data/log /data/duplicity'"`, then redeploy (the node re-initializes on boot). |
+
+### Redeploying after a code fix
+
+`fly deploy --config deploy/witness/fly/fly.toml` from the repo root. A source
+change re-runs the full release compile (~3–4 min); only pure config/`fly.toml`
+edits are fast. After it reports success, always confirm the node actually
+serves — a green build is not a healthy witness:
+
+```bash
+curl -fsS https://<app>.fly.dev/health              # {"status":"ok","witness_did":"did:key:…"}
+git ls-remote https://<app>.fly.dev                 # registry smart-HTTP responds (empty is fine)
+```

deploy/witness/fly/fly.toml

@@ -2,7 +2,7 @@
 #
 # This runs the SAME open `auths-witness-node` image as the rest of
 # deploy/witness (../Dockerfile). Nothing here is network.auths.dev-specific
-# code — only config. A stranger stands up their own witness by copying this
+# code — only config. Your stand up your own witness by copying this
 # file and changing `app`, the region, and the domain.
 #
 # The Dockerfile's build context is the workspace root, so deploy FROM THE
@@ -23,15 +23,18 @@ app = 'auths-network'
 primary_region = 'lhr'
 
 [build]
-  # Relative to the build context (the repo root — run `fly deploy` from there).
-  dockerfile = 'deploy/witness/Dockerfile'
+  # Relative to THIS config file's dir (flyctl resolves the dockerfile path
+  # against the config, not the build context). The context is still the repo
+  # root — run `fly deploy` from there.
+  dockerfile = '../Dockerfile'
 
-# One witness, four roles: witness members (kel receipting, spend anchoring,
-# checkpoint cosigning) AND serve the held refs/auths/* read-only (registry) so
-# the node is its own resolution surface. This overrides the image's default
-# CMD to add the `registry` role and our witness name.
+# Bootstrap roles: KEL receipting, checkpoint cosigning, and serving the held
+# refs/auths/* read-only (registry). The `anchor` role is intentionally omitted
+# until members exist — it fails closed without a populated party registry to
+# resolve against (the RT-2 discipline), whereas the registry role happily
+# serves an empty namespace. Add `anchor` once the registry has parties.
 [processes]
-  app = 'serve --roles anchor,kel,cosign,registry --bind 0.0.0.0:3333 --data-dir /data --registry /data/registry --witness-name auths-network'
+  app = 'serve --roles kel,cosign,registry --bind 0.0.0.0:3333 --data-dir /data --registry /data/registry --witness-name auths-network'
 
 [[mounts]]
   # DURABLE, never tmpfs: the per-seed anchor store is the node's