Phase 1: cluster write-federation - trust multiple writers (edge parse + master-trust script)

docker/fxsupport/linux/ipfs-cluster/ipfs-cluster-container-init.d.sh

@@ -70,11 +70,19 @@ get_poolcreator_peerid() {
   while [ $attempt -le $max_attempts ]; do
     response=$(curl -s --connect-timeout 10 --max-time 15 "${endpoint}" 2>/dev/null)
     cluster_peer_id=$(echo "$response" | jq -r '."ipfs-cluster-peerid" // empty' 2>/dev/null)
+    # Federation: prefer the trusted-peer ARRAY if the pool API provides it; fall back to
+    # the single legacy field. Result is a comma-separated list for CLUSTER_CRDT_TRUSTEDPEERS.
+    cluster_trusted_csv=$(echo "$response" | jq -r '(."ipfs-cluster-trustedpeers" // []) | map(select(. != null and . != "")) | join(",")' 2>/dev/null)
     kubo_peer_id=$(echo "$response" | jq -r '."kubo-peerid" // empty' 2>/dev/null)
 
-    if [ -n "$cluster_peer_id" ] && [ "$cluster_peer_id" != "null" ]; then
-      log "Fetched master cluster peer ID: $cluster_peer_id (attempt $attempt)"
-      export CLUSTER_CRDT_TRUSTEDPEERS="$cluster_peer_id"
+    resolved_trusted="$cluster_trusted_csv"
+    if [ -z "$resolved_trusted" ] || [ "$resolved_trusted" = "null" ]; then
+      resolved_trusted="$cluster_peer_id"
+    fi
+
+    if [ -n "$resolved_trusted" ] && [ "$resolved_trusted" != "null" ]; then
+      log "Fetched trusted cluster peers: $resolved_trusted (attempt $attempt)"
+      export CLUSTER_CRDT_TRUSTEDPEERS="$resolved_trusted"
       if [ -n "$kubo_peer_id" ] && [ "$kubo_peer_id" != "null" ]; then
         MASTER_KUBO_PEERID="$kubo_peer_id"
         log "Fetched master kubo peer ID: $kubo_peer_id"
@@ -182,6 +190,11 @@ append_or_replace "/.env.cluster" "CLUSTER_PEERNAME" "${CLUSTER_PEERNAME}"
     get_poolcreator_peerid
     append_or_replace "/.env.cluster" "CLUSTER_CRDT_TRUSTEDPEERS" "${CLUSTER_CRDT_TRUSTEDPEERS}"
 
+    # Federation: CLUSTER_CRDT_TRUSTEDPEERS may now be a comma-separated set of writers.
+    # The PRIMARY (first) peer is the bootstrap/tunnel target (the master); single-peer
+    # multiaddr construction below must use it, never the whole comma-separated list.
+    PRIMARY_TRUSTED_PEER=$(printf '%s' "${CLUSTER_CRDT_TRUSTEDPEERS}" | cut -d',' -f1)
+
     # Add master's kubo peer to follower's kubo Peering for faster content discovery
     if [ -n "${MASTER_KUBO_PEERID}" ] && [ "${MASTER_KUBO_PEERID}" != "${ipfs_peer_id}" ]; then
       # Add direct kubo addresses from API (non-localhost, non-relay)
@@ -306,8 +319,8 @@ append_or_replace "/.env.cluster" "CLUSTER_PEERNAME" "${CLUSTER_PEERNAME}"
                 }
             } |
             if $trust_peer != "" then
-                .cluster.peer_addresses = ["/ip4/127.0.0.1/tcp/19096/p2p/" + $trust_peer]
-                | .consensus.crdt.trusted_peers = [$trust_peer]
+                .cluster.peer_addresses = ["/ip4/127.0.0.1/tcp/19096/p2p/" + ($trust_peer | split(",")[0])]
+                | .consensus.crdt.trusted_peers = ($trust_peer | split(","))
             else . end
         ' "${IPFS_CLUSTER_PATH}/service.json" > "$service_temp" \
            && [ -s "$service_temp" ] \
@@ -338,7 +351,7 @@ append_or_replace "/.env.cluster" "CLUSTER_PEERNAME" "${CLUSTER_PEERNAME}"
         append_or_replace "/.env.cluster" "CLUSTER_FOLLOWERMODE" "${CLUSTER_FOLLOWERMODE}"
 
         # Construct the DNS fallback address (always reliable)
-        constructed_addr="/dns4/${poolName}.pools.functionyard.fula.network/tcp/9096/p2p/${CLUSTER_CRDT_TRUSTEDPEERS}"
+        constructed_addr="/dns4/${poolName}.pools.functionyard.fula.network/tcp/9096/p2p/${PRIMARY_TRUSTED_PEER}"
 
         # Populate peerstore and select bootstrap address
         if [ -n "${CLUSTER_BOOTSTRAP_ADDRS}" ]; then

tests/e2e/phase-1/10-master.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+#
+# fxe2e Phase-1 e2e — role: SIMULATED MASTER (test server only, NEVER production).
+#
+# Provisions an isolated test cluster master that mirrors the production master's
+# SHAPE (systemd unit -> docker run, env-driven, CLUSTER_CRDT_TRUSTEDPEERS on both the
+# Environment= line and the ExecStart -e flag) but with prefixed names + shifted ports
+# so the REAL phase-1-setup-writer.sh can run with its defaults on the same box.
+# Idempotent: re-run safe. Cluster name carries a random-but-fixed suffix so the
+# derived secret is not guessable on a public test box.
+#
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+. "$HERE/../../../update-scripts/lib/phase-common.sh"
+PC_TAG="fxe2e-master"
+
+CLUSTERNAME="${FXE2E_CLUSTERNAME:-fxe2e-vt9q4z}"
+SECRET="$(printf '%s' "$CLUSTERNAME" | sha256sum | cut -d' ' -f1)"
+BASE=/opt/fxe2e/master; KUBO_DIR="$BASE/kubo"; CL_DIR="$BASE/cluster"
+KUBO_IMAGE="${KUBO_IMAGE:-ipfs/kubo:release}"
+CLUSTER_IMAGE="${CLUSTER_IMAGE:-ipfs/ipfs-cluster:stable}"
+[ "$(id -u)" = 0 ] || die "run as root"
+pc_have docker || die "docker required (run phase-1-setup-writer.sh first or install docker)"
+mkdir -p "$KUBO_DIR" "$CL_DIR"
+
+# ---- kubo (shifted ports: swarm 14001, API 127.0.0.1:15001, gw 127.0.0.1:18080) ----
+# --entrypoint ipfs bypasses the image's auto-init entrypoint (double-init bug); config is
+# applied only on FRESH init (one-shot `ipfs config` needs the repo lock — re-runs with the
+# daemon up would fail; port changes require stopping the unit + wiping is fine: TEST ONLY).
+k() { docker run --rm --entrypoint ipfs -e IPFS_PATH=/data/ipfs -v "$KUBO_DIR":/data/ipfs "$KUBO_IMAGE" "$@"; }
+if [ -f "$KUBO_DIR/config" ]; then info "master kubo repo exists — skip init+config"
+else
+  info "init master kubo"
+  k init --profile=server >/dev/null
+  k config Addresses.API /ip4/127.0.0.1/tcp/15001 >/dev/null
+  k config Addresses.Gateway /ip4/127.0.0.1/tcp/18080 >/dev/null
+  k config --json Addresses.Swarm '["/ip4/0.0.0.0/tcp/14001"]' >/dev/null
+fi
+MASTER_KUBO_PEERID="$(jq -r '.Identity.PeerID // empty' "$KUBO_DIR/config")"; [ -n "$MASTER_KUBO_PEERID" ] || die "no master kubo peer id"
+
+ku_ch="$(cat <<EOF | pc_write_if_changed /etc/systemd/system/fxe2e-master-ipfs.service
+[Unit]
+Description=fxe2e master IPFS (TEST ONLY)
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+User=root
+ExecStartPre=-/usr/bin/docker rm -f fxe2e_m_ipfs
+ExecStart=/usr/bin/docker run -u root --rm --name fxe2e_m_ipfs --network host -e IPFS_PATH=/data/ipfs -v $KUBO_DIR:/data/ipfs $KUBO_IMAGE
+ExecStop=/usr/bin/docker stop -t 30 fxe2e_m_ipfs
+Restart=always
+RestartSec=10s
+TimeoutStopSec=60
+
+[Install]
+WantedBy=multi-user.target
+EOF
+)"
+systemctl daemon-reload; systemctl enable --now fxe2e-master-ipfs.service
+[ "$ku_ch" = changed ] && systemctl restart fxe2e-master-ipfs.service
+for i in $(seq 1 30); do curl -s -X POST http://127.0.0.1:15001/api/v0/id >/dev/null 2>&1 && break; [ "$i" = 30 ] && die "master kubo not healthy on :15001"; sleep 3; done
+info "master kubo healthy ($MASTER_KUBO_PEERID)"
+
+# ---- cluster (shifted ports: swarm 19096, REST 127.0.0.1:19094, proxy 127.0.0.1:19095) ----
+cl() { docker run --rm -e IPFS_CLUSTER_PATH=/data/ipfs-cluster -e CLUSTER_SECRET="$SECRET" -v "$CL_DIR":/data/ipfs-cluster --entrypoint ipfs-cluster-service "$CLUSTER_IMAGE" "$@"; }
+if [ -f "$CL_DIR/identity.json" ]; then info "master cluster identity exists — skip init"; else info "init master cluster"; cl init >/dev/null 2>&1 || cl init >/dev/null; fi
+MASTER_CLUSTER_PEERID="$(jq -r '.id' "$CL_DIR/identity.json")"
+{ [ -n "$MASTER_CLUSTER_PEERID" ] && [ "$MASTER_CLUSTER_PEERID" != null ]; } || die "no master cluster peer id"
+
+# Trust line mirrors prod shape: present on BOTH Environment= and ExecStart -e so
+# phase-1-master-trust.sh (UNIT_PATH/SERVICE_NAME overrides) edits it exactly like prod.
+# Re-runs PRESERVE the current trusted list (master-trust may have appended writers —
+# regenerating from the template must never revert that).
+TRUST_LINE="$MASTER_CLUSTER_PEERID"
+if [ -f /etc/systemd/system/fxe2e-master-ipfscluster.service ]; then
+  cur_trust="$(grep -oE 'CLUSTER_CRDT_TRUSTEDPEERS=[^" ]+' /etc/systemd/system/fxe2e-master-ipfscluster.service | head -1 | cut -d= -f2- || true)"
+  [ -n "$cur_trust" ] && TRUST_LINE="$cur_trust"
+fi
+cl_ch="$(cat <<EOF | pc_write_if_changed /etc/systemd/system/fxe2e-master-ipfscluster.service
+[Unit]
+Description=fxe2e master IPFSCLUSTER (TEST ONLY)
+After=fxe2e-master-ipfs.service
+Requires=fxe2e-master-ipfs.service
+
+[Service]
+Type=simple
+User=root
+Environment="CLUSTER_CRDT_TRUSTEDPEERS=$TRUST_LINE"
+ExecStartPre=-/usr/bin/docker rm -f fxe2e_m_cluster
+ExecStart=/usr/bin/docker run -u root --rm --name fxe2e_m_cluster --network host -e IPFS_CLUSTER_PATH=/data/ipfs-cluster -e CLUSTER_LISTENMULTIADDRESS=/ip4/0.0.0.0/tcp/19096 -e CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS=/ip4/127.0.0.1/tcp/19094 -e CLUSTER_IPFSPROXY_LISTENMULTIADDRESS=/ip4/127.0.0.1/tcp/19095 -e CLUSTER_PINSVCAPI_HTTPLISTENMULTIADDRESS=/ip4/127.0.0.1/tcp/19097 -e CLUSTER_IPFSHTTP_NODEMULTIADDRESS=/ip4/127.0.0.1/tcp/15001 -e CLUSTER_ALLOCATOR_ALLOCATEBY="tag:group,pinqueue,reposize" -e CLUSTER_REPLICATIONFACTORMIN=2 -e CLUSTER_REPLICATIONFACTORMAX=4 -e CLUSTER_DISABLEREPINNING=false -e CLUSTER_CLUSTERNAME=$CLUSTERNAME -e CLUSTER_SECRET=$SECRET -e CLUSTER_FOLLOWERMODE=false -e CLUSTER_CRDT_TRUSTEDPEERS=$TRUST_LINE -e CLUSTER_PEERNAME=fxe2e-master -e CLUSTER_MONITORPINGINTERVAL=15s -v $CL_DIR:/data/ipfs-cluster $CLUSTER_IMAGE daemon --upgrade
+ExecStop=/usr/bin/docker stop -t 30 fxe2e_m_cluster
+Restart=always
+RestartSec=10s
+TimeoutStopSec=60
+
+[Install]
+WantedBy=multi-user.target
+EOF
+)"
+systemctl daemon-reload; systemctl enable --now fxe2e-master-ipfscluster.service
+[ "$cl_ch" = changed ] && systemctl restart fxe2e-master-ipfscluster.service
+for i in $(seq 1 30); do docker exec fxe2e_m_cluster ipfs-cluster-ctl --host /ip4/127.0.0.1/tcp/19094 id >/dev/null 2>&1 && break; [ "$i" = 30 ] && die "master cluster API not healthy on :19094"; sleep 3; done
+info "master cluster healthy"
+
+PUB_IP="${PUBLIC_HOST:-$(hostname -I | awk '{print $1}')}"
+cat <<EOF
+[fxe2e-master] DONE (idempotent).
+  CLUSTERNAME            : $CLUSTERNAME
+  MASTER_CLUSTER_PEERID  : $MASTER_CLUSTER_PEERID
+  MASTER_KUBO_PEERID     : $MASTER_KUBO_PEERID
+  MASTER_CLUSTER_BOOTSTRAP: /ip4/$PUB_IP/tcp/19096/p2p/$MASTER_CLUSTER_PEERID
+Next: run the REAL writer script against this master, e.g.
+  PUBLIC_HOST=$PUB_IP CLUSTERNAME=$CLUSTERNAME \\
+  MASTER_CLUSTER_PEERID=$MASTER_CLUSTER_PEERID \\
+  MASTER_CLUSTER_BOOTSTRAP=/ip4/$PUB_IP/tcp/19096/p2p/$MASTER_CLUSTER_PEERID \\
+  bash ../../../update-scripts/phase-1-setup-writer.sh
+EOF

tests/e2e/phase-1/20-followers.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# fxe2e Phase-1 e2e — role: TWO FOLLOWERS on the test server (TEST ONLY).
+#
+#   follower A ("updated edge")  : trusts MASTER + WRITER   (kubo 24001/25001, cluster 29094/29096)
+#   follower B ("old edge")      : trusts MASTER only       (kubo 34001/35001, cluster 39094/39096)
+#
+# B models a storage provider that has NOT updated — the mixed-fleet/no-forced-upgrade
+# invariant says it must keep operating (serve + accept master-issued pins) and simply
+# not see writer-issued pins. Run AFTER 10-master.sh and phase-1-setup-writer.sh
+# (reads NEW_CLUSTER_PEERID from /opt/fula-writer/.env). Idempotent.
+#
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+. "$HERE/../../../update-scripts/lib/phase-common.sh"
+PC_TAG="fxe2e-followers"
+
+CLUSTERNAME="${FXE2E_CLUSTERNAME:-fxe2e-vt9q4z}"
+SECRET="$(printf '%s' "$CLUSTERNAME" | sha256sum | cut -d' ' -f1)"
+KUBO_IMAGE="${KUBO_IMAGE:-ipfs/kubo:release}"
+CLUSTER_IMAGE="${CLUSTER_IMAGE:-ipfs/ipfs-cluster:stable}"
+[ "$(id -u)" = 0 ] || die "run as root"
+
+MASTER_CL_DIR=/opt/fxe2e/master/cluster
+[ -f "$MASTER_CL_DIR/identity.json" ] || die "master not provisioned — run 10-master.sh first"
+MASTER_ID="$(jq -r '.id' "$MASTER_CL_DIR/identity.json")"
+[ -f /opt/fula-writer/.env ] || die "writer not provisioned — run phase-1-setup-writer.sh first"
+# shellcheck disable=SC1091
+WRITER_ID="$(. /opt/fula-writer/.env; printf '%s' "${NEW_CLUSTER_PEERID:-}")"
+WRITER_KUBO_ID="$(. /opt/fula-writer/.env; printf '%s' "${NEW_KUBO_PEERID:-}")"
+[ -n "$WRITER_ID" ] || die "NEW_CLUSTER_PEERID missing from /opt/fula-writer/.env"
+MASTER_KUBO_ID="$(jq -r '.Identity.PeerID // empty' /opt/fxe2e/master/kubo/config)"
+
+mk_follower() { # $1=name(fA|fB) $2=trusted_csv $3=kubo_swarm $4=kubo_api $5=cl_swarm $6=cl_rest $7=peername
+  local name="$1" trusted="$2" kswarm="$3" kapi="$4" clswarm="$5" clrest="$6" peername="$7"
+  local base="/opt/fxe2e/$name" kdir cdir gw proxy pinsvc
+  gw=$((kapi+1)); proxy=$((clrest+1)); pinsvc=$((clrest+3))   # all <65536, role-unique
+  kdir="$base/kubo"; cdir="$base/cluster"; mkdir -p "$kdir" "$cdir"
+
+  # --entrypoint ipfs bypasses auto-init entrypoint; config applied on fresh init only
+  # (one-shot `ipfs config` needs the repo lock — daemon-up re-runs would fail).
+  if [ ! -f "$kdir/config" ]; then
+    info "[$name] init kubo"
+    docker run --rm --entrypoint ipfs -e IPFS_PATH=/data/ipfs -v "$kdir":/data/ipfs "$KUBO_IMAGE" init --profile=server >/dev/null
+    docker run --rm --entrypoint ipfs -e IPFS_PATH=/data/ipfs -v "$kdir":/data/ipfs "$KUBO_IMAGE" config Addresses.API "/ip4/127.0.0.1/tcp/$kapi" >/dev/null
+    docker run --rm --entrypoint ipfs -e IPFS_PATH=/data/ipfs -v "$kdir":/data/ipfs "$KUBO_IMAGE" config Addresses.Gateway "/ip4/127.0.0.1/tcp/$gw" >/dev/null
+    docker run --rm --entrypoint ipfs -e IPFS_PATH=/data/ipfs -v "$kdir":/data/ipfs "$KUBO_IMAGE" config --json Addresses.Swarm "[\"/ip4/0.0.0.0/tcp/$kswarm\"]" >/dev/null
+  fi
+
+  if [ ! -f "$cdir/identity.json" ]; then
+    info "[$name] init cluster"
+    docker run --rm -e IPFS_CLUSTER_PATH=/data/ipfs-cluster -e CLUSTER_SECRET="$SECRET" -v "$cdir":/data/ipfs-cluster --entrypoint ipfs-cluster-service "$CLUSTER_IMAGE" init >/dev/null 2>&1 || true
+  fi
+  printf '/ip4/127.0.0.1/tcp/19096/p2p/%s\n' "$MASTER_ID" > "$cdir/peerstore"
+  case ",$trusted," in *",$WRITER_ID,"*) printf '/ip4/127.0.0.1/tcp/9096/p2p/%s\n' "$WRITER_ID" >> "$cdir/peerstore";; esac
+
+  docker rm -f "fxe2e_${name}_ipfs" "fxe2e_${name}_cluster" >/dev/null 2>&1 || true
+  docker run -d --restart unless-stopped --name "fxe2e_${name}_ipfs" --network host -e IPFS_PATH=/data/ipfs -v "$kdir":/data/ipfs "$KUBO_IMAGE" >/dev/null
+  for i in $(seq 1 30); do curl -s -X POST "http://127.0.0.1:$kapi/api/v0/id" >/dev/null 2>&1 && break; [ "$i" = 30 ] && die "[$name] kubo not healthy on :$kapi"; sleep 3; done
+
+  docker run -d --restart unless-stopped --name "fxe2e_${name}_cluster" --network host \
+    -e IPFS_CLUSTER_PATH=/data/ipfs-cluster \
+    -e CLUSTER_SECRET="$SECRET" \
+    -e CLUSTER_CLUSTERNAME="$CLUSTERNAME" \
+    -e CLUSTER_FOLLOWERMODE=true \
+    -e CLUSTER_CRDT_TRUSTEDPEERS="$trusted" \
+    -e CLUSTER_LISTENMULTIADDRESS="/ip4/0.0.0.0/tcp/$clswarm" \
+    -e CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS="/ip4/127.0.0.1/tcp/$clrest" \
+    -e CLUSTER_IPFSPROXY_LISTENMULTIADDRESS="/ip4/127.0.0.1/tcp/$proxy" \
+    -e CLUSTER_PINSVCAPI_HTTPLISTENMULTIADDRESS="/ip4/127.0.0.1/tcp/$pinsvc" \
+    -e CLUSTER_IPFSHTTP_NODEMULTIADDRESS="/ip4/127.0.0.1/tcp/$kapi" \
+    -e CLUSTER_REPLICATIONFACTORMIN=2 -e CLUSTER_REPLICATIONFACTORMAX=4 \
+    -e CLUSTER_DISABLEREPINNING=false \
+    -e CLUSTER_PEERNAME="$peername" \
+    -e CLUSTER_MONITORPINGINTERVAL=15s \
+    -v "$cdir":/data/ipfs-cluster \
+    "$CLUSTER_IMAGE" daemon --upgrade --bootstrap "/ip4/127.0.0.1/tcp/19096/p2p/$MASTER_ID" >/dev/null
+  for i in $(seq 1 30); do docker exec "fxe2e_${name}_cluster" ipfs-cluster-ctl --host "/ip4/127.0.0.1/tcp/$clrest" id >/dev/null 2>&1 && break; [ "$i" = 30 ] && die "[$name] cluster API not healthy on :$clrest"; sleep 3; done
+  info "[$name] healthy (trusts: $trusted)"
+
+  # Deterministic bitswap on one box: connect this follower's kubo to master + writer kubo.
+  docker exec "fxe2e_${name}_ipfs" ipfs swarm connect "/ip4/127.0.0.1/tcp/14001/p2p/$MASTER_KUBO_ID" >/dev/null 2>&1 || true
+  [ -n "$WRITER_KUBO_ID" ] && docker exec "fxe2e_${name}_ipfs" ipfs swarm connect "/ip4/127.0.0.1/tcp/4001/p2p/$WRITER_KUBO_ID" >/dev/null 2>&1 || true
+}
+
+mk_follower fA "$MASTER_ID,$WRITER_ID" 24001 25001 29096 29094 fxe2e-follower-new
+mk_follower fB "$MASTER_ID"            34001 35001 39096 39094 fxe2e-follower-old
+
+cat <<EOF
+[fxe2e-followers] DONE.
+  follower A (updated): trusts master+writer   REST :29094
+  follower B (old)    : trusts master only     REST :39094
+Next: bash 30-drills.sh
+EOF