server: synchronize qualified standby before fly releases

.github/workflows/integrations.yml

@@ -300,6 +300,8 @@ jobs:
         run: |
           go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.10 -color
           cd scripts && bun --no-env-file test ci && cd ..
+          python3 scripts/ci/standby-release-test.py
+          python3 scripts/ci/standby-release-install-test.py
           python3 -m unittest discover -s scripts/ci -p 'test_nightly_tip_gate.py'
       - name: Check CI shell scripts
         run: |

.github/workflows/server-deploy.yml

@@ -98,9 +98,37 @@ jobs:
           baseline=$(jq -ce -f "$filter" "$RUNNER_TEMP/machines.json")
           printf 'baseline=%s\n' "$baseline" >> "$GITHUB_OUTPUT"
 
+  standby-preflight:
+    name: Record the healthy fixed standby before DDL
+    needs: qualify
+    if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    environment: production
+    env:
+      STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }}
+      STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }}
+      STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }}
+      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
+      STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.sha }}
+      - name: Require the reviewed consumer and record exact predecessor
+        if: ${{ !inputs.bootstrap }}
+        run: python3 scripts/ci/standby-release-client.py preflight "$RUNNER_TEMP/standby-preflight.json"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: standby-preflight-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/standby-preflight.json
+          if-no-files-found: ignore
+          retention-days: 14
+
   migrate:
     name: Migrate and verify with the qualified image
-    needs: [qualify, preflight]
+    needs: [qualify, preflight, standby-preflight]
     if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -131,18 +159,76 @@ jobs:
           docker run --rm -e DATABASE_DIRECT_URL -e DATABASE_CONNECTION_MODE \
             "$RUNTIME_IMAGE" bun server/dist/verify-migrations.js
 
+  standby:
+    name: Synchronize the qualified ARM standby before replacing Fly
+    needs: [qualify, preflight, migrate]
+    if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    environment: production
+    env:
+      STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }}
+      STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }}
+      STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }}
+      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
+      STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }}
+      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+      BASELINE: ${{ needs.preflight.outputs.baseline }}
+      BOOTSTRAP: ${{ inputs.bootstrap }}
+    steps:
+      - uses: actions/checkout@v6
+        if: ${{ !inputs.bootstrap }}
+        with:
+          ref: ${{ github.sha }}
+      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
+        if: ${{ !inputs.bootstrap }}
+        with:
+          version: "0.4.107"
+      - name: Install the daemonless image transport on the ephemeral runner
+        if: ${{ !inputs.bootstrap }}
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq skopeo
+      - name: Export the exact tested ARM artifact without rebuilding
+        if: ${{ !inputs.bootstrap }}
+        run: bash scripts/ci/standby-release-export.sh "$RUNNER_TEMP/standby-image.tar"
+      - name: Keep a healthy unchanged Fly alternate while replacing Nuremberg
+        if: ${{ !inputs.bootstrap }}
+        run: |
+          test "$STANDBY_SYNC_ENABLED" = true
+          timeout --signal=KILL 10s flyctl machine list --config server/fly.toml --json > "$RUNNER_TEMP/fly-before-standby.json"
+          current=$(jq -ce -f scripts/ci/fly-release-machines.jq "$RUNNER_TEMP/fly-before-standby.json")
+          test "$current" = "$BASELINE"
+      - name: Stream and reconcile the fixed standby
+        if: ${{ !inputs.bootstrap }}
+        run: python3 scripts/ci/standby-release-client.py sync "$RUNNER_TEMP/standby-sync.json" "$RUNNER_TEMP/standby-image.tar"
+      - name: Remove the temporary image archive
+        if: ${{ always() && !inputs.bootstrap }}
+        run: rm -f "$RUNNER_TEMP/standby-image.tar"
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() && !inputs.bootstrap }}
+        with:
+          name: standby-sync-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/standby-sync.json
+          if-no-files-found: ignore
+          retention-days: 14
+
   deploy:
     name: Replace the public Machines and verify readiness
-    needs: [qualify, preflight, migrate]
+    needs: [qualify, preflight, migrate, standby]
     if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}
     runs-on: ubuntu-latest
     timeout-minutes: 20
     environment: production
     outputs:
       bootstrap_baseline: ${{ steps.verify.outputs.bootstrap_baseline }}
     env:
-      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+      STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }}
+      STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }}
+      STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }}
       RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
+      STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }}
+      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
       BASELINE: ${{ needs.preflight.outputs.baseline }}
       BOOTSTRAP: ${{ inputs.bootstrap }}
     steps:
@@ -152,6 +238,9 @@ jobs:
       - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
         with:
           version: "0.4.107"
+      - name: Require fresh exact standby identity and direct TLS readiness
+        if: ${{ !inputs.bootstrap }}
+        run: python3 scripts/ci/standby-release-client.py check "$RUNNER_TEMP/standby-before-fly.json"
       - name: Revalidate the recorded baseline and deploy the same digest
         run: |
           test -n "$FLY_API_TOKEN"
@@ -201,6 +290,18 @@ jobs:
           if [[ "$BOOTSTRAP" = true ]]; then
             printf 'API-only bootstrap is ready. Qualify authenticated paths, transfer routing, and stop the old all-role API before approving production-cutover.\n' >> "$GITHUB_STEP_SUMMARY"
           fi
+      - name: Verify both origins before completing the fleet release
+        if: ${{ !inputs.bootstrap }}
+        run: |
+          python3 scripts/ci/standby-release-client.py complete "$RUNNER_TEMP/standby-complete.json"
+          printf 'Nuremberg passed exact ARM source/digest/config readback and independent direct TLS readiness before and after Fly replacement.\n' >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: standby-final-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/standby-*.json
+          if-no-files-found: ignore
+          retention-days: 14
 
   activate:
     name: Enable workers on the same qualified bootstrap image

.github/workflows/server-test.yml

@@ -1,6 +1,9 @@
 name: Server Tests
 
 on:
+  # Validation-only entry point for a stacked PR whose base is not main.
+  # No publication input exists here; only the guarded reusable call can publish.
+  workflow_dispatch:
   workflow_call:
     inputs:
       publish_image:
@@ -80,7 +83,12 @@ jobs:
         with:
           bun-version: "1.4.0"
       - name: Check server container and Fly configuration contracts
-        run: bun test scripts/docker/server-context.test.ts scripts/docker/prune-workspace.test.ts scripts/ci/fly-release-machines.test.ts
+        run: |
+          bun test scripts/docker/server-context.test.ts scripts/docker/prune-workspace.test.ts scripts/ci/fly-release-machines.test.ts
+          python3 scripts/ci/standby-release-test.py
+          bash -n scripts/ci/standby-release-export.sh
+          sh -n scripts/ci/standby-release-entrypoint.sh
+          sh -n scripts/ci/standby-release-controller-entrypoint.sh
       - name: Export committed server build context
         run: bun --no-env-file scripts/docker/server-context.ts "$GITHUB_SHA" "$RUNNER_TEMP/server-context"
       - name: Build runtime and Coolify targets
@@ -192,6 +200,27 @@ jobs:
           docker run --rm inline-server:ci-coolify wget --version
           docker run --rm --network host -e DATABASE_URL inline-server:ci-coolify bun server/dist/core-production-smoke.js --artifact
 
+      - name: Qualify the daemonless ARM archive and bounded recovery reader
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq skopeo
+          archive="$RUNNER_TEMP/standby-transport-test.tar"
+          registry=inline-standby-ci-registry
+          trap 'docker stop "$registry" >/dev/null 2>&1 || true; rm -f "$archive"' EXIT
+          docker run --detach --rm --name "$registry" --publish 127.0.0.1:55000:5000 registry:2 >/dev/null
+          for attempt in {1..20}; do
+            if curl --silent --fail http://127.0.0.1:55000/v2/ >/dev/null; then break; fi
+            if [[ "$attempt" = 20 ]]; then exit 1; fi
+            sleep 1
+          done
+          image=127.0.0.1:55000/inline-api:transport-test
+          docker tag inline-server:ci-coolify "$image"
+          docker push "$image" >/dev/null
+          immutable=$(docker image inspect --format '{{json .RepoDigests}}' "$image" | jq -er '.[] | select(startswith("127.0.0.1:55000/inline-api@sha256:"))')
+          skopeo copy --src-tls-verify=false --preserve-digests "docker://$immutable" "oci-archive:$archive:release" >/dev/null
+          digest="${immutable##*@}"
+          python3 scripts/ci/standby-release-test.py --oci-archive "$archive" "$GITHUB_SHA" "$digest"
+
       - name: Setup Fly registry client
         if: ${{ inputs.publish_image }}
         uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1

scripts/ci/fly-release-machines.test.ts

@@ -99,13 +99,21 @@ test("both image publishing guards require manual same-repository branches and e
   }
 })
 
+test("manual Server Tests dispatch exposes no publication inputs", async () => {
+  const workflow = Bun.YAML.parse(await Bun.file(resolve(import.meta.dir, "../../.github/workflows/server-test.yml")).text()) as {
+    on: Record<string, { inputs?: Record<string, unknown> } | null>
+  }
+  expect(Object.hasOwn(workflow.on, "workflow_dispatch")).toBe(true)
+  expect(workflow.on.workflow_dispatch?.inputs).toBeUndefined()
+})
+
 type ReleaseJob = {
   if?: string
   needs?: string | string[]
   environment?: string
   uses?: string
   with?: Record<string, unknown>
-  steps?: { name?: string; run?: string; env?: Record<string, string> }[]
+  steps?: { name?: string; run?: string; if?: string; env?: Record<string, string> }[]
 }
 const releaseWorkflow = async () => Bun.YAML.parse(await Bun.file(
   resolve(import.meta.dir, "../../.github/workflows/server-deploy.yml"),
@@ -114,6 +122,32 @@ const releaseWorkflow = async () => Bun.YAML.parse(await Bun.file(
   jobs: Record<string, ReleaseJob>
 }
 
+test("ordinary releases synchronize and re-read Nuremberg before Fly without broad credentials", async () => {
+  const workflow = await releaseWorkflow()
+  expect(workflow.jobs.migrate?.needs).toContain("standby-preflight")
+  expect(workflow.jobs.standby?.needs).toEqual(["qualify", "preflight", "migrate"])
+  expect(workflow.jobs.deploy?.needs).toContain("standby")
+  for (const id of ["standby-preflight", "standby"]) {
+    expect(workflow.jobs[id]?.if).toBe("${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}")
+    expect(workflow.jobs[id]?.environment).toBe("production")
+    expect(JSON.stringify(workflow.jobs[id])).not.toMatch(/COOLIFY_TOKEN|CLOUDFLARE_TOKEN|ROOT_SSH_KEY/)
+  }
+  const standbySteps = workflow.jobs.standby?.steps ?? []
+  const alternate = standbySteps.findIndex((step) => step.name === "Keep a healthy unchanged Fly alternate while replacing Nuremberg")
+  const sync = standbySteps.findIndex((step) => step.run?.includes("standby-release-client.py sync"))
+  expect(alternate).toBe(sync - 1)
+  expect(standbySteps[alternate]?.run).toContain('test "$current" = "$BASELINE"')
+  expect(standbySteps[sync]?.if).toBe("${{ !inputs.bootstrap }}")
+  const steps = workflow.jobs.deploy?.steps ?? []
+  const check = steps.findIndex((step) => step.run?.includes("standby-release-client.py check"))
+  const deploy = steps.findIndex((step) => step.run?.includes("flyctl deploy"))
+  const flyReadback = steps.findIndex((step) => step.run?.includes("fly-release-wait.sh"))
+  const complete = steps.findIndex((step) => step.run?.includes("standby-release-client.py complete"))
+  expect(check).toBe(deploy - 1)
+  expect(complete).toBeGreaterThan(flyReadback)
+  for (const step of [steps[check], steps[complete]]) expect(step?.if).toBe("${{ !inputs.bootstrap }}")
+})
+
 test("publish-only keeps full CI and excludes every production job; normal and failed releases retain their gates", async () => {
   const workflow = await releaseWorkflow()
   expect(workflow.on.workflow_dispatch.inputs.publish_only).toMatchObject({ type: "boolean", default: false })
@@ -148,7 +182,7 @@ test("publish-only keeps full CI and excludes every production job; normal and f
   }
   for (const [publishOnly, expected] of [
     [true, ["validate", "qualify", "prepared"]],
-    [false, ["validate", "qualify", "preflight", "migrate", "deploy"]],
+    [false, ["validate", "qualify", "preflight", "standby-preflight", "migrate", "standby", "deploy"]],
   ] as const) {
     expect(Object.keys(workflow.jobs).filter((id) => runnable(id, publishOnly)).sort()).toEqual([...expected].sort())
     for (const [id, job] of Object.entries(workflow.jobs)) {
@@ -163,10 +197,12 @@ test("publish-only keeps full CI and excludes every production job; normal and f
   expect(runnable("qualify", true, "validate")).toBe(false)
   expect(runnable("migrate", false, "preflight")).toBe(false)
   expect(runnable("deploy", false, "migrate")).toBe(false)
+  expect(runnable("migrate", false, "standby-preflight")).toBe(false)
+  expect(runnable("deploy", false, "standby")).toBe(false)
   expect(JSON.stringify(workflow.jobs.prepared)).not.toContain("secrets.")
   expect(workflow.jobs.prepared?.environment).toBeUndefined()
   expect(Object.keys(workflow.jobs).filter((job) => runnable(job, false, undefined, true)).sort())
-    .toEqual(["activate", "deploy", "migrate", "preflight", "qualify", "validate"])
+    .toEqual(["activate", "deploy", "migrate", "preflight", "qualify", "standby", "standby-preflight", "validate"])
   expect(runnable("activate", false, "deploy", true)).toBe(false)
   for (const ref of ["refs/heads/main", "refs/heads/codex/multi-node-ha", "refs/tags/main"]) {
     for (const publishOnly of [true, false]) {