From 37c2bbbf69a575d24203656dff2d3670f879e08b Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Sat, 17 Oct 2026 01:17:50 +0330 Subject: [PATCH 1/5] server: gate fly releases on qualified standby sync --- .github/workflows/server-deploy.yml | 107 +++- .github/workflows/server-test.yml | 27 +- scripts/ci/fly-release-machines.test.ts | 34 +- scripts/ci/standby-release-client.py | 119 +++++ .../standby-release-controller-entrypoint.sh | 16 + scripts/ci/standby-release-controller.py | 105 ++++ scripts/ci/standby-release-entrypoint.sh | 15 + scripts/ci/standby-release-export.sh | 14 + scripts/ci/standby-release-host.py | 469 +++++++++++++++++ scripts/ci/standby-release-test.py | 474 ++++++++++++++++++ server/docs/fly-deployment.md | 16 +- server/docs/hetzner-standby.md | 5 + server/docs/standby-release-sync.md | 148 ++++++ 13 files changed, 1536 insertions(+), 13 deletions(-) create mode 100644 scripts/ci/standby-release-client.py create mode 100644 scripts/ci/standby-release-controller-entrypoint.sh create mode 100644 scripts/ci/standby-release-controller.py create mode 100644 scripts/ci/standby-release-entrypoint.sh create mode 100644 scripts/ci/standby-release-export.sh create mode 100644 scripts/ci/standby-release-host.py create mode 100644 scripts/ci/standby-release-test.py create mode 100644 server/docs/standby-release-sync.md diff --git a/.github/workflows/server-deploy.yml b/.github/workflows/server-deploy.yml index 6c843f4df..da517dcf8 100644 --- a/.github/workflows/server-deploy.yml +++ b/.github/workflows/server-deploy.yml @@ -98,9 +98,37 @@ jobs: baseline=$(jq -ce -f "$filter" "$RUNNER_TEMP/machines.json") printf 'baseline=%s\n' "$baseline" >> "$GITHUB_OUTPUT" + standby-preflight: + name: Record the healthy fixed standby before DDL + needs: qualify + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: production + env: + STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }} + STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }} + STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }} + RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }} + STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + - name: Require the reviewed consumer and record exact predecessor + if: ${{ !inputs.bootstrap }} + run: python3 scripts/ci/standby-release-client.py preflight "$RUNNER_TEMP/standby-preflight.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: standby-preflight-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/standby-preflight.json + if-no-files-found: ignore + retention-days: 14 + migrate: name: Migrate and verify with the qualified image - needs: [qualify, preflight] + needs: [qualify, preflight, standby-preflight] if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} runs-on: ubuntu-latest timeout-minutes: 15 @@ -131,9 +159,63 @@ jobs: docker run --rm -e DATABASE_DIRECT_URL -e DATABASE_CONNECTION_MODE \ "$RUNTIME_IMAGE" bun server/dist/verify-migrations.js + standby: + name: Synchronize the qualified ARM standby before replacing Fly + needs: [qualify, preflight, migrate] + if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: production + env: + STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }} + STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }} + STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }} + RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }} + STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + BASELINE: ${{ needs.preflight.outputs.baseline }} + BOOTSTRAP: ${{ inputs.bootstrap }} + steps: + - uses: actions/checkout@v6 + if: ${{ !inputs.bootstrap }} + with: + ref: ${{ github.sha }} + - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 + if: ${{ !inputs.bootstrap }} + with: + version: "0.4.107" + - name: Install the daemonless image transport on the ephemeral runner + if: ${{ !inputs.bootstrap }} + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq skopeo + - name: Export the exact tested ARM artifact without rebuilding + if: ${{ !inputs.bootstrap }} + run: bash scripts/ci/standby-release-export.sh "$RUNNER_TEMP/standby-image.tar" + - name: Keep a healthy unchanged Fly alternate while replacing Nuremberg + if: ${{ !inputs.bootstrap }} + run: | + test "$STANDBY_SYNC_ENABLED" = true + timeout --signal=KILL 10s flyctl machine list --config server/fly.toml --json > "$RUNNER_TEMP/fly-before-standby.json" + current=$(jq -ce -f scripts/ci/fly-release-machines.jq "$RUNNER_TEMP/fly-before-standby.json") + test "$current" = "$BASELINE" + - name: Stream and reconcile the fixed standby + if: ${{ !inputs.bootstrap }} + run: python3 scripts/ci/standby-release-client.py sync "$RUNNER_TEMP/standby-sync.json" "$RUNNER_TEMP/standby-image.tar" + - name: Remove the temporary image archive + if: ${{ always() && !inputs.bootstrap }} + run: rm -f "$RUNNER_TEMP/standby-image.tar" + - uses: actions/upload-artifact@v4 + if: ${{ always() && !inputs.bootstrap }} + with: + name: standby-sync-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/standby-sync.json + if-no-files-found: ignore + retention-days: 14 + deploy: name: Replace the public Machines and verify readiness - needs: [qualify, preflight, migrate] + needs: [qualify, preflight, migrate, standby] if: ${{ github.ref == 'refs/heads/main' && !inputs.publish_only }} runs-on: ubuntu-latest timeout-minutes: 20 @@ -141,8 +223,12 @@ jobs: outputs: bootstrap_baseline: ${{ steps.verify.outputs.bootstrap_baseline }} env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + STANDBY_SYNC_ENABLED: ${{ vars.STANDBY_SYNC_ENABLED }} + STANDBY_RELEASE_SSH_KEY: ${{ secrets.STANDBY_RELEASE_SSH_KEY }} + STANDBY_RELEASE_KNOWN_HOSTS: ${{ vars.STANDBY_RELEASE_KNOWN_HOSTS }} RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }} + STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }} + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} BASELINE: ${{ needs.preflight.outputs.baseline }} BOOTSTRAP: ${{ inputs.bootstrap }} steps: @@ -152,6 +238,9 @@ jobs: - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 with: version: "0.4.107" + - name: Require fresh exact standby identity and direct TLS readiness + if: ${{ !inputs.bootstrap }} + run: python3 scripts/ci/standby-release-client.py check "$RUNNER_TEMP/standby-before-fly.json" - name: Revalidate the recorded baseline and deploy the same digest run: | test -n "$FLY_API_TOKEN" @@ -201,6 +290,18 @@ jobs: if [[ "$BOOTSTRAP" = true ]]; then printf 'API-only bootstrap is ready. Qualify authenticated paths, transfer routing, and stop the old all-role API before approving production-cutover.\n' >> "$GITHUB_STEP_SUMMARY" fi + - name: Verify both origins before completing the fleet release + if: ${{ !inputs.bootstrap }} + run: | + python3 scripts/ci/standby-release-client.py complete "$RUNNER_TEMP/standby-complete.json" + printf 'Nuremberg passed exact ARM source/digest/config readback and independent direct TLS readiness before and after Fly replacement.\n' >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: standby-final-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/standby-*.json + if-no-files-found: ignore + retention-days: 14 activate: name: Enable workers on the same qualified bootstrap image diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 5baa062e3..4efe7ca47 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -80,7 +80,12 @@ jobs: with: bun-version: "1.4.0" - name: Check server container and Fly configuration contracts - run: bun test scripts/docker/server-context.test.ts scripts/docker/prune-workspace.test.ts scripts/ci/fly-release-machines.test.ts + run: | + bun test scripts/docker/server-context.test.ts scripts/docker/prune-workspace.test.ts scripts/ci/fly-release-machines.test.ts + python3 scripts/ci/standby-release-test.py + bash -n scripts/ci/standby-release-export.sh + sh -n scripts/ci/standby-release-entrypoint.sh + sh -n scripts/ci/standby-release-controller-entrypoint.sh - name: Export committed server build context run: bun --no-env-file scripts/docker/server-context.ts "$GITHUB_SHA" "$RUNNER_TEMP/server-context" - name: Build runtime and Coolify targets @@ -192,6 +197,26 @@ jobs: docker run --rm inline-server:ci-coolify wget --version docker run --rm --network host -e DATABASE_URL inline-server:ci-coolify bun server/dist/core-production-smoke.js --artifact + - name: Qualify the daemonless ARM archive and bounded recovery reader + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq skopeo + archive="$RUNNER_TEMP/standby-transport-test.tar" + registry=inline-standby-ci-registry + trap 'docker stop "$registry" >/dev/null 2>&1 || true; rm -f "$archive"' EXIT + docker run --detach --rm --name "$registry" --publish 127.0.0.1:55000:5000 registry:2 >/dev/null + for attempt in {1..20}; do + if curl --silent --fail http://127.0.0.1:55000/v2/ >/dev/null; then break; fi + sleep 1 + done + image=127.0.0.1:55000/inline-api:transport-test + docker tag inline-server:ci-coolify "$image" + docker push "$image" >/dev/null + immutable=$(docker image inspect --format '{{json .RepoDigests}}' "$image" | jq -er '.[] | select(startswith("127.0.0.1:55000/inline-api@sha256:"))') + skopeo copy --src-tls-verify=false --preserve-digests "docker://$immutable" "oci-archive:$archive:release" >/dev/null + digest="${immutable##*@}" + python3 scripts/ci/standby-release-test.py --oci-archive "$archive" "$GITHUB_SHA" "$digest" + - name: Setup Fly registry client if: ${{ inputs.publish_image }} uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 diff --git a/scripts/ci/fly-release-machines.test.ts b/scripts/ci/fly-release-machines.test.ts index 1c52b0c8d..425838274 100644 --- a/scripts/ci/fly-release-machines.test.ts +++ b/scripts/ci/fly-release-machines.test.ts @@ -105,7 +105,7 @@ type ReleaseJob = { environment?: string uses?: string with?: Record - steps?: { name?: string; run?: string; env?: Record }[] + steps?: { name?: string; run?: string; if?: string; env?: Record }[] } const releaseWorkflow = async () => Bun.YAML.parse(await Bun.file( resolve(import.meta.dir, "../../.github/workflows/server-deploy.yml"), @@ -114,6 +114,32 @@ const releaseWorkflow = async () => Bun.YAML.parse(await Bun.file( jobs: Record } +test("ordinary releases synchronize and re-read Nuremberg before Fly without broad credentials", async () => { + const workflow = await releaseWorkflow() + expect(workflow.jobs.migrate?.needs).toContain("standby-preflight") + expect(workflow.jobs.standby?.needs).toEqual(["qualify", "preflight", "migrate"]) + expect(workflow.jobs.deploy?.needs).toContain("standby") + for (const id of ["standby-preflight", "standby"]) { + expect(workflow.jobs[id]?.if).toBe("${{ github.ref == 'refs/heads/main' && !inputs.publish_only }}") + expect(workflow.jobs[id]?.environment).toBe("production") + expect(JSON.stringify(workflow.jobs[id])).not.toMatch(/COOLIFY_TOKEN|CLOUDFLARE_TOKEN|ROOT_SSH_KEY/) + } + const standbySteps = workflow.jobs.standby?.steps ?? [] + const alternate = standbySteps.findIndex((step) => step.name === "Keep a healthy unchanged Fly alternate while replacing Nuremberg") + const sync = standbySteps.findIndex((step) => step.run?.includes("standby-release-client.py sync")) + expect(alternate).toBe(sync - 1) + expect(standbySteps[alternate]?.run).toContain('test "$current" = "$BASELINE"') + expect(standbySteps[sync]?.if).toBe("${{ !inputs.bootstrap }}") + const steps = workflow.jobs.deploy?.steps ?? [] + const check = steps.findIndex((step) => step.run?.includes("standby-release-client.py check")) + const deploy = steps.findIndex((step) => step.run?.includes("flyctl deploy")) + const flyReadback = steps.findIndex((step) => step.run?.includes("fly-release-wait.sh")) + const complete = steps.findIndex((step) => step.run?.includes("standby-release-client.py complete")) + expect(check).toBe(deploy - 1) + expect(complete).toBeGreaterThan(flyReadback) + for (const step of [steps[check], steps[complete]]) expect(step?.if).toBe("${{ !inputs.bootstrap }}") +}) + test("publish-only keeps full CI and excludes every production job; normal and failed releases retain their gates", async () => { const workflow = await releaseWorkflow() expect(workflow.on.workflow_dispatch.inputs.publish_only).toMatchObject({ type: "boolean", default: false }) @@ -148,7 +174,7 @@ test("publish-only keeps full CI and excludes every production job; normal and f } for (const [publishOnly, expected] of [ [true, ["validate", "qualify", "prepared"]], - [false, ["validate", "qualify", "preflight", "migrate", "deploy"]], + [false, ["validate", "qualify", "preflight", "standby-preflight", "migrate", "standby", "deploy"]], ] as const) { expect(Object.keys(workflow.jobs).filter((id) => runnable(id, publishOnly)).sort()).toEqual([...expected].sort()) for (const [id, job] of Object.entries(workflow.jobs)) { @@ -163,10 +189,12 @@ test("publish-only keeps full CI and excludes every production job; normal and f expect(runnable("qualify", true, "validate")).toBe(false) expect(runnable("migrate", false, "preflight")).toBe(false) expect(runnable("deploy", false, "migrate")).toBe(false) + expect(runnable("migrate", false, "standby-preflight")).toBe(false) + expect(runnable("deploy", false, "standby")).toBe(false) expect(JSON.stringify(workflow.jobs.prepared)).not.toContain("secrets.") expect(workflow.jobs.prepared?.environment).toBeUndefined() expect(Object.keys(workflow.jobs).filter((job) => runnable(job, false, undefined, true)).sort()) - .toEqual(["activate", "deploy", "migrate", "preflight", "qualify", "validate"]) + .toEqual(["activate", "deploy", "migrate", "preflight", "qualify", "standby", "standby-preflight", "validate"]) expect(runnable("activate", false, "deploy", true)).toBe(false) for (const ref of ["refs/heads/main", "refs/heads/codex/multi-node-ha", "refs/tags/main"]) { for (const publishOnly of [true, false]) { diff --git a/scripts/ci/standby-release-client.py b/scripts/ci/standby-release-client.py new file mode 100644 index 000000000..7afbea1b1 --- /dev/null +++ b/scripts/ci/standby-release-client.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Manual main-release runner transport; no provider mutations outside forced SSH.""" +import hashlib +import importlib.util +import json +import os +import signal +import resource +from pathlib import Path +import subprocess +import sys +import tempfile +import time + +spec = importlib.util.spec_from_file_location("standby_host", Path(__file__).with_name("standby-release-host.py")) +host = importlib.util.module_from_spec(spec) +spec.loader.exec_module(host) + + +def request(action, archive=None): + host.need(os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch" and + os.environ.get("GITHUB_REPOSITORY") == "inline-chat/inline" and + os.environ.get("GITHUB_REF") == "refs/heads/main" and + os.environ.get("STANDBY_SYNC_ENABLED") == "true", "release_not_enabled") + value = {"version": 1, "action": action, "revision": os.environ["GITHUB_SHA"], + "runtimeImage": os.environ["RUNTIME_IMAGE"], "standbyImage": os.environ["STANDBY_IMAGE"], + "runId": int(os.environ["GITHUB_RUN_ID"]), "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), + "issuedAt": int(time.time())} + if action == "sync": + path = Path(archive) + h = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + value.update(archiveBytes=path.stat().st_size, archiveSha256=h.hexdigest()) + host.validate_archive(path, host.validate_request(value)) + return host.validate_request(value) + + +def validate_receipt(value, selected, now=None): + host.need(set(value) == {"version", "ok", "action", "resource", "server", "release", "phase", + "checkedAt", "container", "directTlsReady"}, "receipt_fields") + phase = {"preflight": "prepared", "sync": "verified", "check": "verified", "complete": "complete"}[selected["action"]] + # A completed exact run can be safely checked/re-read without dispatching again. + phases = (phase, "complete") if selected["action"] in ("sync", "check") else (phase,) + host.need(value["version"] == 1 and value["ok"] is True and value["action"] == selected["action"] + and value["resource"] == host.APP and value["server"] == host.SERVER + and value["release"] == host.identity(selected) and value["phase"] in phases + and value["directTlsReady"] is True, "receipt_identity") + host.need(type(value["checkedAt"]) is int and -5 <= (now or time.time()) - value["checkedAt"] <= 60, + "receipt_stale") + host.need(isinstance(value["container"], str) and host.re.fullmatch(host.APP + r"-[0-9]+", value["container"]), + "receipt_container") + + +def main(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(host.Refused("client_deadline"))) + signal.alarm(840) + host.need(len(sys.argv) in (3, 4), "arguments") + action, destination = sys.argv[1:3] + archive = sys.argv[3] if len(sys.argv) == 4 else None + host.need((action == "sync") == (archive is not None), "archive_arguments") + selected = request(action, archive) + # Never persist bearer credentials. This key authorizes only the fixed host + # consumer and exists solely on the ephemeral protected production runner. + with tempfile.TemporaryDirectory(prefix="standby-ssh-", dir=os.environ["RUNNER_TEMP"]) as folder: + os.chmod(folder, 0o700) + key = Path(folder) / "key" + key.write_text(os.environ["STANDBY_RELEASE_SSH_KEY"] + "\n") + key.chmod(0o600) + known = Path(folder) / "known_hosts" + known.write_text(os.environ["STANDBY_RELEASE_KNOWN_HOSTS"] + "\n") + known.chmod(0o600) + # Do not inherit provider credentials into the SSH process. + environment = {"PATH": os.environ["PATH"], "HOME": folder, "LANG": "C.UTF-8"} + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + process = subprocess.Popen(["ssh", "-T", "-F", "/dev/null", "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=10", + "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=2", "-o", "LogLevel=ERROR", + "-o", "UserKnownHostsFile=" + str(known), "-i", str(key), "root@91.107.255.57", + "inline-standby-release-v1"], env=environment, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr) + try: + process.stdin.write(host.encoded(selected) + b"\n") + if archive: + with open(archive, "rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + process.stdin.write(chunk) + process.stdin.close() + process.wait(timeout=810) + host.need(process.returncode == 0, "remote_operation_failed_reconcile") + stdout.seek(0) + data = stdout.read(16385) + host.need(len(data) <= 16384, "receipt_size") + receipt = json.loads(data) + validate_receipt(receipt, selected) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + # A second vantage checks the actual NBG TLS origin, never the canonical LB. + result = subprocess.run(["curl", "--silent", "--fail", "--noproxy", "*", "--max-time", "15", "--max-filesize", "65536", + "--resolve", "api.inline.chat:443:91.107.255.57", "https://api.inline.chat/readyz"], + capture_output=True, timeout=20) + host.need(result.returncode == 0, "independent_origin_tls") + value = json.loads(result.stdout) + host.need(value.get("status") == "ok" and all(value.get("checks", {}).get(key, {}).get("ok") is True + for key in ("database", "clock", "broker")), "independent_origin_readiness") + Path(destination).write_bytes(host.encoded(receipt) + b"\n") + print(json.dumps({"ok": True, "action": action, "runId": selected["runId"], "revision": selected["revision"]})) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + code = str(error) if isinstance(error, host.Refused) else "private_operation_failed" + print(json.dumps({"ok": False, "code": code, "reconcileBeforeRetry": True})) + sys.exit(1) diff --git a/scripts/ci/standby-release-controller-entrypoint.sh b/scripts/ci/standby-release-controller-entrypoint.sh new file mode 100644 index 000000000..a54995f79 --- /dev/null +++ b/scripts/ci/standby-release-controller-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Finland only; root-owned. authorized_keys: from="91.107.255.57",restrict, +# command="/usr/local/libexec/inline-standby-controller-entrypoint" +set -eu +test "$#" = 0 +test "${SSH_ORIGINAL_COMMAND:-}" = inline-standby-controller-v1 +test -z "${SSH_TTY:-}" +exec /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/systemd-run --quiet --pipe --wait --collect \ + --property=Type=exec --property=RuntimeMaxSec=70 --property=TimeoutStopSec=5 \ + --property=KillMode=control-group --property=UMask=0077 --property=LimitCORE=0 \ + --property=MemoryMax=64M --property=MemorySwapMax=0 --property=CPUQuota=20% \ + --property=TasksMax=8 --property=NoNewPrivileges=yes \ + --property=LoadCredentialEncrypted=coolify:/etc/credstore.encrypted/inline-standby-coolify \ + --setenv=SSH_ORIGINAL_COMMAND=inline-standby-controller-v1 \ + /usr/bin/python3 -I /usr/local/libexec/inline-standby-controller --execute diff --git a/scripts/ci/standby-release-controller.py b/scripts/ci/standby-release-controller.py new file mode 100644 index 000000000..39c44566c --- /dev/null +++ b/scripts/ci/standby-release-controller.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Existing Finland controller only: app-scoped SSH facade, bearer never leaves. + +Install only after root reviews the existing token's scope/provenance. A narrow +facade does not make a broad bearer safe to move onto another origin host. +""" +import json +import os +from pathlib import Path +import re +import signal +import sys +import urllib.request + +APP = "m1wmnjpqe0l2csuofcxsnzbf" +SERVER = "scpfllw5tavzzp2ezrufcwff" +ORIGIN = "https://supermanage.inline.chat/api/v1" +TAG = re.compile(r"release-[a-f0-9]{40}-[1-9][0-9]{0,15}-[1-9][0-9]{0,15}\Z") + + +class Refused(Exception): + pass + + +def need(value, code): + if not value: + raise Refused(code) + + +def validate(value): + need(value.get("version") == 1 and value.get("operation") in ("read", "select", "start"), "request") + fields = {"version", "operation"} if value["operation"] == "read" else {"version", "operation", "tag"} + need(set(value) == fields, "fields") + if "tag" in fields: + need(isinstance(value["tag"], str) and TAG.fullmatch(value["tag"]), "tag") + return value + + +def project(app): + settings = app.get("settings", {}) + server = app.get("server", {}) + need(app.get("uuid") == APP and server.get("uuid") == SERVER and server.get("ip") == "91.107.255.57" + and app.get("build_pack") == "dockerimage" and app.get("fqdn") == "https://api.inline.chat" + and app.get("docker_registry_image_name") == "127.0.0.1:5000/inline-api" and not app.get("ports_mappings") + and settings.get("is_auto_deploy_enabled") is False and settings.get("is_preview_deployments_enabled") is False, + "resource_identity_or_intent") + return {"uuid": APP, "server": {"uuid": SERVER, "ip": server["ip"]}, + **{k: app.get(k) for k in ("build_pack", "fqdn", "docker_registry_image_name", "docker_registry_image_tag", "ports_mappings")}, + "settings": {k: settings[k] for k in ("is_auto_deploy_enabled", "is_preview_deployments_enabled")}} + + +class Controller: + def __init__(self, token): + self.token = token + + def api(self, method, suffix, body=None): + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + request = urllib.request.Request(ORIGIN + suffix, method=method, + data=None if body is None else json.dumps(body).encode(), + headers={"Authorization": "Bearer " + self.token, "Content-Type": "application/json"}) + with urllib.request.build_opener(NoRedirect).open(request, timeout=15) as response: + data = response.read(2 * 1024 * 1024 + 1) + need(len(data) <= 2 * 1024 * 1024, "provider_size") + return json.loads(data) + + +def perform(value, controller): + value = validate(value) + path = "/applications/" + APP + before = project(controller.api("GET", path)) + if value["operation"] == "select": + if before["docker_registry_image_tag"] != value["tag"]: + controller.api("PATCH", path, {"docker_registry_image_tag": value["tag"]}) + before = project(controller.api("GET", path)) + need(before["docker_registry_image_tag"] == value["tag"], "select_readback") + elif value["operation"] == "start": + need(before["docker_registry_image_tag"] == value["tag"], "start_tag_changed") + controller.api("POST", path + "/start") + return {"ok": True, "app": before} + + +def main(): + need(os.geteuid() == 0 and sys.argv[1:] == ["--execute"] and + os.environ.get("SSH_ORIGINAL_COMMAND") == "inline-standby-controller-v1" and not os.environ.get("SSH_TTY"), "entrypoint") + signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(Refused("deadline"))) + signal.alarm(65) + line = sys.stdin.buffer.readline(4097) + need(len(line) <= 4096 and line.endswith(b"\n") and sys.stdin.buffer.read(1) == b"", "input") + value = validate(json.loads(line)) + directory = os.environ.get("CREDENTIALS_DIRECTORY", "") + need(directory.startswith("/run/credentials/"), "credential_namespace") + token = (Path(directory) / "coolify").read_text().strip() + need(bool(token) and "\n" not in token, "credential_format") + return perform(value, Controller(token)) + + +if __name__ == "__main__": + try: + print(json.dumps(main())) + except Exception as error: + code = str(error) if isinstance(error, Refused) else "private_operation_failed" + print(json.dumps({"ok": False, "code": code})) + sys.exit(1) diff --git a/scripts/ci/standby-release-entrypoint.sh b/scripts/ci/standby-release-entrypoint.sh new file mode 100644 index 000000000..c40119405 --- /dev/null +++ b/scripts/ci/standby-release-entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Root-owned installed path; the dedicated authorized_keys entry uses `restrict` +# and this command only. Never authorize a personal/root shell key in Actions. +set -eu +test "$#" = 0 +test "${SSH_ORIGINAL_COMMAND:-}" = inline-standby-release-v1 +test -z "${SSH_TTY:-}" +exec /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin \ + /usr/bin/systemd-run --quiet --pipe --wait --collect \ + --property=Type=exec --property=RuntimeMaxSec=800 --property=TimeoutStopSec=15 \ + --property=KillMode=control-group --property=UMask=0077 --property=LimitCORE=0 \ + --property=MemoryMax=256M --property=MemorySwapMax=0 --property=CPUQuota=50% \ + --property=TasksMax=32 --property=NoNewPrivileges=yes \ + --setenv=SSH_ORIGINAL_COMMAND=inline-standby-release-v1 \ + /usr/bin/python3 -I /usr/local/libexec/inline-standby-release --execute diff --git a/scripts/ci/standby-release-export.sh b/scripts/ci/standby-release-export.sh new file mode 100644 index 000000000..8b9ddbbd6 --- /dev/null +++ b/scripts/ci/standby-release-export.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Ephemeral runner only. Export the exact already-qualified ARM manifest; no build. +set -euo pipefail +umask 077 +test "${STANDBY_SYNC_ENABLED:-}" = true +[[ "$STANDBY_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]] +test -n "$FLY_API_TOKEN" +archive=$1 +authdir=$(mktemp -d "$RUNNER_TEMP/standby-registry-XXXXXX") +trap 'rm -rf "$authdir"' EXIT +# Auth file is private to this ephemeral runner and never enters the image stream. +printf '%s' "$FLY_API_TOKEN" | skopeo login --authfile "$authdir/auth.json" --username x --password-stdin registry.fly.io >/dev/null 2>&1 +timeout --signal=KILL 180s skopeo copy --authfile "$authdir/auth.json" --preserve-digests \ + "docker://$STANDBY_IMAGE" "oci-archive:$archive:release" >/dev/null 2>&1 diff --git a/scripts/ci/standby-release-host.py b/scripts/ci/standby-release-host.py new file mode 100644 index 000000000..33d1f00fd --- /dev/null +++ b/scripts/ci/standby-release-host.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Fixed-resource forced SSH consumer. Installation/credentials are separate ops. + +Wire format: one bounded JSON line, followed by exactly archiveBytes OCI tar bytes +for sync only. No caller command, path, host, token, config or env is accepted. +""" +import fcntl +import gzip +import hashlib +import http.client +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.parse + +APP = "m1wmnjpqe0l2csuofcxsnzbf" +SERVER = "scpfllw5tavzzp2ezrufcwff" +REGISTRY = "127.0.0.1:5000/inline-api" +STATE = Path("/var/lib/inline-standby-release") +MAX_ARCHIVE = 768 * 1024 * 1024 +MAX_ROOTFS = 2 * 1024**3 +DIGEST = re.compile(r"sha256:[a-f0-9]{64}\Z") +SHA = re.compile(r"[a-f0-9]{40}\Z") +MEDIA = ("application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json") + + +class Refused(Exception): + pass + + +def need(condition, code): + if not condition: + raise Refused(code) + + +def encoded(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def validate_request(value, now=None): + fields = {"version", "action", "revision", "runtimeImage", "standbyImage", "runId", "attempt", "issuedAt"} + if value.get("action") == "sync": + fields |= {"archiveBytes", "archiveSha256"} + need(set(value) == fields and value["version"] == 1, "request_fields") + need(value["action"] in ("preflight", "sync", "check", "complete"), "request_action") + need(isinstance(value["revision"], str) and SHA.fullmatch(value["revision"]), "request_revision") + for key in ("runtimeImage", "standbyImage"): + need(isinstance(value[key], str) and value[key].startswith("registry.fly.io/inline-api@") + and DIGEST.fullmatch(value[key].split("@")[-1]), "request_image") + need(value["runtimeImage"] != value["standbyImage"], "architecture_pair") + for key in ("runId", "attempt"): + need(type(value[key]) is int and 0 < value[key] < 10**16, "request_run") + need(type(value["issuedAt"]) is int and -30 <= (now or time.time()) - value["issuedAt"] <= 900, + "request_expired") + if value["action"] == "sync": + need(type(value["archiveBytes"]) is int and 0 < value["archiveBytes"] <= MAX_ARCHIVE, "archive_size") + need(isinstance(value["archiveSha256"], str) and re.fullmatch(r"[a-f0-9]{64}", value["archiveSha256"]), "archive_hash") + return value + + +def identity(request): + return {key: request[key] for key in ("revision", "runtimeImage", "standbyImage", "runId", "attempt")} + + +def tag(request): + return f"release-{request['revision']}-{request['runId']}-{request['attempt']}" + + +def env_fingerprint(env): + # Stays in root-only local state, never emitted. Values never leave memory. + stable = {k: v for k, v in env.items() if not k.startswith("COOLIFY_") and k not in ("SOURCE_COMMIT", "HOSTNAME")} + return hashlib.sha256(encoded(stable)).hexdigest() + + +def verify_rootfs(archive, files, manifest, config): + rootfs = config.get("rootfs", {}) + need(rootfs.get("type") == "layers" and isinstance(rootfs.get("diff_ids"), list) + and len(rootfs["diff_ids"]) == len(manifest["layers"]), "rootfs_config") + decoded_total, filesystem_total, entries = 0, 0, 0 + for layer, expected in zip(manifest["layers"], rootfs["diff_ids"]): + need(isinstance(expected, str) and DIGEST.fullmatch(expected), "rootfs_digest") + media = layer.get("mediaType", "") + need(media in ("application/vnd.oci.image.layer.v1.tar+gzip", "application/vnd.docker.image.rootfs.diff.tar.gzip", + "application/vnd.oci.image.layer.v1.tar", "application/vnd.docker.image.rootfs.diff.tar"), "rootfs_compression") + source = archive.extractfile(files["blobs/sha256/" + layer["digest"][7:]]) + decoded = gzip.GzipFile(fileobj=source) if media.endswith(("+gzip", ".gzip")) else source + digest = hashlib.sha256() + class BoundedReader: + def read(self, size=-1): + nonlocal decoded_total + # Refuse giant PAX/sparse metadata allocation requests before read. + need(0 <= size <= 2 * 1024 * 1024, "rootfs_metadata_size") + data = decoded.read(size) + decoded_total += len(data) + need(decoded_total <= MAX_ROOTFS, "rootfs_decoded_limit") + digest.update(data) + return data + reader = BoundedReader() + with tarfile.open(fileobj=reader, mode="r|") as layer_tar: + for member in layer_tar: + entries += 1 + filesystem_total += member.size if member.isfile() else 0 + # Includes sparse files' declared logical size; compressed byte + # and diff-ID checks alone cannot bound their disk allocation. + need(entries <= 200000 and filesystem_total <= MAX_ROOTFS, "rootfs_filesystem_limit") + while reader.read(1024 * 1024): + pass + need("sha256:" + digest.hexdigest() == expected, "rootfs_diff_id") + decoded.close() + return max(decoded_total, filesystem_total) + + +def validate_archive(path, request): + """Verify tar in place, never extract caller paths. Hash every referenced blob.""" + with tarfile.open(path, "r:") as archive: + members = archive.getmembers() + need(len(members) < 2048, "archive_entries") + files = {} + for member in members: + name = member.name.removeprefix("./") + need(member.isfile() or member.isdir(), "archive_links") + need(name in (".", "oci-layout", "index.json", "blobs", "blobs/sha256") + or re.fullmatch(r"blobs/sha256/[a-f0-9]{64}", name), "archive_path") + if member.isfile(): + need(name not in files, "archive_duplicate") + files[name] = member + + def read_json(name): + need(name in files and files[name].size <= 2 * 1024 * 1024, "archive_metadata") + return json.load(archive.extractfile(files[name])) + + need(read_json("oci-layout") == {"imageLayoutVersion": "1.0.0"}, "oci_layout") + index = read_json("index.json") + need(index.get("schemaVersion") == 2 and len(index.get("manifests", [])) == 1, "oci_index") + descriptor = index["manifests"][0] + digest = request["standbyImage"].split("@")[1] + need(descriptor.get("digest") == digest and descriptor.get("mediaType") in MEDIA, "manifest_identity") + manifest = read_json("blobs/sha256/" + digest[7:]) + need(manifest.get("schemaVersion") == 2 and manifest.get("mediaType") == descriptor["mediaType"], "manifest_type") + need(isinstance(manifest.get("layers"), list) and 0 < len(manifest["layers"]) < 128, "manifest_layers") + descriptors = [descriptor, manifest["config"], *manifest["layers"]] + expected = {"oci-layout", "index.json"} + for item in descriptors: + blob = item.get("digest", "") + need(DIGEST.fullmatch(blob) and type(item.get("size")) is int and item["size"] > 0, "blob_descriptor") + name = "blobs/sha256/" + blob[7:] + need(name in files and files[name].size == item["size"], "blob_size") + expected.add(name) + stream = archive.extractfile(files[name]) + h = hashlib.sha256() + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + need("sha256:" + h.hexdigest() == blob, "blob_hash") + need(set(files) == expected, "archive_extra_blob") + config = read_json("blobs/sha256/" + manifest["config"]["digest"][7:]) + need(config.get("architecture") == "arm64" and config.get("os") == "linux", "image_platform") + need(config.get("config", {}).get("Labels", {}).get("org.opencontainers.image.revision") == request["revision"], + "image_revision") + return manifest, descriptors, verify_rootfs(archive, files, manifest, config) + + +def command(args, timeout=20, stdin=None): + result = subprocess.run(args, input=stdin, capture_output=True, timeout=timeout, check=False) + need(result.returncode == 0, "native_command") + return result.stdout + + +class Host: + def api(self, method, suffix, body=None): + if method == "GET" and suffix == "/applications/" + APP: + payload = {"version": 1, "operation": "read"} + elif method == "PATCH" and suffix == "/applications/" + APP and set(body or {}) == {"docker_registry_image_tag"}: + payload = {"version": 1, "operation": "select", "tag": body["docker_registry_image_tag"]} + elif method == "POST" and suffix == "/applications/" + APP + "/start" and set(body or {}) == {"tag"}: + payload = {"version": 1, "operation": "start", "tag": body["tag"]} + else: + raise Refused("controller_operation") + # Only an app-scoped forced-command relay key lives on NBG. The existing + # controller bearer stays on Finland; no bearer is copied to this host. + data = command(["ssh", "-4", "-T", "-F", "/dev/null", "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=5", + "-o", "LogLevel=ERROR", "-o", "UserKnownHostsFile=/etc/inline-standby-release/controller-known-hosts", + "-i", "/etc/inline-standby-release/controller-key", "-p", "6642", "root@95.216.163.150", + "inline-standby-controller-v1"], timeout=45, stdin=encoded(payload) + b"\n") + need(len(data) <= 65536, "controller_response") + value = json.loads(data) + need(value.get("ok") is True and set(value) == {"ok", "app"}, "controller_response") + return value["app"] + + def app(self): + app = self.api("GET", "/applications/" + APP) + server = app.get("server", {}) + settings = app.get("settings", {}) + need(app.get("uuid") == APP and server.get("uuid") == SERVER and server.get("ip") == "91.107.255.57", + "resource_identity") + need(app.get("build_pack") == "dockerimage" and app.get("fqdn") == "https://api.inline.chat" + and app.get("docker_registry_image_name") == REGISTRY, "resource_intent") + need(not app.get("ports_mappings") and settings.get("is_auto_deploy_enabled") is False + and settings.get("is_preview_deployments_enabled") is False, "resource_automation") + return app + + def runtime(self, request=None): + ids = command(["docker", "ps", "--filter", "name=" + APP + "-", "--format", "{{.ID}} co"]).decode().splitlines() + need(len(ids) == 1 and re.fullmatch(r"[a-f0-9]{12,64} co", ids[0]), "runtime_count") + container = json.loads(command(["docker", "inspect", ids[0].split()[0]]))[0] + image = json.loads(command(["docker", "image", "inspect", container["Image"]]))[0] + name = container["Name"].lstrip("/") + need(re.fullmatch(APP + r"-[0-9]+", name), "runtime_resource") + need(container["State"].get("Running") is True and container["State"].get("Health", {}).get("Status") == "healthy" + and not container["State"].get("OOMKilled"), "runtime_health") + config = container["HostConfig"] + need(not config.get("PortBindings") and config.get("Memory") == 1610612736 + and config.get("MemorySwap") == 1610612736 and config.get("NanoCpus") == 2000000000, + "runtime_bounds") + env = dict(item.split("=", 1) for item in container["Config"]["Env"] if "=" in item) + expected = {"NODE_ENV": "production", "INLINE_PROCESS_ROLE": "all", "INLINE_REALTIME_DISTRIBUTED": "1", + "INLINE_DATABASE_QUERY_POOL_MAX": "3", "INLINE_INGRESS_MODE": "cloudflare-optional", + "INLINE_INGRESS_HOST": "api.inline.chat", "INLINE_INGRESS_DIRECT_IP_SOURCE": "x-real-ip", + "ENABLE_DATABASE_HEALTH_MONITOR": "0"} + need(all(env.get(k) == v for k, v in expected.items()) and bool(env.get("REDIS_URL")), "runtime_intent") + redis = urllib.parse.urlsplit(env["REDIS_URL"]) + need(redis.scheme == "redis" and redis.hostname == "172.18.0.1" and redis.port == 16379 + and bool(redis.password), "private_broker_endpoint") + revision = image.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.revision") + need(image.get("Architecture") == "arm64" and image.get("Os") == "linux" and SHA.fullmatch(revision or ""), "runtime_image") + if request: + need(revision == request["revision"] and container["Config"]["Image"] == REGISTRY + ":" + tag(request) + and REGISTRY + "@" + request["standbyImage"].split("@")[1] in image.get("RepoDigests", []), + "runtime_release") + return {"container": name, "imageId": container["Image"], "revision": revision, + "environmentHash": env_fingerprint(env)} + + def ready(self): + # Canonical hostname/SNI and public trust, forced to the local origin. + data = command(["curl", "--silent", "--fail", "--noproxy", "*", "--max-time", "10", "--max-filesize", "65536", + "--resolve", "api.inline.chat:443:127.0.0.1", "https://api.inline.chat/readyz"], timeout=12) + value = json.loads(data) + need(value.get("status") == "ok" and all(value.get("checks", {}).get(key, {}).get("ok") is True + for key in ("database", "clock", "broker")), + "origin_readiness") + + def alternate(self): + # Independent final guard after archive transfer/pull, immediately before + # each controller mutation. Never probe the LB, which could return NBG. + data = command(["curl", "--silent", "--fail", "--noproxy", "*", "--max-time", "5", "--max-filesize", "65536", + "--resolve", "api.inline.chat:443:66.241.124.189", "https://api.inline.chat/readyz"], timeout=7) + value = json.loads(data) + need(value.get("status") == "ok" and all(value.get("checks", {}).get(key, {}).get("ok") is True + for key in ("database", "clock", "broker")), "alternate_origin_unready") + + def registry(self, method, path, body=None, headers=None, stream=None, size=0): + connection = http.client.HTTPConnection("127.0.0.1", 5000, timeout=30) + try: + connection.putrequest(method, path) + for key, value in (headers or {}).items(): + connection.putheader(key, value) + connection.putheader("Content-Length", str(size if stream else len(body or b""))) + connection.endheaders() + if stream: + while chunk := stream.read(1024 * 1024): + connection.send(chunk) + elif body: + connection.send(body) + response = connection.getresponse() + result = (response.status, dict(response.getheaders()), response.read(2 * 1024 * 1024 + 1)) + need(len(result[2]) <= 2 * 1024 * 1024, "registry_response") + return result + finally: + connection.close() + + def mirror(self, path, request, manifest, descriptors): + reference = tag(request) + status, _, raw = self.registry("GET", "/v2/inline-api/manifests/" + reference, + headers={"Accept": ",".join(MEDIA)}) + digest = request["standbyImage"].split("@")[1] + if status == 200: + need("sha256:" + hashlib.sha256(raw).hexdigest() == digest, "immutable_tag_conflict") + else: + need(status == 404, "registry_read") + with tarfile.open(path, "r:") as archive: + entries = {m.name.removeprefix("./"): m for m in archive.getmembers() if m.isfile()} + for item in descriptors[1:]: + blob = item["digest"] + status, _, _ = self.registry("HEAD", "/v2/inline-api/blobs/" + blob) + if status == 200: + continue + need(status == 404, "registry_blob_read") + status, headers, _ = self.registry("POST", "/v2/inline-api/blobs/uploads/") + need(status == 202, "registry_blob_start") + location = headers.get("Location", headers.get("location", "")) + parsed = urllib.parse.urlsplit(location) + need(parsed.scheme in ("", "http") and parsed.netloc in ("", "127.0.0.1:5000") + and parsed.path.startswith("/v2/inline-api/blobs/uploads/"), "registry_location") + query = parsed.query + ("&" if parsed.query else "") + urllib.parse.urlencode({"digest": blob}) + member = entries["blobs/sha256/" + blob[7:]] + status, _, _ = self.registry("PUT", parsed.path + "?" + query, + headers={"Content-Type": "application/octet-stream"}, stream=archive.extractfile(member), size=member.size) + need(status == 201, "registry_blob_write") + member = entries["blobs/sha256/" + digest[7:]] + status, _, _ = self.registry("PUT", "/v2/inline-api/manifests/" + reference, + body=archive.extractfile(member).read(), headers={"Content-Type": manifest["mediaType"]}) + need(status == 201, "registry_manifest_write") + status, _, raw = self.registry("GET", "/v2/inline-api/manifests/" + reference, headers={"Accept": ",".join(MEDIA)}) + need(status == 200 and "sha256:" + hashlib.sha256(raw).hexdigest() == digest, "registry_readback") + command(["docker", "pull", REGISTRY + "@" + digest], timeout=120) + + def headroom(self, rootfs_bytes=0, archive_bytes=0): + fields = dict(line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines()) + need(int(fields["MemAvailable"].split()[0]) * 1024 >= 2 * 1024**3, "memory_headroom") + # The incoming archive is already on disk. Reserve registry blobs, Docker + # compressed cache, the full decoded filesystem, and 1 GiB untouched. + required = max(3 * 1024**3, 2 * archive_bytes + rootfs_bytes + 1024**3) + need(shutil.disk_usage(STATE).free >= required and shutil.disk_usage("/var/lib/docker").free >= required, + "disk_headroom") + + +def save(path, value): + temporary = path.with_suffix(".tmp") + with temporary.open("wb") as stream: + stream.write(encoded(value)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + fd = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def perform(request, host, journal, write, archive=None): + selected = identity(request) + action = request["action"] + same = journal is not None and journal["release"] == selected + if action == "preflight": + if journal and not same: + need(journal["phase"] == "complete" and + (request["runId"], request["attempt"]) > (journal["release"]["runId"], journal["release"]["attempt"]), + "previous_release_unresolved_or_newer") + if same: + need(journal["phase"] == "prepared", "preflight_already_consumed") + app = host.app() + current = host.runtime() + host.ready() + if same: + need(current == journal["before"] and app["docker_registry_image_tag"] == journal["previousTag"], "predecessor_changed") + else: + journal = {"phase": "prepared", "release": selected, "before": current, + "previousTag": app["docker_registry_image_tag"], "preparedAt": int(time.time())} + write(journal) + elif action == "sync": + need(same, "preflight_missing") + if journal["phase"] in ("verified", "complete"): + current = host.runtime(request) + host.ready() + need(host.app()["docker_registry_image_tag"] == tag(request) and current == journal["after"] + and current["environmentHash"] == journal["before"]["environmentHash"], "runtime_changed") + elif journal["phase"] == "dispatching": + # A late successful deploy may close a previous timeout, but an old, + # failed or partial deployment never authorizes another start request. + need(host.app()["docker_registry_image_tag"] == tag(request), "dispatch_requires_reconciliation") + current = host.runtime(request) + need(current["environmentHash"] == journal["before"]["environmentHash"], "runtime_environment_changed") + host.ready() + journal.update(phase="verified", after=current, verifiedAt=int(time.time())) + write(journal) + else: + need(journal["phase"] == "prepared" and time.time() - journal["preparedAt"] <= 1800, "sync_requires_reconciliation") + app = host.app() + need(host.runtime() == journal["before"] and app["docker_registry_image_tag"] == journal["previousTag"], "predecessor_changed") + host.ready() + host.headroom() + manifest, descriptors, rootfs_bytes = validate_archive(archive, request) + host.headroom(rootfs_bytes, request["archiveBytes"]) + host.mirror(archive, request, manifest, descriptors) + host.alternate() + journal.update(phase="patching", archiveSha256=request["archiveSha256"]) + write(journal) # Intent is durable BEFORE the first controller mutation. + host.api("PATCH", "/applications/" + APP, {"docker_registry_image_tag": tag(request)}) + need(host.app()["docker_registry_image_tag"] == tag(request), "patch_readback") + host.alternate() + journal["phase"] = "dispatching" + write(journal) + # A timeout or malformed response is ambiguous. Observe once, never redispatch. + try: + host.api("POST", "/applications/" + APP + "/start", {"tag": tag(request)}) + except Exception: + pass + deadline = time.monotonic() + 300 + current = None + while time.monotonic() < deadline: + try: + candidate = host.runtime(request) + need(candidate["environmentHash"] == journal["before"]["environmentHash"], "runtime_environment_changed") + host.ready() + current = candidate + break + except Exception: + time.sleep(5) + need(current is not None, "dispatch_unresolved") + journal.update(phase="verified", after=current, verifiedAt=int(time.time())) + write(journal) + else: + need(same and journal["phase"] in ("verified", "complete"), "release_not_verified") + need(host.app()["docker_registry_image_tag"] == tag(request), "resource_tag_changed") + current = host.runtime(request) + need(current == journal["after"] and current["environmentHash"] == journal["before"]["environmentHash"], "runtime_changed") + host.ready() + if action == "complete": + journal.update(phase="complete", completedAt=int(time.time())) + write(journal) + # Deliberate safe projection: never return controller payloads or environment hashes. + return {"version": 1, "ok": True, "action": action, "resource": APP, "server": SERVER, + "release": selected, "phase": journal["phase"], "checkedAt": int(time.time()), + "container": journal.get("after", journal["before"])["container"], "directTlsReady": True} + + +def main(): + need(os.geteuid() == 0 and sys.argv[1:] == ["--execute"] and + os.environ.get("SSH_ORIGINAL_COMMAND") == "inline-standby-release-v1" and not os.environ.get("SSH_TTY"), "entrypoint") + os.umask(0o077) + signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(Refused("deadline"))) + signal.alarm(780) + line = sys.stdin.buffer.readline(16385) + need(len(line) <= 16384 and line.endswith(b"\n"), "request_line") + request = validate_request(json.loads(line)) + STATE.mkdir(mode=0o700, parents=True, exist_ok=True) + with (STATE / "lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused("release_busy") + path = STATE / "journal.json" + journal = json.loads(path.read_bytes()) if path.exists() else None + with tempfile.TemporaryDirectory(prefix="incoming-", dir=STATE) as directory: + archive = Path(directory) / "image.tar" + if request["action"] == "sync": + Host().headroom() + remaining = request["archiveBytes"] + h = hashlib.sha256() + with archive.open("wb") as output: + while remaining: + data = sys.stdin.buffer.read(min(1024 * 1024, remaining)) + need(bool(data), "archive_truncated") + h.update(data) + output.write(data) + remaining -= len(data) + need(sys.stdin.buffer.read(1) == b"" and h.hexdigest() == request["archiveSha256"], "archive_transport") + else: + need(sys.stdin.buffer.read(1) == b"", "unexpected_payload") + return perform(request, Host(), journal, lambda value: save(path, value), archive) + + +if __name__ == "__main__": + try: + print(json.dumps(main())) + except Exception as error: + # No exception text from providers, subprocesses, credentials or JSON bodies. + code = str(error) if isinstance(error, Refused) else "private_operation_failed" + print(json.dumps({"version": 1, "ok": False, "code": code, "reconcileBeforeRetry": True})) + sys.exit(1) diff --git a/scripts/ci/standby-release-test.py b/scripts/ci/standby-release-test.py new file mode 100644 index 000000000..705d0cf9c --- /dev/null +++ b/scripts/ci/standby-release-test.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""Synthetic-only acceptance: no Docker, providers, SSH, credentials or env files.""" +import copy +import hashlib +import gzip +import importlib.util +import io +import json +import sys +from pathlib import Path +import tarfile +import tempfile +import time +import unittest +from unittest.mock import patch + + +def module(name, filename): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(filename)) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +host = module("standby_host_tests", "standby-release-host.py") +client = module("standby_client_tests", "standby-release-client.py") +controller = module("standby_controller_tests", "standby-release-controller.py") +REVISION = "1" * 40 + + +def request(action="preflight"): + return {"version": 1, "action": action, "revision": REVISION, + "runtimeImage": "registry.fly.io/inline-api@sha256:" + "2" * 64, + "standbyImage": "registry.fly.io/inline-api@sha256:" + "3" * 64, + "runId": 100, "attempt": 1, "issuedAt": int(time.time())} + + +def image_archive(directory, config=None, extra=None, change_layer=False): + value = request("sync") + config = config or {"architecture": "arm64", "os": "linux", "config": { + "Labels": {"org.opencontainers.image.revision": REVISION}}} + layer_buffer = io.BytesIO() + with tarfile.open(fileobj=layer_buffer, mode="w:") as layer_tar: + item = tarfile.TarInfo("synthetic.txt") + item.size = 17 + layer_tar.addfile(item, io.BytesIO(b"synthetic content")) + layer_tar_bytes = layer_buffer.getvalue() + config.setdefault("rootfs", {"type": "layers", "diff_ids": ["sha256:" + hashlib.sha256(layer_tar_bytes).hexdigest()]}) + config_data = host.encoded(config) + layer_data = gzip.compress(layer_tar_bytes, mtime=0) + def descriptor(data, media): + return {"mediaType": media, "size": len(data), "digest": "sha256:" + hashlib.sha256(data).hexdigest()} + conf = descriptor(config_data, "application/vnd.oci.image.config.v1+json") + layer = descriptor(layer_data, "application/vnd.oci.image.layer.v1.tar+gzip") + manifest_data = host.encoded({"schemaVersion": 2, "mediaType": host.MEDIA[0], "config": conf, "layers": [layer]}) + manifest = descriptor(manifest_data, host.MEDIA[0]) + entries = {"oci-layout": host.encoded({"imageLayoutVersion": "1.0.0"}), + "index.json": host.encoded({"schemaVersion": 2, "manifests": [manifest]}), + "blobs/sha256/" + manifest["digest"][7:]: manifest_data, + "blobs/sha256/" + conf["digest"][7:]: config_data, + "blobs/sha256/" + layer["digest"][7:]: layer_data if not change_layer else b"x" * len(layer_data)} + entries.update(extra or {}) + path = Path(directory) / "image.tar" + with tarfile.open(path, "w:") as archive: + for name, data in entries.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + value.update(standbyImage="registry.fly.io/inline-api@" + manifest["digest"], archiveBytes=path.stat().st_size, + archiveSha256=hashlib.sha256(path.read_bytes()).hexdigest()) + return path, value + + +class FakeHost: + def __init__(self): + self.tag = "previous-retained-release" + self.current = {"container": host.APP + "-123", "imageId": "old", "revision": "0" * 40, + "environmentHash": "secret-fingerprint-not-for-output"} + self.events = [] + self.desired = None + self.dispatch_timeout = False + self.patch_timeout = False + self.replace_error = False + self.unhealthy = False + self.no_headroom = False + self.alternate_unhealthy = False + + def app(self): + return {"docker_registry_image_tag": self.tag} + + def runtime(self, value=None): + if value: + host.need(self.current["revision"] == value["revision"], "runtime_release") + return copy.deepcopy(self.current) + + def ready(self): + host.need(not self.unhealthy, "origin_readiness") + + def headroom(self, *args): + host.need(not self.no_headroom, "memory_headroom") + + def alternate(self): + host.need(not self.alternate_unhealthy, "alternate_origin_unready") + + def mirror(self, path, value, manifest, descriptors): + self.events.append("mirror") + self.desired = value + + def api(self, method, suffix, body=None): + if method == "PATCH": + self.events.append("patch") + self.tag = body["docker_registry_image_tag"] + if self.patch_timeout: + raise TimeoutError("private-provider-body-token") + else: + assert method == "POST" and suffix == "/applications/" + host.APP + "/start" + self.events.append("dispatch") + if not self.replace_error: + self.current.update(container=host.APP + "-456", imageId="new", revision=self.desired["revision"]) + if self.dispatch_timeout: + raise TimeoutError("private-provider-body-token") + return {} + + +class TransactionTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.path, self.value = image_archive(self.directory.name) + self.runtime = FakeHost() + self.journal = None + + def write(self, value): + self.journal = copy.deepcopy(value) + self.runtime.events.append("journal:" + value["phase"]) + + def run_action(self, action): + value = dict(self.value, action=action) + if action != "sync": + del value["archiveBytes"], value["archiveSha256"] + return host.perform(value, self.runtime, self.journal, self.write, self.path) + + def test_normal_release_preserves_environment_and_binds_both_digests(self): + self.run_action("preflight") + receipt = self.run_action("sync") + self.assertEqual(self.runtime.events, ["journal:prepared", "mirror", "journal:patching", "patch", + "journal:dispatching", "dispatch", "journal:verified"]) + self.assertEqual(receipt["release"], host.identity(self.value)) + self.assertNotIn("secret-fingerprint", json.dumps(receipt)) + self.assertEqual(self.run_action("check")["phase"], "verified") + self.assertEqual(self.run_action("complete")["phase"], "complete") + + def test_dispatch_timeout_reconciles_without_duplicate_request(self): + self.runtime.dispatch_timeout = True + self.run_action("preflight") + self.run_action("sync") + self.run_action("sync") + self.assertEqual(self.runtime.events.count("dispatch"), 1) + + def test_uncertain_patch_cannot_be_repeated(self): + self.runtime.patch_timeout = True + self.run_action("preflight") + with self.assertRaises(TimeoutError): + self.run_action("sync") + self.assertEqual(self.journal["phase"], "patching") + with self.assertRaises(host.Refused): + self.run_action("sync") + self.assertEqual(self.runtime.events.count("patch"), 1) + self.assertNotIn("dispatch", self.runtime.events) + + def test_failed_dispatch_remains_fenced_and_does_not_rollback(self): + self.run_action("preflight") + self.runtime.replace_error = True + with patch.object(host.time, "monotonic", side_effect=[0, 301]): + with self.assertRaisesRegex(host.Refused, "dispatch_unresolved"): + self.run_action("sync") + with self.assertRaises(host.Refused): + self.run_action("sync") + self.assertEqual(self.runtime.events.count("dispatch"), 1) + self.assertEqual(self.journal["phase"], "dispatching") + self.assertEqual(self.runtime.current["imageId"], "old") + + def test_late_successful_dispatch_can_close_without_redispatch(self): + self.run_action("preflight") + self.runtime.replace_error = True + with patch.object(host.time, "monotonic", side_effect=[0, 301]): + with self.assertRaises(host.Refused): + self.run_action("sync") + self.runtime.current.update(container=host.APP + "-456", imageId="new", revision=self.value["revision"]) + self.assertEqual(self.run_action("sync")["phase"], "verified") + self.assertEqual(self.runtime.events.count("dispatch"), 1) + + def test_missing_or_changed_predecessor_refused_before_mutation(self): + with self.assertRaisesRegex(host.Refused, "preflight_missing"): + self.run_action("sync") + self.run_action("preflight") + self.runtime.current["imageId"] = "other" + with self.assertRaisesRegex(host.Refused, "predecessor_changed"): + self.run_action("sync") + self.assertNotIn("mirror", self.runtime.events) + + def test_unhealthy_or_no_headroom_refused_before_update(self): + self.run_action("preflight") + for field in ("unhealthy", "no_headroom"): + setattr(self.runtime, field, True) + with self.assertRaises(host.Refused): + self.run_action("sync") + setattr(self.runtime, field, False) + self.assertNotIn("patch", self.runtime.events) + + def test_fly_loss_after_archive_mirror_stops_before_app_mutation(self): + self.run_action("preflight") + self.runtime.alternate_unhealthy = True + with self.assertRaisesRegex(host.Refused, "alternate_origin_unready"): + self.run_action("sync") + self.assertIn("mirror", self.runtime.events) + self.assertNotIn("patch", self.runtime.events) + self.assertNotIn("dispatch", self.runtime.events) + + def test_stale_or_overlapping_release_cannot_overwrite_newer(self): + self.run_action("preflight") + for phase in ("prepared", "patching", "dispatching", "verified", "complete"): + self.journal["phase"] = phase + for run in (99, 101): + value = dict(self.value, action="preflight", runId=run) + allowed = phase == "complete" and run == 101 + if allowed: + result = host.perform(value, self.runtime, self.journal, lambda _: None) + self.assertEqual(result["phase"], "prepared") + else: + with self.assertRaisesRegex(host.Refused, "previous_release_unresolved_or_newer"): + host.perform(value, self.runtime, self.journal, lambda _: None) + + def test_runtime_change_after_sync_blocks_fly_gate(self): + self.run_action("preflight") + self.run_action("sync") + self.runtime.current["environmentHash"] = "different" + with self.assertRaisesRegex(host.Refused, "runtime_changed"): + self.run_action("check") + + def test_preserves_root_only_atomic_journal(self): + path = Path(self.directory.name) / "journal.json" + with patch.object(host.os, "O_DIRECTORY", getattr(host.os, "O_DIRECTORY", 0)): + host.save(path, {"phase": "dispatching"}) + self.assertEqual(json.loads(path.read_bytes()), {"phase": "dispatching"}) + self.assertFalse(path.with_suffix(".tmp").exists()) + + +class ArtifactTests(unittest.TestCase): + def test_exact_archive_passes_without_docker(self): + with tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory) + manifest, descriptors, rootfs_bytes = host.validate_archive(path, value) + self.assertEqual(manifest["config"], descriptors[1]) + + def test_corrupt_layer_wrong_platform_and_wrong_revision_rejected(self): + for config, corrupt, code in [ + (None, True, "blob_hash"), + ({"architecture": "amd64", "os": "linux"}, False, "image_platform"), + ({"architecture": "arm64", "os": "linux", "config": {"Labels": {"org.opencontainers.image.revision": "a" * 40}}}, False, "image_revision"), + ]: + with self.subTest(code=code), tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory, config, change_layer=corrupt) + with self.assertRaisesRegex(host.Refused, code): + host.validate_archive(path, value) + + def test_decoded_filesystem_and_diff_id_limits_precede_any_registry_write(self): + with tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory) + with patch.object(host, "MAX_ROOTFS", 10), self.assertRaisesRegex(host.Refused, "rootfs_(decoded|filesystem)_limit"): + host.validate_archive(path, value) + with tempfile.TemporaryDirectory() as directory: + config = {"architecture": "arm64", "os": "linux", "config": {"Labels": {"org.opencontainers.image.revision": REVISION}}, + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + "0" * 64]}} + path, value = image_archive(directory, config) + with self.assertRaisesRegex(host.Refused, "rootfs_diff_id"): + host.validate_archive(path, value) + + def test_extra_blob_and_path_traversal_are_not_extracted_or_imported(self): + for name, code in [("../../escape", "archive_path"), ("blobs/sha256/" + "f" * 64, "archive_extra_blob")]: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory, extra={name: b"bad"}) + with self.assertRaisesRegex(host.Refused, code): + host.validate_archive(path, value) + + def test_request_rejects_stale_replay_wrong_registry_extra_command_and_oversize(self): + value = request() + for change in [{"issuedAt": value["issuedAt"] - 901}, {"issuedAt": value["issuedAt"] + 31}, + {"standbyImage": "elsewhere.example/a@sha256:" + "3" * 64}, + {"command": "shell"}, {"revision": "main"}, {"runId": True}, + {"runtimeImage": value["standbyImage"]}]: + with self.subTest(change=change), self.assertRaises(host.Refused): + host.validate_request(dict(value, **change), now=value["issuedAt"]) + with tempfile.TemporaryDirectory() as directory: + _, value = image_archive(directory) + with self.assertRaisesRegex(host.Refused, "archive_size"): + host.validate_request(dict(value, archiveBytes=host.MAX_ARCHIVE + 1)) + + def test_existing_conflicting_registry_tag_is_never_replaced(self): + with tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory) + manifest, descriptors, rootfs_bytes = host.validate_archive(path, value) + instance = host.Host() + with patch.object(instance, "registry", return_value=(200, {}, b"foreign manifest")) as registry: + with self.assertRaisesRegex(host.Refused, "immutable_tag_conflict"): + instance.mirror(path, value, manifest, descriptors) + registry.assert_called_once() + + def test_original_blob_and_manifest_bytes_mirror_then_exact_digest_pull(self): + with tempfile.TemporaryDirectory() as directory: + path, value = image_archive(directory) + manifest, descriptors, rootfs_bytes = host.validate_archive(path, value) + instance = host.Host() + blobs, manifests = {}, {} + def registry(method, route, body=None, headers=None, stream=None, size=0): + if method == "GET": + return (200, {}, manifests[route]) if route in manifests else (404, {}, b"") + if method == "HEAD": + return (200 if route.split("/")[-1] in blobs else 404, {}, b"") + if method == "POST": + return 202, {"Location": "http://127.0.0.1:5000/v2/inline-api/blobs/uploads/synthetic"}, b"" + if stream: + data = stream.read() + digest = host.urllib.parse.parse_qs(host.urllib.parse.urlsplit(route).query)["digest"][0] + self.assertEqual("sha256:" + hashlib.sha256(data).hexdigest(), digest) + self.assertEqual(len(data), size) + blobs[digest] = data + return 201, {}, b"" + manifests[route] = body + return 201, {}, b"" + with patch.object(instance, "registry", side_effect=registry), patch.object(host, "command") as command: + instance.mirror(path, value, manifest, descriptors) + command.assert_called_once_with(["docker", "pull", host.REGISTRY + "@" + value["standbyImage"].split("@")[1]], timeout=120) + self.assertEqual(len(blobs), 2) + self.assertEqual("sha256:" + hashlib.sha256(next(iter(manifests.values()))).hexdigest(), value["standbyImage"].split("@")[1]) + + def test_receipt_identity_freshness_and_safe_projection(self): + selected = request("check") + receipt = {"version": 1, "ok": True, "action": "check", "resource": host.APP, "server": host.SERVER, + "release": host.identity(selected), "phase": "verified", "checkedAt": selected["issuedAt"], + "container": host.APP + "-456", "directTlsReady": True} + client.validate_receipt(receipt, selected, now=selected["issuedAt"]) + for change in [{"checkedAt": selected["issuedAt"] - 61}, {"server": "other"}, {"phase": "prepared"}, + {"release": dict(receipt["release"], attempt=2)}, {"rawProviderBody": "private"}]: + with self.subTest(change=change), self.assertRaises(client.host.Refused): + client.validate_receipt(dict(receipt, **change), selected, now=selected["issuedAt"]) + + +class NativeProjectionTests(unittest.TestCase): + def fixture(self): + value = request() + env = {"NODE_ENV": "production", "INLINE_PROCESS_ROLE": "all", "INLINE_REALTIME_DISTRIBUTED": "1", + "INLINE_DATABASE_QUERY_POOL_MAX": "3", "INLINE_INGRESS_MODE": "cloudflare-optional", + "INLINE_INGRESS_HOST": "api.inline.chat", "INLINE_INGRESS_DIRECT_IP_SOURCE": "x-real-ip", + "ENABLE_DATABASE_HEALTH_MONITOR": "0", "REDIS_URL": "redis://fixture:synthetic-private@172.18.0.1:16379", + "SYNTHETIC_SECRET": "never-output"} + container = {"Name": "/" + host.APP + "-456", "Image": "exact-image-id", + "State": {"Running": True, "Health": {"Status": "healthy"}, "OOMKilled": False}, + "Config": {"Env": [k + "=" + v for k, v in env.items()], "Image": host.REGISTRY + ":" + host.tag(value)}, + "HostConfig": {"Memory": 1610612736, "MemorySwap": 1610612736, + "NanoCpus": 2000000000, "PortBindings": {}}} + image = {"Architecture": "arm64", "Os": "linux", "Config": {"Labels": {"org.opencontainers.image.revision": value["revision"]}}, + "RepoDigests": [host.REGISTRY + "@" + value["standbyImage"].split("@")[1]]} + return value, container, image + + def inspect(self, value, container, image): + with patch.object(host, "command", side_effect=[b"aabbccddeeff co\n", host.encoded([container]), host.encoded([image])]): + return host.Host().runtime(value) + + def test_native_identity_and_configuration_are_verified_not_app_git_metadata(self): + value, container, image = self.fixture() + projected = self.inspect(value, container, image) + self.assertEqual(projected["revision"], value["revision"]) + self.assertNotIn("never-output", json.dumps(projected)) + for mutate in [lambda c, i: i.update(RepoDigests=[]), lambda c, i: i.update(Architecture="amd64"), + lambda c, i: c["HostConfig"].update(PortBindings={"8000/tcp": [{}]}), + lambda c, i: c["State"]["Health"].update(Status="starting"), + lambda c, i: c["Config"].update(Env=[e for e in c["Config"]["Env"] if not e.startswith("REDIS_URL=")]), + lambda c, i: c["Config"].update(Env=[e.replace("INLINE_PROCESS_ROLE=all", "INLINE_PROCESS_ROLE=api") for e in c["Config"]["Env"]])]: + c, i = copy.deepcopy(container), copy.deepcopy(image) + mutate(c, i) + with self.assertRaises(host.Refused): + self.inspect(value, c, i) + + def test_broker_degradation_does_not_pass_ordinary_release(self): + good = {"status": "ok", "checks": {k: {"ok": True} for k in ("database", "clock", "broker")}} + with patch.object(host, "command", return_value=host.encoded(good)): + host.Host().ready() + + good["checks"]["broker"]["ok"] = False + with patch.object(host, "command", return_value=host.encoded(good)), self.assertRaisesRegex(host.Refused, "origin_readiness"): + host.Host().ready() + + +class ControllerBoundaryTests(unittest.TestCase): + def test_nbg_relay_rejects_other_resources_and_never_loads_controller_token(self): + with patch.object(host, "command", return_value=host.encoded({"ok": True, "app": {"uuid": host.APP}})) as command: + host.Host().api("GET", "/applications/" + host.APP) + args, kwargs = command.call_args + self.assertIn("inline-standby-controller-v1", args[0]) + self.assertIn("root@95.216.163.150", args[0]) + self.assertEqual(json.loads(kwargs["stdin"]), {"version": 1, "operation": "read"}) + with patch.object(host, "command") as command, self.assertRaisesRegex(host.Refused, "controller_operation"): + host.Host().api("PATCH", "/applications/other", {"docker_registry_image_tag": "latest"}) + command.assert_not_called() + + def fixture(self): + return {"uuid": controller.APP, "server": {"uuid": controller.SERVER, "ip": "91.107.255.57"}, + "build_pack": "dockerimage", "fqdn": "https://api.inline.chat", "ports_mappings": None, + "docker_registry_image_name": "127.0.0.1:5000/inline-api", "docker_registry_image_tag": "previous", + "settings": {"is_auto_deploy_enabled": False, "is_preview_deployments_enabled": False}, + "env": "private-provider-payload"} + + def test_fixed_resource_only_and_no_provider_or_credential_payload_leaks(self): + app = self.fixture() + api = unittest.mock.Mock() + api.api.return_value = app + result = controller.perform({"version": 1, "operation": "read"}, api) + self.assertNotIn("private-provider-payload", json.dumps(result)) + api.api.assert_called_once_with("GET", "/applications/" + controller.APP) + for value in [{"version": 1, "operation": "read", "target": "other"}, + {"version": 1, "operation": "select", "tag": "latest"}, + {"version": 1, "operation": "shell"}]: + with self.assertRaises(controller.Refused): + controller.perform(value, api) + app["uuid"] = "another-app" + with self.assertRaises(controller.Refused): + controller.perform({"version": 1, "operation": "start", "tag": host.tag(request())}, api) + + def test_select_is_tag_only_idempotent_and_start_requires_exact_tag(self): + app = self.fixture() + events = [] + class API: + def api(self, method, path, body=None): + events.append((method, path, body)) + if method == "PATCH": + self_tag = body["docker_registry_image_tag"] + app["docker_registry_image_tag"] = self_tag + return app + desired = host.tag(request()) + with self.assertRaisesRegex(controller.Refused, "start_tag_changed"): + controller.perform({"version": 1, "operation": "start", "tag": desired}, API()) + controller.perform({"version": 1, "operation": "select", "tag": desired}, API()) + controller.perform({"version": 1, "operation": "select", "tag": desired}, API()) + controller.perform({"version": 1, "operation": "start", "tag": desired}, API()) + mutations = [event for event in events if event[0] != "GET"] + self.assertEqual(mutations, [("PATCH", "/applications/" + controller.APP, {"docker_registry_image_tag": desired}), + ("POST", "/applications/" + controller.APP + "/start", None)]) + + def test_request_requires_manual_main_same_repository_and_explicit_activation(self): + good = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REPOSITORY": "inline-chat/inline", + "GITHUB_REF": "refs/heads/main", "STANDBY_SYNC_ENABLED": "true", "GITHUB_SHA": REVISION, + "RUNTIME_IMAGE": request()["runtimeImage"], "STANDBY_IMAGE": request()["standbyImage"], + "GITHUB_RUN_ID": "100", "GITHUB_RUN_ATTEMPT": "1"} + with patch.dict(client.os.environ, good, clear=True): + client.request("preflight") + for key, wrong in [("GITHUB_EVENT_NAME", "push"), ("GITHUB_REPOSITORY", "outsider/inline"), + ("GITHUB_REF", "refs/heads/other"), ("STANDBY_SYNC_ENABLED", "")]: + with patch.dict(client.os.environ, dict(good, **{key: wrong}), clear=True), self.assertRaisesRegex(client.host.Refused, "release_not_enabled"): + client.request("preflight") + + +if __name__ == "__main__": + if len(sys.argv) == 5 and sys.argv[1] == "--oci-archive": + path, revision, digest = Path(sys.argv[2]), sys.argv[3], sys.argv[4] + with path.open("rb") as stream: + archive_hash = hashlib.file_digest(stream, "sha256").hexdigest() + value = dict(request("sync"), revision=revision, standbyImage="registry.fly.io/inline-api@" + digest, + archiveBytes=path.stat().st_size, archiveSha256=archive_hash) + host.validate_request(value) + _, _, rootfs_bytes = host.validate_archive(path, value) + print(json.dumps({"ok": True, "revision": revision, "manifestDigest": digest, "rootfsBytes": rootfs_bytes})) + else: + unittest.main() diff --git a/server/docs/fly-deployment.md b/server/docs/fly-deployment.md index c0d8f5b22..1325143f6 100644 --- a/server/docs/fly-deployment.md +++ b/server/docs/fly-deployment.md @@ -80,9 +80,12 @@ does not establish these effective settings. ### Release contract with a warm fallback Since October 16, 2026, `api.inline.chat` uses Cloudflare load balancing with -Fly preferred and a running Nuremberg fallback. The Fly workflow publishes both -architecture-specific images but deploys and verifies only Fly. It does not -refresh Nuremberg or prove that automatic fallback remains usable. +Fly preferred and a running Nuremberg fallback. The manual workflow publishes both +architecture-specific images. Ordinary releases now require the paired standby +preflight, sync and exact readback described in +[paired release operations](standby-release-sync.md). The required consumer/key +installation and production environment activation are separate operations; +missing activation fails before DDL. Source implementation is not live rollout. Before releasing the next selected `main` SHA to Fly, qualify that same SHA's ARM64 image on Nuremberg against the intended schema and runtime configuration. @@ -94,9 +97,10 @@ and qualify Nuremberg before replacing Fly, keeping another ready origin during each replacement. Verify direct HTTP/V2/V3 and cross-node recovery, then verify canonical routing and both effective release identities after the Fly rollout. -The current workflow does not enforce this fleet gate. Until a reviewed guard -is installed, it is an explicit operator prerequisite, and a green Fly-only -workflow is not completion of the fleet release. Deliberate mixed-version +The automated gate covers exact image/runtime intent and direct readiness, while +schema compatibility, authenticated acceptance, rollback and routing topology +remain operator prerequisites. A green Fly job alone is not completion of the +fleet release. Deliberate mixed-version overlap needs a recorded compatibility decision and an owner/deadline for finishing it; it must not leave an indefinitely stale standby. See [the standby release procedure](hetzner-standby.md#rolling-upgrades-and-removing-a-node). diff --git a/server/docs/hetzner-standby.md b/server/docs/hetzner-standby.md index 9a39c6351..f7bd82852 100644 --- a/server/docs/hetzner-standby.md +++ b/server/docs/hetzner-standby.md @@ -137,6 +137,11 @@ and [Full](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes/ ## Release and qualification +The [paired manual release workflow](standby-release-sync.md) implements exact +ARM standby synchronization before ordinary Fly replacement. Its restricted +consumer installation and activation are separate; these must be qualified +before enabling future releases. It does not change DNS/LB configuration. + 1. Qualify one committed source revision in the amd64 runtime and native ARM64 Coolify CI lanes. After all jobs pass, retain the `runtime_image` and `standby_image` immutable digests, source revision and workflow run URL. diff --git a/server/docs/standby-release-sync.md b/server/docs/standby-release-sync.md new file mode 100644 index 000000000..67749e620 --- /dev/null +++ b/server/docs/standby-release-sync.md @@ -0,0 +1,148 @@ +# Paired Fly and Nuremberg releases + +The manual `server-deploy.yml` workflow implements a required standby gate for +ordinary production releases. Publication-only dispatches still publish both +tested architectures without deployment. The separate first-API `bootstrap` +path keeps its existing API-only and protected worker-activation gates; it does +not run standby synchronization. Bootstrap cannot adopt an existing all-role +Fly fleet and is not an ordinary-release bypass. + +**Activation is separate from source implementation.** Until the reviewed +consumers, restricted keys and production environment settings below are +installed, ordinary releases fail before DDL. Committing or pushing this code +does not install anything, dispatch a release or change production. + +## Ordinary release order + +1. Qualify both native images and all server tests for the frozen main SHA. + Only outputs of the successful reusable workflow are eligible. +2. Read healthy managed Fly Machines and the fixed healthy Nuremberg app before + DDL. Bind the predecessor container/configuration to this run and attempt. +3. Apply and verify the existing migration ledger once. The production approval + must establish additive old/new/rollback compatibility; metadata checks cannot + prove schema or application compatibility. +4. Copy the selected ARM digest into an OCI archive without rebuilding. Re-read + Fly immediately before sending it; its healthy baseline must still match. + Refuse to replace the sole healthy origin during a Fly incident. +5. The restricted Nuremberg consumer verifies the transport hash/size, original + manifest/config/compressed-layer digests, source revision, ARM64 platform and + ordered decoded rootfs digests. It bounds both decoded tar bytes and declared + file sizes, including sparse files, to 2 GiB. It reserves space for the local + registry, Docker cache, expanded filesystem and another 1 GiB, plus 2 GiB + available RAM for replacement. An unsupported image format fails before pull. +6. Mirror exact bytes only into the existing loopback registry under a unique + SHA/run/attempt tag. Never overwrite a conflicting tag. Recheck Fly directly + after transfer/pull, immediately before tag selection and start; its pinned + origin address must stay current with the qualified Fly origin. The controller relay + changes only the fixed app's image tag and starts it once. No environment, + routing, proxy, resource-limit, database or other-app setting is written. +7. Require one healthy matching container, original ARM RepoDigest, source label, + unchanged runtime environment, all-role/distributed/private-broker intent, + resource caps and unpublished backend. Verify canonical-SNI public-trust TLS + readiness directly against Nuremberg from both host and Actions, including + database/clock/broker checks. Canonical LB readiness cannot substitute for it. +8. Read Nuremberg again immediately before Fly, retain Fly's independent exact + baseline/image/count/service-health gates, then read Nuremberg after Fly. + Retain the safe run-bound receipts as workflow artifacts. + +This is image/configuration/readiness acceptance. It does not prove authenticated +HTTP/V2/V3, worker execution, cross-node delivery, capacity, Cloudflare pool order, +or compatible rollback. Existing qualification and origin/certificate monitoring +remain necessary. The workflow never changes Cloudflare routing. + +## Restricted credential and installation boundary + +Review and install root-owned, non-writable helpers through existing approved +admin access. Do not place a personal/root shell key or a broad Coolify bearer +in Actions. Do not read or export managed CLI token configuration. + +| Location | Installed source / state | +| --- | --- | +| Nuremberg | `standby-release-host.py` → `/usr/local/libexec/inline-standby-release`; `standby-release-entrypoint.sh` → `/usr/local/libexec/inline-standby-release-entrypoint` | +| Nuremberg | root-only `/var/lib/inline-standby-release/`; `/etc/inline-standby-release/controller-key` and `controller-known-hosts` | +| Existing Finland controller | `standby-release-controller.py` → `/usr/local/libexec/inline-standby-controller`; `standby-release-controller-entrypoint.sh` → `/usr/local/libexec/inline-standby-controller-entrypoint` | +| Finland only | reviewed existing Coolify token encrypted as `/etc/credstore.encrypted/inline-standby-coolify`, exposed only through systemd `LoadCredentialEncrypted=coolify:...` | + +The existing controller already holds management authority for these apps. Keep +its bearer there: host-key encryption would not protect a copied broader bearer +from Nuremberg root. Before installation, establish the existing token's actual +scope, provenance and approved use privately. No suitable token, stored path, +or root credential is created or asserted by this implementation. + +Use two dedicated keys with distinct scopes: + +- Actions → Nuremberg: `restrict,command="/usr/local/libexec/inline-standby-release-entrypoint"`. + Only original command `inline-standby-release-v1` is accepted. No PTY, + forwarding, shell, arbitrary argv, caller paths or provider credentials. +- Nuremberg → Finland port 6642: + `from="91.107.255.57",restrict,command="/usr/local/libexec/inline-standby-controller-entrypoint"`. + Only original command `inline-standby-controller-v1`; the relay accepts safe + app metadata read, one validated release tag selection, or start of that exact + app/tag. It returns an explicit non-secret projection. It cannot select another + resource, change environment/configuration, access keys, run shell or forward. + +Both entrypoints use clean environments and transient bounded systemd units, +core dumps disabled and control-group termination. The bearer exists only in the +Finland private credential namespace. The image stream contains no registry auth. +Private request/command/provider diagnostics are suppressed, including failures. +The deployment key is an authorization boundary: it trusts the protected workflow, +not merely a caller-supplied OCI source label. Review all workflows able to access +that environment secret and restrict production environments to main. + +After helper hash/permission, native unit, host-key and fixed-target read-only +qualification, configure in the protected `production` environment: + +- secret `STANDBY_RELEASE_SSH_KEY`: the dedicated forced-command key, never an + operator key; +- variable `STANDBY_RELEASE_KNOWN_HOSTS`: independently verified pinned Nuremberg + public host key for `91.107.255.57`, never live unauthenticated `ssh-keyscan`; +- variable `STANDBY_SYNC_ENABLED=true`, last, after an explicitly authorized + rollout rehearsal. Production dispatch remains separately manual. + +No helper/key/credential/environment setting is installed by a workflow run. +Actual transport, controller version/endpoints, systemd credential loading and +rolling replacement/drain need an authorized rehearsal before activation. The +ARM CI lane tests real daemonless OCI export/validation on its disposable image; +local synthetic tests alone do not establish provider execution. + +## Uncertainty, recovery and independent deployment + +The root-only journal is fsynced before controller PATCH and start. A partial or +uncertain mutation fences subsequent releases. A late successful start can close +only after exact live runtime/tag/config/readiness reconciliation; it never sends +another start request. Known successful retries only re-read. Older runs cannot +replace a newer completed run, and another run cannot overwrite an unfinished +one. Actions has one non-cancelling release concurrency group; Nuremberg also has +a non-blocking native lock. Out-of-band admin changes remain possible and are +detected by live readback, not prevented by a cross-provider transaction. + +If a job fails, do not delete the journal or blindly rerun deploy. Record its +phase, exact desired/retained images, controller deployment outcome and both +runtime identities. Reconcile the provider's final state with approved admin +access; retain the journal before explicitly rearming a failed attempt. No +automatic DDL rollback or fleet rollback occurs. A failed standby gate leaves +Fly's existing deployment untouched, but DDL may already have succeeded. + +The paired path intentionally needs both providers. It is **not the emergency +deployment path** when Hetzner/Coolify, Actions or Fly builders are unavailable. +Keep current and compatible rollback source/images/configuration encrypted in +the independent [release escrow](server-release-escrow.md), with its independent +credential and full readback receipts. An explicitly reviewed operator can +recover a retained qualified AMD image on an independent workstation, import it +into an available registry and deploy the immutable image to Fly using the +existing [Fly release/rollback contract](fly-deployment.md#migration-safety-and-rollback), +without this paired workflow or a new build. For Fly failure, use the retained +ARM image and existing controller or reviewed controller-bypass recovery path. +An independent new-source build additionally needs available source/dependency +mirrors and build capacity; retained-image recovery does not prove that ability. + +Do not add a blind `skip_standby` switch. An emergency release records the target, +source/digest, schema/configuration/rollback evidence, unavailable provider and +mixed-fleet owner/deadline. When the provider returns, reconcile its actual +runtime before synchronizing. The normal next release fails closed on drift; +the existing origin observer watches readiness, not image freshness, so there is +no claim of a continuously active version-drift reconciler. + +Implementation references: [Skopeo digest-preserving copy](https://github.com/podman-container-tools/skopeo/blob/main/docs/skopeo-copy.1.md), +[Coolify app update](https://coolify.io/docs/api/endpoints/applications/update-application-by-uuid), +[Coolify app start](https://coolify.io/docs/api/endpoints/applications/start-application-by-uuid). From 8ad3f46c8e9ab259e1819c8dd4aeb8f4277b0772 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Sat, 17 Oct 2026 01:18:29 +0330 Subject: [PATCH 2/5] ci: allow validation-only server dispatch for stacked changes --- .github/workflows/server-test.yml | 3 +++ scripts/ci/fly-release-machines.test.ts | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 4efe7ca47..09a4e302f 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -1,6 +1,9 @@ name: Server Tests on: + # Validation-only entry point for a stacked PR whose base is not main. + # No publication input exists here; only the guarded reusable call can publish. + workflow_dispatch: workflow_call: inputs: publish_image: diff --git a/scripts/ci/fly-release-machines.test.ts b/scripts/ci/fly-release-machines.test.ts index 425838274..5b20a35ef 100644 --- a/scripts/ci/fly-release-machines.test.ts +++ b/scripts/ci/fly-release-machines.test.ts @@ -99,6 +99,14 @@ test("both image publishing guards require manual same-repository branches and e } }) +test("manual Server Tests dispatch exposes no publication inputs", async () => { + const workflow = Bun.YAML.parse(await Bun.file(resolve(import.meta.dir, "../../.github/workflows/server-test.yml")).text()) as { + on: Record } | null> + } + expect(Object.hasOwn(workflow.on, "workflow_dispatch")).toBe(true) + expect(workflow.on.workflow_dispatch?.inputs).toBeUndefined() +}) + type ReleaseJob = { if?: string needs?: string | string[] From 1759617481d0c9b17814142fb200ba671335f420 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Sat, 17 Oct 2026 01:33:02 +0330 Subject: [PATCH 3/5] ci: bound standby registry readiness retries --- .github/workflows/server-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 09a4e302f..3af2266e1 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -210,6 +210,7 @@ jobs: docker run --detach --rm --name "$registry" --publish 127.0.0.1:55000:5000 registry:2 >/dev/null for attempt in {1..20}; do if curl --silent --fail http://127.0.0.1:55000/v2/ >/dev/null; then break; fi + if [[ "$attempt" = 20 ]]; then exit 1; fi sleep 1 done image=127.0.0.1:55000/inline-api:transport-test From 8ef6b173fe3b03e4b7ed83b550fd7356f2b729c8 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Sat, 17 Oct 2026 01:39:19 +0330 Subject: [PATCH 4/5] server: bind standby relay to audited controller projection --- .github/workflows/integrations.yml | 1 + scripts/ci/standby-release-controller.py | 5 +++-- scripts/ci/standby-release-test.py | 23 ++++++++++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index fa6c7d04a..c922c9ee8 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -300,6 +300,7 @@ jobs: run: | go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.10 -color cd scripts && bun --no-env-file test ci && cd .. + python3 scripts/ci/standby-release-test.py python3 -m unittest discover -s scripts/ci -p 'test_nightly_tip_gate.py' - name: Check CI shell scripts run: | diff --git a/scripts/ci/standby-release-controller.py b/scripts/ci/standby-release-controller.py index 39c44566c..106493221 100644 --- a/scripts/ci/standby-release-controller.py +++ b/scripts/ci/standby-release-controller.py @@ -38,7 +38,8 @@ def validate(value): def project(app): settings = app.get("settings", {}) - server = app.get("server", {}) + # Actual Docker-image application GET schema: server belongs to destination. + server = app.get("destination", {}).get("server", {}) need(app.get("uuid") == APP and server.get("uuid") == SERVER and server.get("ip") == "91.107.255.57" and app.get("build_pack") == "dockerimage" and app.get("fqdn") == "https://api.inline.chat" and app.get("docker_registry_image_name") == "127.0.0.1:5000/inline-api" and not app.get("ports_mappings") @@ -59,7 +60,7 @@ def redirect_request(self, *args, **kwargs): return None request = urllib.request.Request(ORIGIN + suffix, method=method, data=None if body is None else json.dumps(body).encode(), - headers={"Authorization": "Bearer " + self.token, "Content-Type": "application/json"}) + headers={"Authorization": "Bearer " + self.token, "Content-Type": "application/json", "User-Agent": "coolify-cli"}) with urllib.request.build_opener(NoRedirect).open(request, timeout=15) as response: data = response.read(2 * 1024 * 1024 + 1) need(len(data) <= 2 * 1024 * 1024, "provider_size") diff --git a/scripts/ci/standby-release-test.py b/scripts/ci/standby-release-test.py index 705d0cf9c..528625ecd 100644 --- a/scripts/ci/standby-release-test.py +++ b/scripts/ci/standby-release-test.py @@ -393,6 +393,20 @@ def test_broker_degradation_does_not_pass_ordinary_release(self): class ControllerBoundaryTests(unittest.TestCase): + def test_controller_transport_matches_audited_user_agent_without_redirects(self): + class Response: + def __enter__(self): return self + def __exit__(self, *args): return None + def read(self, limit): return b"{}" + opener = unittest.mock.Mock() + opener.open.return_value = Response() + with patch.object(controller.urllib.request, "build_opener", return_value=opener) as build: + controller.Controller("synthetic-token").api("GET", "/applications/" + controller.APP) + req = opener.open.call_args.args[0] + self.assertEqual(req.get_header("User-agent"), "coolify-cli") + self.assertEqual(req.full_url, controller.ORIGIN + "/applications/" + controller.APP) + self.assertIsNone(build.call_args.args[0].redirect_request(None, None, 302, "redirect", {}, "https://foreign.invalid")) + def test_nbg_relay_rejects_other_resources_and_never_loads_controller_token(self): with patch.object(host, "command", return_value=host.encoded({"ok": True, "app": {"uuid": host.APP}})) as command: host.Host().api("GET", "/applications/" + host.APP) @@ -405,7 +419,7 @@ def test_nbg_relay_rejects_other_resources_and_never_loads_controller_token(self command.assert_not_called() def fixture(self): - return {"uuid": controller.APP, "server": {"uuid": controller.SERVER, "ip": "91.107.255.57"}, + return {"uuid": controller.APP, "destination": {"server": {"uuid": controller.SERVER, "ip": "91.107.255.57"}}, "build_pack": "dockerimage", "fqdn": "https://api.inline.chat", "ports_mappings": None, "docker_registry_image_name": "127.0.0.1:5000/inline-api", "docker_registry_image_tag": "previous", "settings": {"is_auto_deploy_enabled": False, "is_preview_deployments_enabled": False}, @@ -427,6 +441,13 @@ def test_fixed_resource_only_and_no_provider_or_credential_payload_leaks(self): with self.assertRaises(controller.Refused): controller.perform({"version": 1, "operation": "start", "tag": host.tag(request())}, api) + def test_actual_destination_server_wins_over_misleading_top_level_metadata(self): + app = self.fixture() + app["server"] = app["destination"]["server"] + app["destination"] = {"server": {"uuid": "foreign-server", "ip": "192.0.2.1"}} + with self.assertRaisesRegex(controller.Refused, "resource_identity_or_intent"): + controller.project(app) + def test_select_is_tag_only_idempotent_and_start_requires_exact_tag(self): app = self.fixture() events = [] From cfb0c6f0fb7c55fb301aa4c11d2f8549b540c383 Mon Sep 17 00:00:00 2001 From: Mohammad Rajabi Date: Sat, 17 Oct 2026 13:25:05 +0330 Subject: [PATCH 5/5] server: prepare idempotent standby install and read-only probe --- .github/workflows/integrations.yml | 1 + scripts/ci/standby-release-host.py | 19 +++ scripts/ci/standby-release-install-test.py | 76 ++++++++++ scripts/ci/standby-release-install.json | 33 ++++ scripts/ci/standby-release-install.py | 166 +++++++++++++++++++++ scripts/ci/standby-release-test.py | 24 +++ server/docs/standby-release-sync.md | 62 ++++++++ 7 files changed, 381 insertions(+) create mode 100644 scripts/ci/standby-release-install-test.py create mode 100644 scripts/ci/standby-release-install.json create mode 100644 scripts/ci/standby-release-install.py diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index c922c9ee8..9eb78aa8d 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -301,6 +301,7 @@ jobs: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.10 -color cd scripts && bun --no-env-file test ci && cd .. python3 scripts/ci/standby-release-test.py + python3 scripts/ci/standby-release-install-test.py python3 -m unittest discover -s scripts/ci -p 'test_nightly_tip_gate.py' - name: Check CI shell scripts run: | diff --git a/scripts/ci/standby-release-host.py b/scripts/ci/standby-release-host.py index 33d1f00fd..3011e1060 100644 --- a/scripts/ci/standby-release-host.py +++ b/scripts/ci/standby-release-host.py @@ -46,6 +46,9 @@ def encoded(value): def validate_request(value, now=None): + if value.get("action") == "probe": + need(value == {"version": 1, "action": "probe"}, "probe_fields") + return value fields = {"version", "action", "revision", "runtimeImage", "standbyImage", "runId", "attempt", "issuedAt"} if value.get("action") == "sync": fields |= {"archiveBytes", "archiveSha256"} @@ -336,6 +339,19 @@ def save(path, value): os.close(fd) +def probe(host): + app = host.app() + current = host.runtime() + host.ready() + host.alternate() + host.headroom() + return {"version": 1, "ok": True, "action": "probe", "resource": APP, "server": SERVER, + "checkedAt": int(time.time()), "container": current["container"], "revision": current["revision"], + "configuredTag": app["docker_registry_image_tag"], "directTlsReady": True, + "alternateTlsReady": True, "headroomReady": True, "releaseQualified": False, + "journalChanged": False, "deploymentDispatched": False} + + def perform(request, host, journal, write, archive=None): selected = identity(request) action = request["action"] @@ -432,6 +448,9 @@ def main(): line = sys.stdin.buffer.readline(16385) need(len(line) <= 16384 and line.endswith(b"\n"), "request_line") request = validate_request(json.loads(line)) + if request["action"] == "probe": + need(sys.stdin.buffer.read(1) == b"", "unexpected_payload") + return probe(Host()) STATE.mkdir(mode=0o700, parents=True, exist_ok=True) with (STATE / "lock").open("a") as lock: try: diff --git a/scripts/ci/standby-release-install-test.py b/scripts/ci/standby-release-install-test.py new file mode 100644 index 000000000..e1be101e4 --- /dev/null +++ b/scripts/ci/standby-release-install-test.py @@ -0,0 +1,76 @@ +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("installer", Path(__file__).with_name("standby-release-install.py")) +installer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(installer) + + +class InstallTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.guard = patch.object(installer, "ROOT", self.root) + self.guard.start() + self.addCleanup(self.guard.stop) + # Fixtures test safe path logic without requiring local root ownership. + self.owner = patch.object(installer, "parents_safe") + self.owner.start() + self.addCleanup(self.owner.stop) + + def test_exact_install_is_idempotent_and_leaves_no_keys_tokens_or_services(self): + first = installer.install("nbg") + self.assertEqual(first["installedNewFiles"], 2) + second = installer.install("nbg") + self.assertEqual(second["installedNewFiles"], 0) + self.assertFalse(first["keysChanged"] or first["credentialsChanged"] or first["servicesRestarted"]) + self.assertEqual(len(list(self.root.rglob("inline-standby-release"))), 3) + self.assertTrue(all(x["state"] == "matching" for x in second["files"])) + + def test_conflict_rejected_before_any_partial_install(self): + path = installer.rooted("/usr/local/libexec/inline-standby-controller-entrypoint") + path.parent.mkdir(parents=True) + path.write_bytes(b"foreign") + path.chmod(0o700) + with self.assertRaisesRegex(installer.Refused, "existing_file_conflict"): + installer.install("controller") + self.assertFalse(path.with_name("inline-standby-controller").exists()) + self.assertEqual(path.read_bytes(), b"foreign") + + def test_verify_rejects_missing_dedicated_directory(self): + installer.install("nbg") + installer.rooted("/etc/inline-standby-release").rmdir() + with patch.object(installer, "host_identity"), patch.object(installer.sys, "argv", ["installer", "verify", "nbg"]), \ + self.assertRaisesRegex(installer.Refused, "installation_incomplete"): + installer.main() + self.assertIn({"path": "/etc/inline-standby-release", "state": "absent"}, installer.directories("nbg")) + + def test_hash_tampering_rejected_and_no_output_files_created(self): + manifest = json.loads((installer.BUNDLE / "standby-release-install.json").read_bytes()) + manifest["roles"]["nbg"][0]["sha256"] = "0" * 64 + original = Path.read_bytes + def read(path): + return json.dumps(manifest).encode() if path == installer.BUNDLE / "standby-release-install.json" else original(path) + with patch.object(Path, "read_bytes", read), self.assertRaisesRegex(installer.Refused, "bundle_hash"): + installer.install("nbg") + self.assertEqual(list(self.root.iterdir()), []) + + def test_symlink_path_rejected_by_real_guard(self): + self.owner.stop() + # root ownership is checked after the symlink itself, so this portable + # fixture reaches the symlink refusal without a real root fixture. + link = self.root / "link" + link.symlink_to(self.root / "foreign") + with self.assertRaisesRegex(installer.Refused, "symlink_path"): + installer.parents_safe(link) + self.owner.start() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/standby-release-install.json b/scripts/ci/standby-release-install.json new file mode 100644 index 000000000..ab9046866 --- /dev/null +++ b/scripts/ci/standby-release-install.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "roles": { + "nbg": [ + { + "source": "standby-release-host.py", + "destination": "/usr/local/libexec/inline-standby-release", + "sha256": "3194ea775209a3b8750122cb1050a15f24ce59fd02188b22e52ad6ac5b1ffd4a", + "mode": 448 + }, + { + "source": "standby-release-entrypoint.sh", + "destination": "/usr/local/libexec/inline-standby-release-entrypoint", + "sha256": "128e1697d6c0ec3bc1e075c4f2f8c7967edb8a0839fe1a99279473fee339072d", + "mode": 448 + } + ], + "controller": [ + { + "source": "standby-release-controller.py", + "destination": "/usr/local/libexec/inline-standby-controller", + "sha256": "1028025925843eb960354e139582ed29030f51c22df4916a7a8e2e0eb4dbd039", + "mode": 448 + }, + { + "source": "standby-release-controller-entrypoint.sh", + "destination": "/usr/local/libexec/inline-standby-controller-entrypoint", + "sha256": "ab8227515d7bcafc6e18574a6aa93f8c53be25e8fe7cc92282fa0a73ad0381de", + "mode": 448 + } + ] + } +} diff --git a/scripts/ci/standby-release-install.py b/scripts/ci/standby-release-install.py new file mode 100644 index 000000000..26e04736a --- /dev/null +++ b/scripts/ci/standby-release-install.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Review-only until approved: fixed helper install, no keys/token/deploy/restart. + +plan is local/read-only; install/verify must run through approved existing root +access on the exact role. Conflicts fail closed, successful repeats are no-ops. +""" +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile + +ROOT = Path("/") +BUNDLE = Path(__file__).resolve().parent +TARGETS = { + "nbg": {"ip": "91.107.255.57", "files": { + "standby-release-host.py": "/usr/local/libexec/inline-standby-release", + "standby-release-entrypoint.sh": "/usr/local/libexec/inline-standby-release-entrypoint"}, + "directories": ["/etc/inline-standby-release", "/var/lib/inline-standby-release"]}, + "controller": {"ip": "95.216.163.150", "files": { + "standby-release-controller.py": "/usr/local/libexec/inline-standby-controller", + "standby-release-controller-entrypoint.sh": "/usr/local/libexec/inline-standby-controller-entrypoint"}, + "directories": []}} + + +class Refused(Exception): + pass + + +def need(value, code): + if not value: + raise Refused(code) + + +def rooted(path): + return ROOT / path.lstrip("/") + + +def parents_safe(path): + for parent in [path, *path.parents]: + if parent.exists() or parent.is_symlink(): + metadata = parent.lstat() + need(not stat.S_ISLNK(metadata.st_mode), "symlink_path") + need(metadata.st_uid == 0 and not metadata.st_mode & 0o022, "unsafe_owner_or_mode") + if parent == ROOT: + break + + +def selected(role): + need(role in TARGETS, "role") + manifest = json.loads((BUNDLE / "standby-release-install.json").read_bytes()) + need(manifest.get("version") == 1 and set(manifest.get("roles", {})) == set(TARGETS), "manifest") + items = manifest["roles"][role] + need(len(items) == 2 and {x["source"]: x["destination"] for x in items} == TARGETS[role]["files"], "manifest_targets") + out = [] + for item in items: + source = BUNDLE / item["source"] + need(source.is_file() and not source.is_symlink(), "bundle_file") + data = source.read_bytes() + need(item.get("mode") == 0o700 and hashlib.sha256(data).hexdigest() == item.get("sha256"), "bundle_hash") + out.append((item, data)) + return out + + +def host_identity(role): + need(os.geteuid() == 0, "root_required") + result = subprocess.run(["ip", "-j", "address", "show"], capture_output=True, timeout=5) + need(result.returncode == 0, "host_identity") + addresses = {entry.get("local") for interface in json.loads(result.stdout) for entry in interface.get("addr_info", [])} + need(TARGETS[role]["ip"] in addresses, "wrong_host") + + +def inspect(role): + files = [] + for item, data in selected(role): + path = rooted(item["destination"]) + parents_safe(path) + exists = path.exists() + if exists: + metadata = path.lstat() + need(stat.S_ISREG(metadata.st_mode) and stat.S_IMODE(metadata.st_mode) == 0o700 + and path.read_bytes() == data, "existing_file_conflict") + files.append({"path": item["destination"], "sha256": item["sha256"], "state": "matching" if exists else "absent"}) + for directory in TARGETS[role]["directories"]: + path = rooted(directory) + parents_safe(path) + if path.exists(): + need(path.is_dir() and stat.S_IMODE(path.stat().st_mode) == 0o700, "existing_directory_conflict") + return files + + +def directories(role): + result = [] + for directory in TARGETS[role]["directories"]: + path = rooted(directory) + parents_safe(path) + exists = path.exists() + if exists: + need(path.is_dir() and stat.S_IMODE(path.stat().st_mode) == 0o700, "existing_directory_conflict") + result.append({"path": directory, "state": "matching" if exists else "absent"}) + return result + + +def sync_directory(path): + descriptor = os.open(path, os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def install(role): + files = inspect(role) # Validate ALL inputs/conflicts before any file mutation. + for directory in TARGETS[role]["directories"]: + path = rooted(directory) + existed = path.exists() + path.mkdir(mode=0o700, parents=True, exist_ok=True) + if not existed: + sync_directory(path.parent) + for item, data in selected(role): + path = rooted(item["destination"]) + if path.exists(): + continue + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=".inline-standby-install-", dir=path.parent) + try: + with os.fdopen(descriptor, "wb") as output: + os.fchmod(output.fileno(), 0o700) + output.write(data) + output.flush() + os.fsync(output.fileno()) + # Do not overwrite a file created concurrently since preflight. + os.link(temporary, path) + sync_directory(path.parent) + finally: + os.unlink(temporary) + return {"files": inspect(role), "directories": directories(role), "installedNewFiles": sum(x["state"] == "absent" for x in files), + "keysChanged": False, "credentialsChanged": False, "servicesRestarted": False, "deploymentDispatched": False} + + +def main(): + need(len(sys.argv) == 3 and sys.argv[1] in ("plan", "install", "verify"), "arguments") + action, role = sys.argv[1:] + need(role in TARGETS, "role") + if action == "plan": + return {"ok": True, "action": action, "role": role, + "files": [item for item, _ in selected(role)], "writes": False} + host_identity(role) + os.umask(0o077) + if action == "install": + result = install(role) + else: + result = {"files": inspect(role), "directories": directories(role)} + need(all(x["state"] == "matching" for x in [*result["files"], *result["directories"]]), "installation_incomplete") + return {"ok": True, "action": action, "role": role, **result} + + +if __name__ == "__main__": + try: + print(json.dumps(main())) + except Exception as error: + print(json.dumps({"ok": False, "code": str(error) if isinstance(error, Refused) else "private_install_failure"})) + sys.exit(1) diff --git a/scripts/ci/standby-release-test.py b/scripts/ci/standby-release-test.py index 528625ecd..5c2d4da19 100644 --- a/scripts/ci/standby-release-test.py +++ b/scripts/ci/standby-release-test.py @@ -134,6 +134,30 @@ def write(self, value): self.journal = copy.deepcopy(value) self.runtime.events.append("journal:" + value["phase"]) + def test_probe_is_read_only_and_cannot_qualify_a_release(self): + result = host.probe(self.runtime) + self.assertFalse(result["releaseQualified"]) + self.assertFalse(result["journalChanged"]) + self.assertFalse(result["deploymentDispatched"]) + self.assertEqual(self.runtime.events, []) + self.assertNotIn("secret-fingerprint", json.dumps(result)) + host.validate_request({"version": 1, "action": "probe"}) + with self.assertRaisesRegex(host.Refused, "probe_fields"): + host.validate_request({"version": 1, "action": "probe", "tag": "release-untrusted"}) + + def test_native_probe_does_not_create_state_directory_or_lock(self): + payload = io.BytesIO(b'{"version":1,"action":"probe"}\n') + fake_stdin = unittest.mock.Mock(buffer=payload) + missing = Path(self.directory.name) / "never-created" + with patch.object(host.os, "geteuid", return_value=0), patch.object(host.sys, "argv", ["helper", "--execute"]), \ + patch.dict(host.os.environ, {"SSH_ORIGINAL_COMMAND": "inline-standby-release-v1"}, clear=True), \ + patch.object(host.sys, "stdin", fake_stdin), patch.object(host, "STATE", missing), \ + patch.object(host, "Host", return_value=self.runtime), patch.object(host.signal, "signal"), \ + patch.object(host.signal, "alarm"): + self.assertEqual(host.main()["action"], "probe") + self.assertFalse(missing.exists()) + self.assertEqual(self.runtime.events, []) + def run_action(self, action): value = dict(self.value, action=action) if action != "sync": diff --git a/server/docs/standby-release-sync.md b/server/docs/standby-release-sync.md index 67749e620..d3dd64d47 100644 --- a/server/docs/standby-release-sync.md +++ b/server/docs/standby-release-sync.md @@ -105,6 +105,68 @@ rolling replacement/drain need an authorized rehearsal before activation. The ARM CI lane tests real daemonless OCI export/validation on its disposable image; local synthetic tests alone do not establish provider execution. +### Install without deploying + +`scripts/ci/standby-release-install.json` pins the four reviewed helper hashes. +The installer verifies all inputs and conflicts before writing anything, refuses +symlinks/unsafe ownership or existing different files, and leaves exact matches +untouched. It creates only the helpers and dedicated root-only NBG directories. +It does not create keys, read/install tokens, change authorized keys, reload SSH, +start/restart any service, or dispatch a deployment. A conflict requires an +explicit reviewed upgrade; the installer never overwrites it. + +Prepare/review locally: + +```sh +python3 scripts/ci/standby-release-install.py plan nbg +python3 scripts/ci/standby-release-install.py plan controller +``` + +After approved existing root access has staged this exact installer, manifest, +and the selected role's two source files together in a root-only directory: + +```sh +# NBG only: installer checks that this host owns91.107.255.57. +python3 -I standby-release-install.py install nbg +python3 -I standby-release-install.py verify nbg +# Existing Finland controller only: must own95.216.163.150. +python3 -I standby-release-install.py install controller +python3 -I standby-release-install.py verify controller +``` + +Credential/key authorization and pins remain separate reviewed operations. +After those are installed, exercise the actual NBG forced lane using only: + +```sh +printf '%s\n' '{"version":1,"action":"probe"}' | + ssh -T -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile=VERIFIED_NBG_PUBLIC_HOST_PIN -i DEDICATED_FORCED_KEY \ + root@91.107.255.57 inline-standby-release-v1 +``` + +`probe` accepts no release/tag/archive fields. It branches before state-directory +creation, journal/lock handling and release mutation code. It calls the actual +controller read relay, inspects existing Docker runtime in memory and checks +direct strict TLS/readiness/headroom for both origins. Its safe result includes +`releaseQualified=false`, `journalChanged=false`, and +`deploymentDispatched=false`. It neither repairs unhealthy state nor alters +readiness. Do not use `preflight` as a smoke: that action reserves a release in +the durable journal. + +On NBG, the actual controller forced lane can independently be tested with +`{"version":1,"operation":"read"}` over the existing pinned dedicated relay +key to `root@95.216.163.150:6642` and original command +`inline-standby-controller-v1`. The installed controller entrypoint must load +the host-encrypted credential; running a plain Python mock is not native proof. +No token or provider response should be printed beyond the safe projection. + +These no-op probes can qualify helper hashes/permissions, forced-command and +native credential transport, current raw provider shape, runtime intent and +direct health without serving-container replacement. They **cannot** qualify +mirror/pull/PATCH/start, rolling replacement/drain, final image identity or +rollback. Keep `STANDBY_SYNC_ENABLED` unset until the separately authorized real +rehearsal; a successful probe does not waive this gate. + ## Uncertainty, recovery and independent deployment The root-only journal is fsynced before controller PATCH and start. A partial or