From 0be9a7a4373b8327949af577b21dd3f1441a758d Mon Sep 17 00:00:00 2001 From: Alan Cruz Date: Wed, 2 Sep 2026 14:51:39 -0400 Subject: [PATCH] =?UTF-8?q?ci(cluster):=20harden=20the=20two-instance=20sm?= =?UTF-8?q?oke=20=E2=80=94=20nginx=20probe,=20SIGTERM=20drain,=20nightly?= =?UTF-8?q?=20tier,=20build=20features?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skeleton probed nginx by proxying /health to instances that were not started yet; with nginx's default max_fails=1 fail_timeout=10s those refused connects blacklisted both upstreams for 10s, so the smoke's first request through the front could land inside that window as "no live upstreams". The stop step also used SIGINT, which is not what a rolling deploy sends. - Upstreams carry max_fails=0 and readiness probes an nginx-local /nginx-health location, so the probe never touches the upstreams. - The stop step sends SIGTERM and asserts each instance drains: it must exit within 10s and log the drain line; needing SIGKILL fails the step. - A `nightly` dispatch input plus a 06:00 UTC schedule set a NIGHTLY env for the recovery cases later cluster PRs add. - hfs is built with subscriptions and elasticsearch so later cases need no workflow change; a build-hts job produces hts-cluster-smoke-binary for the terminology cache-coherency case. Tests: YAML validated locally; dispatched on the branch (the run is linked from the PR). The smoke script is unchanged. --- .github/workflows/cluster-smoke.yml | 123 +++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cluster-smoke.yml b/.github/workflows/cluster-smoke.yml index 14d161e8ea..dce7dcbd9c 100644 --- a/.github/workflows/cluster-smoke.yml +++ b/.github/workflows/cluster-smoke.yml @@ -7,11 +7,12 @@ name: HFS Cluster Smoke # (discussion #223). It lands on main ahead of the feature phases because a # workflow is only dispatchable once it exists on the default branch; dispatch # it with ref= to run the branch's version of this file and of -# the smoke script. The skeleton asserts only behavior that is already +# the smoke script. The base checks assert behavior that is already # cluster-safe (shared-Postgres CRUD visibility A→B, round-robin through the -# front) — it calibrates the two-instance harness. Phase 1 adds the SoF -# $export A→B round-trip; Phase 3 adds the WebSocket fan-out A→B case; the -# nightly kill-9 recovery cases (A1/E1) get a schedule trigger when they land. +# front, SIGTERM drain on both instances) — they calibrate the two-instance +# harness. Later cluster PRs add the SoF $sql-export A→B round-trip, the +# WebSocket fan-out A→B case, and the HTS cache-coherency case; the nightly +# tier (kill -9 recovery for the durable job paths) gates on NIGHTLY below. # # Cloned from bulk-export-smoke.yml: same self-hosted runners, remote Docker # host (containers are reached at $DOCKER_HOST_IP, the hfs binaries run on the @@ -19,9 +20,21 @@ name: HFS Cluster Smoke on: workflow_dispatch: + inputs: + nightly: + description: "Also run the nightly tier (kill -9 recovery cases, once they land)" + type: boolean + default: false + schedule: + # Nightly tier: the slow, timing-sensitive recovery cases run here, not + # on every dispatch. Scheduled runs execute main's copy of this file. + - cron: "0 6 * * *" env: CARGO_TERM_COLOR: always + # True on the nightly schedule or a dispatch with nightly=true; the + # recovery steps gate on it. Empty otherwise. + NIGHTLY: ${{ github.event_name == 'schedule' || inputs.nightly == true }} CARGO_BUILD_JOBS: 1 CARGO_PROFILE_DEV_DEBUG: 0 DOCKER_HOST: ${{ secrets.DOCKER_HOST }} @@ -49,9 +62,12 @@ jobs: echo 'rustflags = ["-C", "link-arg=-fuse-ld=lld", "-C", "link-arg=-Wl,-zstack-size=8388608"]' >> ~/.cargo/config.toml - name: Build HFS binary + # subscriptions and elasticsearch are built in now so the later + # cluster cases (WebSocket fan-out, composite-sync durability) do not + # have to touch this job; the R4/postgres smoke below ignores them. run: | cargo build -p helios-hfs --no-default-features \ - --features R4,postgres + --features R4,postgres,subscriptions,elasticsearch - name: Upload HFS binary uses: actions/upload-artifact@v7 @@ -60,6 +76,41 @@ jobs: path: target/debug/hfs retention-days: 1 + build-hts: + name: Build HTS for cluster smoke + runs-on: [self-hosted, Linux] + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + - name: Configure Rust to use LLD + run: | + mkdir -p ~/.cargo + rm -f ~/.cargo/config.toml + echo '[target.x86_64-unknown-linux-gnu]' >> ~/.cargo/config.toml + echo 'linker = "clang"' >> ~/.cargo/config.toml + echo 'rustflags = ["-C", "link-arg=-fuse-ld=lld", "-C", "link-arg=-Wl,-zstack-size=8388608"]' >> ~/.cargo/config.toml + + - name: Build HTS binary + # Consumed by the terminology cache-coherency case (two hts processes + # on the shared Postgres) once it lands; built here so that case only + # adds a job that downloads this artifact. + run: | + cargo build -p helios-hts --no-default-features \ + --features R4,postgres + + - name: Upload HTS binary + uses: actions/upload-artifact@v7 + with: + name: hts-cluster-smoke-binary + path: target/debug/hts + retention-days: 1 + cluster-smoke: name: Cluster smoke (R4 / postgres) needs: build @@ -142,11 +193,20 @@ jobs: '' close; } upstream hfs_cluster { - server $RUNNER_IP:$HFS_PORT_A; - server $RUNNER_IP:$HFS_PORT_B; + # max_fails=0: never blacklist an upstream. The default + # (max_fails=1 fail_timeout=10s) turns one refused connect into + # a 10s "no live upstreams" window — a probe that races instance + # startup would poison every request that follows it. + server $RUNNER_IP:$HFS_PORT_A max_fails=0; + server $RUNNER_IP:$HFS_PORT_B max_fails=0; } server { listen 80; + # Answered by nginx itself — readiness probes must not go + # through the upstreams, which may not be started yet. + location = /nginx-health { + return 200 "ok\n"; + } location / { proxy_pass http://hfs_cluster; proxy_http_version 1.1; @@ -166,12 +226,12 @@ jobs: docker start "$NGINX_CONTAINER" echo "NGINX_CONTAINER=$NGINX_CONTAINER" >> "$GITHUB_ENV" - # hfs is not up yet, so accept any HTTP answer (502 included) as - # proof that nginx itself is serving. + # Probe the nginx-local endpoint: hfs is not up yet, and a proxied + # request would count as an upstream failure. for i in {1..30}; do NGINX_PORT=$(docker port "$NGINX_CONTAINER" 80 2>/dev/null | head -1 | sed 's/.*://') if [ -n "$NGINX_PORT" ] \ - && curl -s -o /dev/null "http://$DOCKER_HOST_IP:$NGINX_PORT/health"; then + && curl -sf -o /dev/null "http://$DOCKER_HOST_IP:$NGINX_PORT/nginx-health"; then echo "NGINX_PORT=$NGINX_PORT" >> "$GITHUB_ENV" echo "FRONT_URL=http://$DOCKER_HOST_IP:$NGINX_PORT" >> "$GITHUB_ENV" echo "nginx front is ready on port $NGINX_PORT" @@ -250,22 +310,41 @@ jobs: SMOKE_RUN_SUFFIX="${{ github.run_id }}-${{ github.run_attempt }}" \ ./crates/hfs/tests/cluster/run_external_cluster_smoke.sh - - name: Stop HFS instances gracefully + - name: Stop HFS instances gracefully (SIGTERM) if: always() + # SIGTERM is what a rolling deploy sends. Each instance must log the + # drain and exit on its own within the window; needing SIGKILL is a + # failure of this step (the process is still killed so nothing leaks). run: | - for pid in "${HFS_PID_A:-}" "${HFS_PID_B:-}"; do - if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then - kill -INT "$pid" 2>/dev/null || true - for _ in {1..50}; do - if kill -0 "$pid" 2>/dev/null; then - sleep 0.2 - else - break - fi - done + failed=0 + stop_one() { + local name="$1" pid="$2" log="$3" + if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then + echo "HFS instance $name is not running" + return 0 + fi + kill -TERM "$pid" 2>/dev/null || true + for _ in {1..50}; do + if kill -0 "$pid" 2>/dev/null; then + sleep 0.2 + else + break + fi + done + if kill -0 "$pid" 2>/dev/null; then + echo "::error::HFS instance $name did not exit within 10s of SIGTERM" kill -9 "$pid" 2>/dev/null || true + return 1 fi - done + if ! grep -q 'Shutdown signal received' "$log"; then + echo "::error::HFS instance $name exited on SIGTERM without logging the drain" + return 1 + fi + echo "HFS instance $name drained on SIGTERM" + } + stop_one A "${HFS_PID_A:-}" "${HFS_LOG_A:-/dev/null}" || failed=1 + stop_one B "${HFS_PID_B:-}" "${HFS_LOG_B:-/dev/null}" || failed=1 + exit "$failed" - name: Collect container logs if: always()