From 4931faf4a693ed780449976284797cada423af34 Mon Sep 17 00:00:00 2001 From: bxvtr <148579658+bxvtr@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:13:51 +0200 Subject: [PATCH 1/3] fix: handle transient clean-room convergence states Stream Ansible output live after FORMAT and wait on first-reconcile Argo OutOfSync/Missing instead of failing immediately. --- .../test_clean_room_automation_contract.sh | 304 +++++++++++++++++- tools/deploy-clean-room | 10 +- tools/lib/clean-room-common.sh | 89 +++-- tools/verify-clean-room | 10 +- 4 files changed, 368 insertions(+), 45 deletions(-) diff --git a/tests/unit/test_clean_room_automation_contract.sh b/tests/unit/test_clean_room_automation_contract.sh index a92bc4f..478d736 100755 --- a/tests/unit/test_clean_room_automation_contract.sh +++ b/tests/unit/test_clean_room_automation_contract.sh @@ -176,6 +176,35 @@ def main() -> None: raise SystemExit("deploy must use file-based private-runtime extra-vars") print("PASS: private-runtime extra-vars are file-based") + if "clean_room_run_playbook" not in lib: + raise SystemExit("shared live playbook helper is missing") + if "tee" not in lib or "PIPESTATUS[0]" not in lib: + raise SystemExit("run_playbook must tee live output and preserve PIPESTATUS[0]") + if re.search(r'>\s*"\$log"\s+2>&1', deploy) or re.search(r'>\s*"\$log"\s+2>&1', verify): + raise SystemExit("operator tools must not buffer ansible-playbook output until exit") + if "clean_room_run_playbook" not in deploy or "clean_room_run_playbook" not in verify: + raise SystemExit("deploy and verify must use the shared live playbook helper") + print("PASS: ansible-playbook output is live-teed with preserved exit code") + + if re.search(r'health in \{[^}]*Missing', lib): + raise SystemExit("Missing Argo health must not fail immediately") + if re.search(r'sync in \{[^}]*Unknown', lib): + raise SystemExit("Unknown Argo sync must not fail immediately") + if 'health == "Degraded"' not in lib: + raise SystemExit("Degraded Argo health must remain an immediate failure") + if "JSONDecodeError" not in lib: + raise SystemExit("Argo evaluator must fail closed on malformed JSON") + for name, text in (("deploy", deploy), ("verify", verify)): + if "clean_room_eval_argo_json" not in text: + raise SystemExit(f"{name} must evaluate Argo Applications via the shared helper") + if "not yet Synced+Healthy" not in text: + raise SystemExit(f"{name} must retry when the Argo evaluator returns WAIT") + if "empty or unhealthy" not in text: + raise SystemExit(f"{name} must fail immediately on empty/unhealthy Argo sets") + if "before timeout" not in text: + raise SystemExit(f"{name} must keep the bounded Argo wait timeout") + print("PASS: Argo evaluator distinguishes PASS/WAIT/FAIL-FAST") + synthetic_forbidden = ( "BEGIN PRIVATE KEY", "BEGIN RSA PRIVATE KEY", @@ -470,6 +499,95 @@ root = Path(sys.argv[1]) ), encoding="utf-8", ) +(root / "argo-missing.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "monitoring"}, + "status": { + "sync": {"status": "OutOfSync"}, + "health": {"status": "Missing"}, + }, + } + ] + } + ), + encoding="utf-8", +) +(root / "argo-synced-progressing.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "root"}, + "status": { + "sync": {"status": "Synced"}, + "health": {"status": "Progressing"}, + }, + } + ] + } + ), + encoding="utf-8", +) +(root / "argo-unknown.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "mlflow"}, + "status": { + "sync": {"status": "Synced"}, + "health": {"status": "Unknown"}, + }, + } + ] + } + ), + encoding="utf-8", +) +(root / "argo-mixed-wait.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "root"}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Healthy"}}, + }, + { + "metadata": {"name": "monitoring"}, + "status": { + "sync": {"status": "OutOfSync"}, + "health": {"status": "Missing"}, + }, + }, + ] + } + ), + encoding="utf-8", +) +(root / "argo-mixed-fail.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "monitoring"}, + "status": { + "sync": {"status": "OutOfSync"}, + "health": {"status": "Missing"}, + }, + }, + { + "metadata": {"name": "postgres"}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Degraded"}}, + }, + ] + } + ), + encoding="utf-8", +) +(root / "argo-malformed.json").write_text("{not-json\n", encoding="utf-8") (root / "missing-recap.log").write_text("ok: all tasks completed\n", encoding="utf-8") (root / "malformed-recap.log").write_text( "PLAY RECAP *********************************************************************\n" @@ -608,6 +726,124 @@ if [[ "$pending_rc" -eq 1 ]]; then else fail "non-converged Applications wait rather than pass (rc=${pending_rc})" fi +set +e +clean_room_eval_argo_json "${HELPER_DIR}/argo-missing.json" >/dev/null +missing_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-synced-progressing.json" >/dev/null +synced_progressing_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-unknown.json" >/dev/null +unknown_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-wait.json" >/dev/null +mixed_wait_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-fail.json" >/dev/null +mixed_fail_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-malformed.json" >/dev/null +malformed_argo_rc=$? +set -e +if [[ "$missing_rc" -eq 1 ]]; then + pass "OutOfSync/Missing Applications wait" +else + fail "OutOfSync/Missing Applications wait (rc=${missing_rc})" +fi +if [[ "$synced_progressing_rc" -eq 1 ]]; then + pass "Synced/Progressing Applications wait" +else + fail "Synced/Progressing Applications wait (rc=${synced_progressing_rc})" +fi +if [[ "$unknown_rc" -eq 1 ]]; then + pass "Unknown Argo health waits within the bounded timeout" +else + fail "Unknown Argo health waits within the bounded timeout (rc=${unknown_rc})" +fi +if [[ "$mixed_wait_rc" -eq 1 ]]; then + pass "Healthy plus transient Applications wait" +else + fail "Healthy plus transient Applications wait (rc=${mixed_wait_rc})" +fi +if [[ "$mixed_fail_rc" -eq 3 ]]; then + pass "transient plus Degraded Applications fail immediately" +else + fail "transient plus Degraded Applications fail immediately (rc=${mixed_fail_rc})" +fi +if [[ "$malformed_argo_rc" -eq 2 ]]; then + pass "malformed Argo JSON fails closed" +else + fail "malformed Argo JSON fails closed (rc=${malformed_argo_rc})" +fi + +write_playbook_stub() { + local dest="$1" + local exit_code="$2" + cat >"$dest" <&2 +printf '%s\n' "The scratch volume has no filesystem. Set scratch_storage_allow_format=true only after verifying that this is the intended Terraform-managed scratch volume." +cat <<'REC' +PLAY RECAP ********************************************************************* +reference-node : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +REC +exit ${exit_code} +STUB + chmod +x "$dest" +} + +PB_STUBS="${TMP_ROOT}/playbook-stubs" +mkdir -p "$PB_STUBS" +write_playbook_stub "${PB_STUBS}/ansible-ok" 0 +write_playbook_stub "${PB_STUBS}/ansible-fail" 7 +ANSIBLE_CONFIG_FILE="/dev/null" +PB_OK_LOG="${TMP_ROOT}/playbook-ok.log" +PB_FAIL_LOG="${TMP_ROOT}/playbook-fail.log" +ANSIBLE_PLAYBOOK="${PB_STUBS}/ansible-ok" +set +e +pb_ok_out="$(clean_room_run_playbook "$PB_OK_LOG" site.yml 2>&1)" +pb_ok_rc=$? +set -e +if [[ "$pb_ok_rc" -eq 0 ]]; then + pass "successful run_playbook returns 0" +else + fail "successful run_playbook returns 0 (rc=${pb_ok_rc})" +fi +assert_contains "successful run_playbook is operator-visible" "LIVE-STDOUT" "$pb_ok_out" +assert_contains "successful run_playbook stderr is operator-visible" "LIVE-STDERR" "$pb_ok_out" +if grep -Fq "LIVE-STDOUT" "$PB_OK_LOG" && grep -Fq "LIVE-STDERR" "$PB_OK_LOG"; then + pass "successful run_playbook retains a complete log" +else + fail "successful run_playbook retains a complete log" +fi +ANSIBLE_PLAYBOOK="${PB_STUBS}/ansible-fail" +set +e +pb_fail_out="$(clean_room_run_playbook "$PB_FAIL_LOG" site.yml 2>&1)" +pb_fail_rc=$? +set -e +if [[ "$pb_fail_rc" -eq 7 ]]; then + pass "failed run_playbook returns the ansible-playbook exit code" +else + fail "failed run_playbook returns the ansible-playbook exit code (rc=${pb_fail_rc})" +fi +assert_contains "failed run_playbook is operator-visible" "LIVE-STDOUT" "$pb_fail_out" +if grep -Fq "LIVE-STDOUT" "$PB_FAIL_LOG"; then + pass "failed run_playbook retains a complete log" +else + fail "failed run_playbook retains a complete log" +fi +if [[ "$pb_fail_rc" -eq 0 ]]; then + fail "pipeline behavior cannot convert an Ansible failure into success" +else + pass "pipeline behavior cannot convert an Ansible failure into success" +fi +if clean_room_is_blank_scratch_gate "$PB_FAIL_LOG"; then + pass "blank-scratch gate remains detectable from the live-teed log" +else + fail "blank-scratch gate remains detectable from the live-teed log" +fi +if clean_room_parse_play_recap "$PB_FAIL_LOG" 1 >/dev/null; then + pass "PLAY RECAP parsing still works from the live-teed log" +else + fail "PLAY RECAP parsing still works from the live-teed log" +fi +unset ANSIBLE_PLAYBOOK ANSIBLE_CONFIG_FILE if clean_room_eval_pods_json "${HELPER_DIR}/pods-ok.json" >/dev/null; then pass "completed Jobs are treated as healthy" @@ -1431,13 +1667,75 @@ set +e out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" rc=$? set -e -if [[ "$rc" -ne 0 ]]; then - pass "Degraded Applications fail deploy wait" +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "empty or unhealthy"; then + pass "Degraded Applications fail deploy wait immediately" +else + fail "Degraded Applications fail deploy wait immediately (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then + fail "Degraded Applications must not enter the Argo wait loop" +else + pass "Degraded Applications must not enter the Argo wait loop" +fi +if printf '%s' "$out" | grep -Fq "before timeout"; then + fail "Degraded Applications must not wait until timeout" +else + pass "Degraded Applications must not wait until timeout" +fi + +read -r fx home stubs <<<"$(prepare_runtime argomiss)" +cp "${HELPER_DIR}/argo-missing.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not yet Synced+Healthy" && printf '%s' "$out" | grep -Fq "before timeout"; then + pass "OutOfSync/Missing waits then times out" +else + fail "OutOfSync/Missing waits then times out (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "empty or unhealthy"; then + fail "OutOfSync/Missing must not fail immediately as unhealthy" +else + pass "OutOfSync/Missing must not fail immediately as unhealthy" +fi + +read -r fx home stubs <<<"$(prepare_runtime argomixw)" +cp "${HELPER_DIR}/argo-mixed-wait.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then + pass "mixed Healthy plus Missing waits" else - fail "Degraded Applications fail deploy wait" + fail "mixed Healthy plus Missing waits (rc=${rc})" printf '%s\n' "$out" fi +read -r fx home stubs <<<"$(prepare_runtime argomixf)" +cp "${HELPER_DIR}/argo-mixed-fail.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "empty or unhealthy"; then + pass "mixed Missing plus Degraded fails immediately" +else + fail "mixed Missing plus Degraded fails immediately (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then + fail "mixed Missing plus Degraded must not wait" +else + pass "mixed Missing plus Degraded must not wait" +fi + read -r fx home stubs <<<"$(prepare_runtime argotime)" cp "${HELPER_DIR}/argo-pending.json" "${home}/argo.json" cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" diff --git a/tools/deploy-clean-room b/tools/deploy-clean-room index 1eb8be3..822db1c 100755 --- a/tools/deploy-clean-room +++ b/tools/deploy-clean-room @@ -123,15 +123,7 @@ wait_for_ssh() { } run_playbook() { - local log="$1" - local rc=0 - shift - set +e - ANSIBLE_CONFIG="$ANSIBLE_CONFIG_FILE" "$ANSIBLE_PLAYBOOK" "$@" >"$log" 2>&1 - rc=$? - cat "$log" - set +e - return "$rc" + clean_room_run_playbook "$@" } wait_for_argo() { diff --git a/tools/lib/clean-room-common.sh b/tools/lib/clean-room-common.sh index 43c24bf..74a523c 100755 --- a/tools/lib/clean-room-common.sh +++ b/tools/lib/clean-room-common.sh @@ -196,6 +196,22 @@ sys.exit(0) PY } +# Stream ansible-playbook stdout/stderr live while retaining a complete log. +# Returns the ansible-playbook exit code (PIPESTATUS[0]), never tee's. +# Callers must wrap the invocation with set +e if they need to inspect a +# non-zero status under set -e. pipefail is restored before return. +clean_room_run_playbook() { + local log="$1" + local rc=0 + shift + set +e + set +o pipefail + ANSIBLE_CONFIG="${ANSIBLE_CONFIG_FILE:-}" "${ANSIBLE_PLAYBOOK:?ansible-playbook is not set}" "$@" 2>&1 | tee "$log" + rc="${PIPESTATUS[0]}" + set -o pipefail + return "$rc" +} + clean_room_eval_argo_json() { local json_file="$1" local py @@ -204,33 +220,58 @@ clean_room_eval_argo_json() { import json import sys -data = json.load(open(sys.argv[1], encoding="utf-8")) -items = data.get("items") -if not isinstance(items, list) or not items: - print("FAIL: no Argo Applications found") +# PASS: every Application is Synced + Healthy. +# WAIT: incomplete first-reconcile states, including the live-proven +# OutOfSync/Missing pair. Unknown means status is not yet assessed, not +# that the Application is Healthy or terminally Degraded; the bounded +# waiter still times out if it never converges. +# FAIL-FAST (3): Health Degraded — resources were assessed as unhealthy. +# FAIL (2): empty set or JSON that cannot be evaluated. + +try: + with open(sys.argv[1], encoding="utf-8") as handle: + data = json.load(handle) +except (OSError, UnicodeError, json.JSONDecodeError) as exc: + print(f"FAIL: Argo Application JSON is not evaluable ({type(exc).__name__})") sys.exit(2) -immediate = [] -pending = [] -for item in items: - name = ((item.get("metadata") or {}).get("name")) or "unnamed" - status = item.get("status") or {} - sync = ((status.get("sync") or {}).get("status")) or "" - health = ((status.get("health") or {}).get("status")) or "" - if health in {"Degraded", "Missing", "Unknown"} or sync in {"Unknown"}: - immediate.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}") - continue - if sync != "Synced" or health != "Healthy": - pending.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}") +try: + items = data.get("items") + if not isinstance(items, list) or not items: + print("FAIL: no Argo Applications found") + sys.exit(2) -if immediate: - print("FAIL: " + "; ".join(immediate)) - sys.exit(3) -if pending: - print("WAIT: " + "; ".join(pending)) - sys.exit(1) -print("PASS: all Argo Applications are Synced and Healthy") -sys.exit(0) + immediate = [] + pending = [] + for item in items: + if not isinstance(item, dict): + print("FAIL: Argo Application item is not an object") + sys.exit(2) + name = ((item.get("metadata") or {}).get("name")) or "unnamed" + status = item.get("status") or {} + if not isinstance(status, dict): + status = {} + sync = ((status.get("sync") or {}).get("status")) or "" + health = ((status.get("health") or {}).get("status")) or "" + if health == "Degraded": + immediate.append(f"{name} sync={sync or 'unset'} health={health}") + continue + if sync != "Synced" or health != "Healthy": + pending.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}") + + if immediate: + print("FAIL: " + "; ".join(immediate)) + sys.exit(3) + if pending: + print("WAIT: " + "; ".join(pending)) + sys.exit(1) + print("PASS: all Argo Applications are Synced and Healthy") + sys.exit(0) +except SystemExit: + raise +except Exception as exc: + print(f"FAIL: Argo Application evaluation failed ({type(exc).__name__})") + sys.exit(2) PY } diff --git a/tools/verify-clean-room b/tools/verify-clean-room index 70fbd7d..b2e04bf 100755 --- a/tools/verify-clean-room +++ b/tools/verify-clean-room @@ -131,15 +131,7 @@ read_boot_id() { } run_playbook() { - local log="$1" - local rc=0 - shift - set +e - ANSIBLE_CONFIG="$ANSIBLE_CONFIG_FILE" "$ANSIBLE_PLAYBOOK" "$@" >"$log" 2>&1 - rc=$? - cat "$log" - set +e - return "$rc" + clean_room_run_playbook "$@" } require_zero_change_playbook() { From b10f59d1bad90ac7242dae718b5e1db0f91c2619 Mon Sep 17 00:00:00 2001 From: bxvtr <148579658+bxvtr@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:24:05 +0200 Subject: [PATCH 2/3] fix: document shared ansible config usage --- tools/deploy-clean-room | 2 ++ tools/verify-clean-room | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/deploy-clean-room b/tools/deploy-clean-room index 822db1c..74ba455 100755 --- a/tools/deploy-clean-room +++ b/tools/deploy-clean-room @@ -76,6 +76,8 @@ while [[ $# -gt 0 ]]; do done TF_DIR="${ROOT}/terraform" +# Consumed by clean_room_run_playbook from the sourced shared helper. +# shellcheck disable=SC2034 ANSIBLE_CONFIG_FILE="${ROOT}/ansible/ansible.cfg" INVENTORY="${ROOT}/ansible/inventory/local.yml" PRIVATE_VARS="${ROOT}/ansible/extra-vars/private-runtime.yml" diff --git a/tools/verify-clean-room b/tools/verify-clean-room index b2e04bf..7eb40f9 100755 --- a/tools/verify-clean-room +++ b/tools/verify-clean-room @@ -79,6 +79,8 @@ while [[ $# -gt 0 ]]; do done TF_DIR="${ROOT}/terraform" +# Consumed by clean_room_run_playbook from the sourced shared helper. +# shellcheck disable=SC2034 ANSIBLE_CONFIG_FILE="${ROOT}/ansible/ansible.cfg" INVENTORY="${ROOT}/ansible/inventory/local.yml" PRIVATE_VARS="${ROOT}/ansible/extra-vars/private-runtime.yml" From d44bfeef6aa5d3e726cf3d584451331994303c60 Mon Sep 17 00:00:00 2001 From: bxvtr <148579658+bxvtr@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:02:29 +0200 Subject: [PATCH 3/3] fix: document shared test harness usage --- tests/unit/test_clean_room_automation_contract.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_clean_room_automation_contract.sh b/tests/unit/test_clean_room_automation_contract.sh index 478d736..a04408a 100755 --- a/tests/unit/test_clean_room_automation_contract.sh +++ b/tests/unit/test_clean_room_automation_contract.sh @@ -792,6 +792,8 @@ PB_STUBS="${TMP_ROOT}/playbook-stubs" mkdir -p "$PB_STUBS" write_playbook_stub "${PB_STUBS}/ansible-ok" 0 write_playbook_stub "${PB_STUBS}/ansible-fail" 7 +# Test harness assignment; consumed by clean_room_run_playbook from the sourced helper. +# shellcheck disable=SC2034 ANSIBLE_CONFIG_FILE="/dev/null" PB_OK_LOG="${TMP_ROOT}/playbook-ok.log" PB_FAIL_LOG="${TMP_ROOT}/playbook-fail.log" @@ -812,6 +814,8 @@ if grep -Fq "LIVE-STDOUT" "$PB_OK_LOG" && grep -Fq "LIVE-STDERR" "$PB_OK_LOG"; t else fail "successful run_playbook retains a complete log" fi +# Test harness assignment; consumed by clean_room_run_playbook from the sourced helper. +# shellcheck disable=SC2034 ANSIBLE_PLAYBOOK="${PB_STUBS}/ansible-fail" set +e pb_fail_out="$(clean_room_run_playbook "$PB_FAIL_LOG" site.yml 2>&1)"