fix: wait through transient argo degradation

tests/unit/test_clean_room_automation_contract.sh

@@ -190,20 +190,26 @@ def main() -> None:
         raise SystemExit("Missing Argo health must not fail immediately")
     if re.search(r'sync in \{[^}]*Unknown', lib):
         raise SystemExit("Unknown Argo sync must not fail immediately")
-    if 'health == "Degraded"' not in lib:
-        raise SystemExit("Degraded Argo health must remain an immediate failure")
+    if 'health == "Degraded"' in lib:
+        raise SystemExit("Degraded Argo health must wait, not fail immediately")
+    if "sys.exit(3)" in lib:
+        raise SystemExit("Argo evaluator must not keep a fail-fast exit 3")
     if "JSONDecodeError" not in lib:
         raise SystemExit("Argo evaluator must fail closed on malformed JSON")
     for name, text in (("deploy", deploy), ("verify", verify)):
         if "clean_room_eval_argo_json" not in text:
             raise SystemExit(f"{name} must evaluate Argo Applications via the shared helper")
         if "not yet Synced+Healthy" not in text:
             raise SystemExit(f"{name} must retry when the Argo evaluator returns WAIT")
-        if "empty or unhealthy" not in text:
-            raise SystemExit(f"{name} must fail immediately on empty/unhealthy Argo sets")
+        if "invalid or empty" not in text:
+            raise SystemExit(f"{name} must fail immediately on invalid/empty Argo evaluation")
+        if "empty or unhealthy" in text:
+            raise SystemExit(f"{name} must not treat valid unhealthy Applications as immediate failure")
+        if "2|3" in text:
+            raise SystemExit(f"{name} must not treat evaluator exit 3 as a live fail-fast class")
         if "before timeout" not in text:
             raise SystemExit(f"{name} must keep the bounded Argo wait timeout")
-    print("PASS: Argo evaluator distinguishes PASS/WAIT/FAIL-FAST")
+    print("PASS: Argo evaluator distinguishes PASS/WAIT/INVALID")
 
     synthetic_forbidden = (
         "BEGIN PRIVATE KEY",
@@ -567,7 +573,40 @@ root = Path(sys.argv[1])
     ),
     encoding="utf-8",
 )
-(root / "argo-mixed-fail.json").write_text(
+(root / "argo-outofsync-degraded.json").write_text(
+    json.dumps(
+        {
+            "items": [
+                {
+                    "metadata": {"name": "monitoring"},
+                    "status": {
+                        "sync": {"status": "OutOfSync"},
+                        "health": {"status": "Degraded"},
+                    },
+                }
+            ]
+        }
+    ),
+    encoding="utf-8",
+)
+(root / "argo-mixed-degraded.json").write_text(
+    json.dumps(
+        {
+            "items": [
+                {
+                    "metadata": {"name": "root"},
+                    "status": {"sync": {"status": "Synced"}, "health": {"status": "Healthy"}},
+                },
+                {
+                    "metadata": {"name": "postgres"},
+                    "status": {"sync": {"status": "Synced"}, "health": {"status": "Degraded"}},
+                },
+            ]
+        }
+    ),
+    encoding="utf-8",
+)
+(root / "argo-missing-degraded.json").write_text(
     json.dumps(
         {
             "items": [
@@ -587,6 +626,8 @@ root = Path(sys.argv[1])
     ),
     encoding="utf-8",
 )
+(root / "argo-items-not-list.json").write_text(json.dumps({"items": {"name": "root"}}), encoding="utf-8")
+(root / "argo-non-object.json").write_text(json.dumps({"items": ["not-an-object"]}), encoding="utf-8")
 (root / "argo-malformed.json").write_text("{not-json\n", encoding="utf-8")
 (root / "missing-recap.log").write_text("ok: all tasks completed\n", encoding="utf-8")
 (root / "malformed-recap.log").write_text(
@@ -704,42 +745,66 @@ fi
 set +e
 clean_room_eval_argo_json "${HELPER_DIR}/argo-empty.json" >/dev/null
 empty_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-items-not-list.json" >/dev/null
+items_not_list_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-non-object.json" >/dev/null
+non_object_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-malformed.json" >/dev/null
+malformed_argo_rc=$?
 clean_room_eval_argo_json "${HELPER_DIR}/argo-degraded.json" >/dev/null
 degraded_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-outofsync-degraded.json" >/dev/null
+outofsync_degraded_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-pending.json" >/dev/null
+pending_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-missing.json" >/dev/null
+missing_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-synced-progressing.json" >/dev/null
+synced_progressing_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-unknown.json" >/dev/null
+unknown_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-wait.json" >/dev/null
+mixed_wait_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-degraded.json" >/dev/null
+mixed_degraded_rc=$?
+clean_room_eval_argo_json "${HELPER_DIR}/argo-missing-degraded.json" >/dev/null
+missing_degraded_rc=$?
 set -e
 if [[ "$empty_rc" -eq 2 ]]; then
   pass "empty Application set fails"
 else
   fail "empty Application set fails (rc=${empty_rc})"
 fi
-if [[ "$degraded_rc" -eq 3 ]]; then
-  pass "Degraded Application fails immediately"
+if [[ "$items_not_list_rc" -eq 2 ]]; then
+  pass "items not a list fails closed"
 else
-  fail "Degraded Application fails immediately (rc=${degraded_rc})"
+  fail "items not a list fails closed (rc=${items_not_list_rc})"
+fi
+if [[ "$non_object_rc" -eq 2 ]]; then
+  pass "non-object Application item fails closed"
+else
+  fail "non-object Application item fails closed (rc=${non_object_rc})"
+fi
+if [[ "$malformed_argo_rc" -eq 2 ]]; then
+  pass "malformed Argo JSON fails closed"
+else
+  fail "malformed Argo JSON fails closed (rc=${malformed_argo_rc})"
+fi
+if [[ "$degraded_rc" -eq 1 ]]; then
+  pass "Synced/Degraded Applications wait"
+else
+  fail "Synced/Degraded Applications wait (rc=${degraded_rc})"
+fi
+if [[ "$outofsync_degraded_rc" -eq 1 ]]; then
+  pass "OutOfSync/Degraded Applications wait"
+else
+  fail "OutOfSync/Degraded Applications wait (rc=${outofsync_degraded_rc})"
 fi
-set +e
-clean_room_eval_argo_json "${HELPER_DIR}/argo-pending.json" >/dev/null
-pending_rc=$?
-set -e
 if [[ "$pending_rc" -eq 1 ]]; then
   pass "non-converged Applications wait rather than pass"
 else
   fail "non-converged Applications wait rather than pass (rc=${pending_rc})"
 fi
-set +e
-clean_room_eval_argo_json "${HELPER_DIR}/argo-missing.json" >/dev/null
-missing_rc=$?
-clean_room_eval_argo_json "${HELPER_DIR}/argo-synced-progressing.json" >/dev/null
-synced_progressing_rc=$?
-clean_room_eval_argo_json "${HELPER_DIR}/argo-unknown.json" >/dev/null
-unknown_rc=$?
-clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-wait.json" >/dev/null
-mixed_wait_rc=$?
-clean_room_eval_argo_json "${HELPER_DIR}/argo-mixed-fail.json" >/dev/null
-mixed_fail_rc=$?
-clean_room_eval_argo_json "${HELPER_DIR}/argo-malformed.json" >/dev/null
-malformed_argo_rc=$?
-set -e
 if [[ "$missing_rc" -eq 1 ]]; then
   pass "OutOfSync/Missing Applications wait"
 else
@@ -760,15 +825,39 @@ if [[ "$mixed_wait_rc" -eq 1 ]]; then
 else
   fail "Healthy plus transient Applications wait (rc=${mixed_wait_rc})"
 fi
-if [[ "$mixed_fail_rc" -eq 3 ]]; then
-  pass "transient plus Degraded Applications fail immediately"
-else
-  fail "transient plus Degraded Applications fail immediately (rc=${mixed_fail_rc})"
-fi
-if [[ "$malformed_argo_rc" -eq 2 ]]; then
-  pass "malformed Argo JSON fails closed"
-else
-  fail "malformed Argo JSON fails closed (rc=${malformed_argo_rc})"
+if [[ "$mixed_degraded_rc" -eq 1 ]]; then
+  pass "Healthy plus Degraded Applications wait"
+else
+  fail "Healthy plus Degraded Applications wait (rc=${mixed_degraded_rc})"
+fi
+if [[ "$missing_degraded_rc" -eq 1 ]]; then
+  pass "Missing plus Degraded Applications wait"
+else
+  fail "Missing plus Degraded Applications wait (rc=${missing_degraded_rc})"
+fi
+non_final_passed=0
+for wait_fixture in \
+  argo-degraded.json \
+  argo-outofsync-degraded.json \
+  argo-pending.json \
+  argo-missing.json \
+  argo-synced-progressing.json \
+  argo-unknown.json \
+  argo-mixed-wait.json \
+  argo-mixed-degraded.json \
+  argo-missing-degraded.json
+do
+  set +e
+  clean_room_eval_argo_json "${HELPER_DIR}/${wait_fixture}" >/dev/null
+  wait_pass_rc=$?
+  set -e
+  if [[ "$wait_pass_rc" -eq 0 ]]; then
+    fail "non-final Application fixture must not PASS (${wait_fixture})"
+    non_final_passed=1
+  fi
+done
+if [[ "$non_final_passed" -eq 0 ]]; then
+  pass "no valid non-final Application combination may PASS"
 fi
 
 write_playbook_stub() {
@@ -1657,35 +1746,41 @@ set +e
 out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)"
 rc=$?
 set -e
-if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Eq 'empty or unhealthy|no Argo Applications'; then
+if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Eq 'invalid or empty|no Argo Applications'; then
   pass "empty Application set fails deploy wait"
 else
   fail "empty Application set fails deploy wait (rc=${rc})"
   printf '%s\n' "$out"
 fi
+if printf '%s' "$out" | grep -Fq "empty or unhealthy"; then
+  fail "empty Application set must not use the stale unhealthy fail-fast message"
+else
+  pass "empty Application set must not use the stale unhealthy fail-fast message"
+fi
 
 read -r fx home stubs <<<"$(prepare_runtime argodeg)"
-cp "${HELPER_DIR}/argo-degraded.json" "${home}/argo.json"
+cp "${HELPER_DIR}/argo-outofsync-degraded.json" "${home}/argo.json"
 cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json"
 set +e
 out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)"
 rc=$?
 set -e
-if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "empty or unhealthy"; then
-  pass "Degraded Applications fail deploy wait immediately"
+if printf '%s' "$out" | grep -Eq 'empty or unhealthy|invalid or empty'; then
+  fail "Degraded Applications must not fail immediately as invalid/unhealthy"
 else
-  fail "Degraded Applications fail deploy wait immediately (rc=${rc})"
-  printf '%s\n' "$out"
+  pass "Degraded Applications must not fail immediately as invalid/unhealthy"
 fi
-if printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then
-  fail "Degraded Applications must not enter the Argo wait loop"
+if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then
+  pass "Degraded Applications enter the bounded Argo wait loop"
 else
-  pass "Degraded Applications must not enter the Argo wait loop"
+  fail "Degraded Applications enter the bounded Argo wait loop (rc=${rc})"
+  printf '%s\n' "$out"
 fi
-if printf '%s' "$out" | grep -Fq "before timeout"; then
-  fail "Degraded Applications must not wait until timeout"
+if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "before timeout"; then
+  pass "persistent Degraded Applications fail via timeout"
 else
-  pass "Degraded Applications must not wait until timeout"
+  fail "persistent Degraded Applications fail via timeout (rc=${rc})"
+  printf '%s\n' "$out"
 fi
 
 read -r fx home stubs <<<"$(prepare_runtime argomiss)"
@@ -1701,10 +1796,10 @@ else
   fail "OutOfSync/Missing waits then times out (rc=${rc})"
   printf '%s\n' "$out"
 fi
-if printf '%s' "$out" | grep -Fq "empty or unhealthy"; then
-  fail "OutOfSync/Missing must not fail immediately as unhealthy"
+if printf '%s' "$out" | grep -Eq 'empty or unhealthy|invalid or empty'; then
+  fail "OutOfSync/Missing must not fail immediately as invalid/unhealthy"
 else
-  pass "OutOfSync/Missing must not fail immediately as unhealthy"
+  pass "OutOfSync/Missing must not fail immediately as invalid/unhealthy"
 fi
 
 read -r fx home stubs <<<"$(prepare_runtime argomixw)"
@@ -1722,22 +1817,22 @@ else
 fi
 
 read -r fx home stubs <<<"$(prepare_runtime argomixf)"
-cp "${HELPER_DIR}/argo-mixed-fail.json" "${home}/argo.json"
+cp "${HELPER_DIR}/argo-missing-degraded.json" "${home}/argo.json"
 cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json"
 set +e
 out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)"
 rc=$?
 set -e
-if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "empty or unhealthy"; then
-  pass "mixed Missing plus Degraded fails immediately"
+if printf '%s' "$out" | grep -Eq 'empty or unhealthy|invalid or empty'; then
+  fail "mixed Missing plus Degraded must not fail immediately as invalid/unhealthy"
 else
-  fail "mixed Missing plus Degraded fails immediately (rc=${rc})"
-  printf '%s\n' "$out"
+  pass "mixed Missing plus Degraded must not fail immediately as invalid/unhealthy"
 fi
-if printf '%s' "$out" | grep -Fq "not yet Synced+Healthy"; then
-  fail "mixed Missing plus Degraded must not wait"
+if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not yet Synced+Healthy" && printf '%s' "$out" | grep -Fq "before timeout"; then
+  pass "mixed Missing plus Degraded waits then times out"
 else
-  pass "mixed Missing plus Degraded must not wait"
+  fail "mixed Missing plus Degraded waits then times out (rc=${rc})"
+  printf '%s\n' "$out"
 fi
 
 read -r fx home stubs <<<"$(prepare_runtime argotime)"

tools/deploy-clean-room

@@ -151,9 +151,9 @@ wait_for_argo() {
       1)
         clean_room_info "Argo Applications not yet Synced+Healthy (${attempt}/${ARGO_ATTEMPTS})"
         ;;
-      2|3)
+      2)
         rm -f "$tmp"
-        clean_room_die "Argo Application set is empty or unhealthy"
+        clean_room_die "Argo Application evaluation is invalid or empty"
         ;;
       *)
         rm -f "$tmp"

tools/lib/clean-room-common.sh

@@ -220,13 +220,14 @@ clean_room_eval_argo_json() {
 import json
 import sys
 
-# PASS: every Application is Synced + Healthy.
-# WAIT: incomplete first-reconcile states, including the live-proven
-# OutOfSync/Missing pair. Unknown means status is not yet assessed, not
-# that the Application is Healthy or terminally Degraded; the bounded
-# waiter still times out if it never converges.
-# FAIL-FAST (3): Health Degraded — resources were assessed as unhealthy.
-# FAIL (2): empty set or JSON that cannot be evaluated.
+# PASS (0): every Application is Synced + Healthy.
+# WAIT (1): structurally evaluable set that is not yet entirely
+# Synced + Healthy. Valid non-final statuses include OutOfSync,
+# Missing, Progressing, Degraded, Unknown, Suspended, and unset.
+# Live Greenfield reconciliation has shown Missing and Degraded to be
+# transient. The bounded waiter is the safety boundary; no non-final
+# state can PASS.
+# FAIL (2): malformed JSON, empty set, or a shape that cannot be evaluated.
 
 try:
     with open(sys.argv[1], encoding="utf-8") as handle:
@@ -236,12 +237,17 @@ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
     sys.exit(2)
 
 try:
+    if not isinstance(data, dict):
+        print("FAIL: Argo Application JSON is not an object")
+        sys.exit(2)
     items = data.get("items")
-    if not isinstance(items, list) or not items:
+    if not isinstance(items, list):
+        print("FAIL: Argo Application items is not a list")
+        sys.exit(2)
+    if not items:
         print("FAIL: no Argo Applications found")
         sys.exit(2)
 
-    immediate = []
     pending = []
     for item in items:
         if not isinstance(item, dict):
@@ -253,15 +259,9 @@ try:
             status = {}
         sync = ((status.get("sync") or {}).get("status")) or ""
         health = ((status.get("health") or {}).get("status")) or ""
-        if health == "Degraded":
-            immediate.append(f"{name} sync={sync or 'unset'} health={health}")
-            continue
         if sync != "Synced" or health != "Healthy":
             pending.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}")
 
-    if immediate:
-        print("FAIL: " + "; ".join(immediate))
-        sys.exit(3)
     if pending:
         print("WAIT: " + "; ".join(pending))
         sys.exit(1)

tools/verify-clean-room

@@ -188,8 +188,8 @@ wait_for_argo() {
       1)
         clean_room_info "Argo Applications not yet Synced+Healthy (${attempt}/${ARGO_ATTEMPTS})"
         ;;
-      2|3)
-        clean_room_die "Argo Application set is empty or unhealthy"
+      2)
+        clean_room_die "Argo Application evaluation is invalid or empty"
         ;;
       *)
         clean_room_info "waiting for Argo Applications (${attempt}/${ARGO_ATTEMPTS})"