fix: enforce the CPU-only automatic qualification boundary

.radiance/tests.yaml

@@ -46,9 +46,9 @@ change_map:
 - pattern: '^(radiance_r4d_attn|patch_r4d|patch_unified_attention_lds)'
   areas: [attention, libr4d]
 profiles:
-  pr: [control-plane-validate, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, qwen-tool-grammar-cpu, client-tool-gate-cpu]
-  upgrade: [control-plane-validate, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, parser-shared-engine-image, xgrammar-spec-image, qwen-open-object-image, qwen-tool-grammar-image, client-tool-gate-cpu, client-tool-live, fp8-kv-calibration-image, tunableop-preflight-image, r9700-tp2-smoke, tool-schema-live, open-object-tool-live, betterbench-standard, correctness-equivalence]
-  release: [control-plane-validate, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, parser-shared-engine-image, xgrammar-spec-image, qwen-open-object-image, qwen-tool-grammar-image, client-tool-gate-cpu, client-tool-live, fp8-kv-calibration-image, tunableop-preflight-image, r9700-tp2-smoke, tool-schema-live, open-object-tool-live, betterbench-standard, correctness-equivalence, capacity-qualification]
+  pr: [control-plane-validate, automatic-cpu-boundary, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, qwen-tool-grammar-cpu, client-tool-gate-cpu]
+  upgrade: [control-plane-validate, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, parser-shared-engine-image, xgrammar-spec-image, qwen-open-object-image, fp8-kv-calibration-image, tunableop-preflight-image, r9700-tp2-smoke, tool-schema-live, open-object-tool-live, betterbench-standard, correctness-equivalence, qwen-tool-grammar-image, client-tool-gate-cpu, client-tool-live]
+  release: [control-plane-validate, python-compile, kv-offload-registration-cpu, kv-offload-rank-sharded-cpu, kv-offload-restore-cpu, parser-shared-engine-image, xgrammar-spec-image, qwen-open-object-image, fp8-kv-calibration-image, tunableop-preflight-image, r9700-tp2-smoke, tool-schema-live, open-object-tool-live, betterbench-standard, correctness-equivalence, capacity-qualification, qwen-tool-grammar-image, client-tool-gate-cpu, client-tool-live]
 tests:
 - id: client-tool-gate-cpu
   class: contract
@@ -66,6 +66,14 @@ tests:
   automatic: false
   ci: false
   description: Synthetic QA keypress and nested Int64 package regressions, auto/non-strict and strict, streaming and non-streaming.
+- id: automatic-cpu-boundary
+  class: contract
+  hardware: cpu
+  areas: [control-plane]
+  command: python -m unittest discover -s tests -p test_automatic_cpu_boundary.py -v
+  automatic: true
+  ci: true
+  description: Reject malformed eligibility flags and hardware-dependent automatic entries before executing any command.
 - id: qwen-tool-grammar-cpu
   class: contract
   hardware: cpu

docs/maintenance/TESTING_POLICY.md

@@ -29,6 +29,11 @@ python tools/radiance.py qualify release
 `qualify` prints the entire qualification plan by default; `--run-auto` executes the safe automatic subset
 while leaving candidate-image, physical-GPU, and maintenance-window gates explicit.
 
+`automatic` and `ci` must be YAML booleans, not quoted strings or integers.
+Automatic entries must require `hardware: cpu`. Both registry validation and
+the CLI execution boundary reject violations; the complete selected plan is
+checked before any command starts.
+
 ## Evidence classes
 
 - **Static/unit/contract** — deterministic source or pure-Python invariants.

tests/test_automatic_cpu_boundary.py

@@ -0,0 +1,75 @@
+"""Automatic checks must never execute hardware/manual entries through truthiness."""
+
+import contextlib
+import io
+import subprocess
+import unittest
+from unittest.mock import patch
+
+from tools import radiance, validate_control_plane
+
+
+def entry(**overrides):
+    return {
+        "id": "fixture", "class": "contract", "hardware": "cpu", "areas": ["fixture"],
+        "command": "must not run a GPU probe", "automatic": True, "ci": False,
+        **overrides,
+    }
+
+
+class AutomaticCPUBoundaryTests(unittest.TestCase):
+    def run_plan(self, entries, dry_run=False):
+        with patch.object(radiance.subprocess, "run") as run:
+            run.return_value = subprocess.CompletedProcess([], 0)
+            with contextlib.redirect_stdout(io.StringIO()):
+                status = radiance.run_auto(entries, dry_run)
+            return status, run.call_count
+
+    def test_cpu_automatic_check_executes(self):
+        self.assertEqual(self.run_plan([entry()]), (0, 1))
+
+    def test_manual_hardware_entries_are_skipped(self):
+        self.assertEqual(self.run_plan([entry(hardware="r9700-tp2", automatic=False)]), (0, 0))
+
+    def test_non_boolean_flags_fail_without_execution(self):
+        for flag in ("automatic", "ci"):
+            for value in ("false", "true", 0, 1, None):
+                with self.subTest(flag=flag, value=value):
+                    status, calls = self.run_plan([entry(**{flag: value})])
+                    self.assertNotEqual(status, 0)
+                    self.assertEqual(calls, 0)
+
+    def test_non_cpu_automatic_entry_fails_without_execution(self):
+        for hardware in ("candidate-image", "generic-rocm", "r9700", "r9700-tp2",
+                         "maintenance-window-r9700-tp2"):
+            with self.subTest(hardware=hardware):
+                status, calls = self.run_plan([entry(hardware=hardware)])
+                self.assertNotEqual(status, 0)
+                self.assertEqual(calls, 0)
+
+    def test_complete_plan_is_validated_before_any_command(self):
+        status, calls = self.run_plan([entry(), entry(id="bad", hardware="r9700")])
+        self.assertNotEqual(status, 0)
+        self.assertEqual(calls, 0)
+
+    def test_valid_cpu_dry_run_does_not_execute(self):
+        self.assertEqual(self.run_plan([entry()], dry_run=True), (0, 0))
+
+    def test_registry_validator_rejects_non_boolean_flags(self):
+        for flag in ("automatic", "ci"):
+            for value in ("false", "true", 0, 1, None):
+                with self.subTest(flag=flag, value=value):
+                    errors = []
+                    doc = {"hardware_levels": ["cpu"], "tests": [entry(**{flag: value})]}
+                    validate_control_plane.check_tests(radiance.REPO, doc, errors)
+                    self.assertTrue(any(flag in error and "boolean" in error for error in errors), errors)
+
+    def test_registry_validator_rejects_automatic_hardware(self):
+        errors = []
+        doc = {"hardware_levels": ["cpu", "r9700"], "tests": [entry(hardware="r9700")]}
+        validate_control_plane.check_tests(radiance.REPO, doc, errors)
+        self.assertTrue(any("automatic" in error and "cpu" in error for error in errors), errors)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/radiance.py

@@ -85,8 +85,18 @@ def show_plan(items: list[dict[str, Any]]) -> None:
 
 
 def run_auto(items: list[dict[str, Any]], dry_run: bool) -> int:
+    # Validate the whole plan before launching even its first CPU command.
+    # In particular, quoted YAML "false" is truthy in Python.
     for item in items:
-        if not item.get("automatic"):
+        for flag in ("automatic", "ci"):
+            if not isinstance(item.get(flag, False), bool):
+                print(f"FAIL {item['id']}: {flag} must be a YAML boolean")
+                return 1
+        if item.get("automatic") is True and item.get("hardware") != "cpu":
+            print(f"FAIL {item['id']}: automatic checks must require cpu hardware")
+            return 1
+    for item in items:
+        if item.get("automatic") is not True:
             print(f"SKIP {item['id']}: requires {item['hardware']}")
             continue
         print(f"{'PLAN' if dry_run else 'RUN '} {item['id']}: {item['command']}")

tools/validate_control_plane.py

@@ -102,12 +102,17 @@ def check_tests(repo: Path, tests_doc: dict[str, Any], errors: list[str]) -> set
     for test_id, item in by_id.items():
         if item.get("hardware") not in hardware:
             errors.append(f"test {test_id}: unknown hardware level {item.get('hardware')!r}")
+        for flag in ("automatic", "ci"):
+            if not isinstance(item.get(flag, False), bool):
+                errors.append(f"test {test_id}: {flag} must be a YAML boolean")
+        if item.get("automatic") is True and item.get("hardware") != "cpu":
+            errors.append(f"test {test_id}: automatic checks must require cpu hardware")
         if not isinstance(item.get("command"), str) or not item["command"].strip():
             errors.append(f"test {test_id}: command must be non-empty")
         areas = item.get("areas")
         if not isinstance(areas, list) or not areas:
             errors.append(f"test {test_id}: areas must be a non-empty list")
-        if item.get("ci") and not item.get("automatic"):
+        if item.get("ci") is True and item.get("automatic") is not True:
             errors.append(f"test {test_id}: CI tests must also be automatic")
 
     for profile, ids in (tests_doc.get("profiles") or {}).items():