diff --git a/.github/workflows/repository-validation.yml b/.github/workflows/repository-validation.yml index c6be43e..f43e989 100644 --- a/.github/workflows/repository-validation.yml +++ b/.github/workflows/repository-validation.yml @@ -37,6 +37,10 @@ jobs: - name: Run agent safety unit tests run: ./tests/unit/test_agent_safety.sh + - name: Run clean-room automation contract tests + # Stubbed/static only. Does not contact OCI, SSH, Kubernetes, or download Terraform. + run: ./tests/unit/test_clean_room_automation_contract.sh + - name: Run safe repository validation run: ./tools/validate-safe @@ -46,10 +50,16 @@ jobs: files=() while IFS= read -r -d '' file; do files+=("$file") - done < <(git ls-files -z -- '*.sh') + done < <( + git ls-files -z -- \ + '*.sh' \ + tools/bootstrap-cloud-shell \ + tools/deploy-clean-room \ + tools/verify-clean-room + ) if [ "${#files[@]}" -eq 0 ]; then - echo "No tracked *.sh files found; ShellCheck step succeeds." + echo "No tracked shell files found; ShellCheck step succeeds." exit 0 fi @@ -290,6 +300,9 @@ jobs: "ansible/extra-vars/private-runtime.yml.example", "tools/render-ansible-inventory", "tools/check-cloud-shell-readiness", + "tools/bootstrap-cloud-shell", + "tools/deploy-clean-room", + "tools/verify-clean-room", ] for path in required: if not Path(path).is_file(): @@ -359,6 +372,19 @@ jobs: raise SystemExit("check-cloud-shell-readiness must not invoke mutating terraform commands") if "ansible-playbook -i" in preflight: raise SystemExit("check-cloud-shell-readiness must not invoke ansible-playbook against inventory") + if "python3.12" not in preflight: + raise SystemExit("check-cloud-shell-readiness must require python3.12") + + for tool_name in ( + "tools/bootstrap-cloud-shell", + "tools/deploy-clean-room", + "tools/verify-clean-room", + ): + tool_text = Path(tool_name).read_text(encoding="utf-8") + if re.search(r"(?m)^\s*tmux\s+(new|attach|kill)", tool_text): + raise SystemExit(f"{tool_name} must not start tmux") + if "-auto-approve" in tool_text: + raise SystemExit(f"{tool_name} must not use terraform -auto-approve") tfvars = Path("terraform/terraform.tfvars.example").read_text(encoding="utf-8") for field in ( diff --git a/CHANGELOG.md b/CHANGELOG.md index b9f14a1..56e5f64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added +- Added resumable clean-room operator automation (`tools/bootstrap-cloud-shell`, `tools/deploy-clean-room`, `tools/verify-clean-room`) with state-aware reruns, explicit APPLY, FORMAT, and REBOOT gates, and an optional tmux warning. - Added a minimal Cursor/AI-assisted governance workflow: implementation and independent review rules, plus human-facing `docs/AI_AGENT_WORKFLOW.md` - Configured Terraform to use the native OCI Object Storage backend with an externally supplied state bucket and environment-specific object key. - Defined the OCI Cloud Shell operator workflow for Terraform authentication, remote-state initialization, and deterministic Terraform-to-Ansible handoff. diff --git a/tests/unit/test_clean_room_automation_contract.sh b/tests/unit/test_clean_room_automation_contract.sh new file mode 100755 index 0000000..58ad052 --- /dev/null +++ b/tests/unit/test_clean_room_automation_contract.sh @@ -0,0 +1,1706 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Static and stubbed contract tests for clean-room operator automation. +# Does not contact OCI, SSH, Kubernetes, or download Terraform. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +PASS_COUNT=0 +FAIL_COUNT=0 +TMP_ROOT="" + +# ShellCheck cannot see that cleanup is invoked indirectly by the EXIT trap. +# shellcheck disable=SC2317 +cleanup() { + if [[ -n "${TMP_ROOT}" && -d "${TMP_ROOT}" ]]; then + rm -rf "${TMP_ROOT}" + fi +} +trap cleanup EXIT + +pass() { + printf 'PASS: %s\n' "$1" + PASS_COUNT=$((PASS_COUNT + 1)) +} + +fail() { + printf 'FAIL: %s\n' "$1" + FAIL_COUNT=$((FAIL_COUNT + 1)) +} + +assert_contains() { + local name="$1" + local needle="$2" + local text="$3" + if printf '%s' "$text" | grep -Fq "$needle"; then + pass "$name" + else + fail "$name" + fi +} + +assert_not_contains() { + local name="$1" + local needle="$2" + local text="$3" + if printf '%s' "$text" | grep -Fq "$needle"; then + fail "$name" + else + pass "$name" + fi +} + +python3 <<'PY' +from __future__ import annotations + +import json +import re +import tempfile +from pathlib import Path + +ROOT = Path(".").resolve() +TOOLS = { + "bootstrap": ROOT / "tools/bootstrap-cloud-shell", + "deploy": ROOT / "tools/deploy-clean-room", + "verify": ROOT / "tools/verify-clean-room", + "lib": ROOT / "tools/lib/clean-room-common.sh", + "readiness": ROOT / "tools/check-cloud-shell-readiness", +} + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def main() -> None: + for name, path in TOOLS.items(): + if not path.is_file(): + raise SystemExit(f"missing {path}") + text = read(path) + if name != "lib" and not text.startswith("#!/usr/bin/env bash"): + raise SystemExit(f"{path} must use #!/usr/bin/env bash") + if "set -x" in text or "set -o xtrace" in text: + raise SystemExit(f"{path} must not enable xtrace") + for token in (".clean-room-step", ".deployment-state", "last_completed_step"): + if token in text: + raise SystemExit(f"{path} must not use hidden step-state {token}") + print("PASS: tools exist without hidden step-state or xtrace") + + bootstrap = read(TOOLS["bootstrap"]) + deploy = read(TOOLS["deploy"]) + verify = read(TOOLS["verify"]) + lib = read(TOOLS["lib"]) + readiness = read(TOOLS["readiness"]) + combined = "\n".join([bootstrap, deploy, verify, lib, readiness]) + + for needle in ( + "WARNING: tmux not detected.", + "Running the clean-room workflow inside tmux is recommended", + ): + if needle not in lib: + raise SystemExit(f"tmux warning missing from shared helper: {needle}") + for name, text in (("bootstrap", bootstrap), ("deploy", deploy), ("verify", verify)): + if "clean_room_warn_tmux" not in text: + raise SystemExit(f"{name} must print the shared tmux warning") + if "tmux new" in combined or "tmux attach" in combined or "apt-get install tmux" in combined: + raise SystemExit("tools must not install or start tmux") + print("PASS: tmux warning is present and tmux remains optional") + + if "python3.12 -m venv" not in bootstrap: + raise SystemExit("bootstrap must create the venv with python3.12") + if "ansible-core==2.21.2" not in bootstrap and 'CLEAN_ROOM_ANSIBLE_CORE_PIN="2.21.2"' not in lib: + raise SystemExit("bootstrap/lib must pin ansible-core==2.21.2") + if "rm -rf" in bootstrap and "tradingchassis-ansible" in bootstrap: + if re.search(r"rm\s+-rf[^\n]*tradingchassis-ansible", bootstrap): + raise SystemExit("bootstrap must not blindly rm -rf the Ansible venv") + if "git clone" in bootstrap: + raise SystemExit("bootstrap must not clone the repository") + print("PASS: bootstrap uses python3.12 and does not clone or destroy the venv") + + if "require_cmd python3.12" not in readiness: + raise SystemExit("readiness must require python3.12") + if re.search(r"(?m)^require_cmd python3$", readiness): + raise SystemExit("readiness must not treat generic python3 as the Ansible interpreter") + print("PASS: readiness requires python3.12") + + if "-auto-approve" in deploy: + raise SystemExit("deploy-clean-room must not use -auto-approve") + if not re.search(r"-e\s+scratch_storage_allow_format=true", deploy): + raise SystemExit("deploy must be able to pass the one-time FORMAT extra-var") + if re.search(r"-e\s+scratch_storage_allow_format=true", verify): + raise SystemExit("verify-clean-room must never pass scratch_storage_allow_format=true") + if re.search(r"(?m)^\s*terraform\s+apply\b", verify): + raise SystemExit("verify-clean-room must not run terraform apply") + if "APPLY" not in deploy or "FORMAT" not in deploy: + raise SystemExit("deploy must implement APPLY and FORMAT gates") + if "REBOOT" not in verify: + raise SystemExit("verify must implement the REBOOT gate") + print("PASS: APPLY/FORMAT/REBOOT gates and no verify apply/format") + + for forbidden in ( + "rollout restart", + "kubectl delete job", + "microk8s kubectl delete", + "init-mlflow-postgres", + "Monitoring operator", + "sleep 300", + "sleep 5m", + ): + if forbidden in deploy or forbidden in verify: + raise SystemExit(f"operator tools must not include recovery hack: {forbidden}") + print("PASS: no Monitoring/Postgres recovery commands") + + if "detailed-exitcode" not in deploy or "detailed-exitcode" not in verify: + raise SystemExit("tools must use terraform -detailed-exitcode") + if "show -json" not in deploy: + raise SystemExit("deploy must inspect terraform show -json") + print("PASS: Terraform detailed-exitcode and JSON inspection are present") + + if '-e "@${PRIVATE_VARS}"' not in deploy or "private-runtime.yml" not in deploy: + raise SystemExit("deploy must use file-based private-runtime extra-vars") + print("PASS: private-runtime extra-vars are file-based") + + synthetic_forbidden = ( + "BEGIN PRIVATE KEY", + "BEGIN RSA PRIVATE KEY", + ) + for path in TOOLS.values(): + text = read(path) + for token in synthetic_forbidden: + if token in text: + raise SystemExit(f"{path} must not contain {token}") + unix_home_root = "/" + "home" + home_user_re = re.compile( + re.escape(unix_home_root) + r"/[A-Za-z0-9._-]+(?:/|$)" + ) + if home_user_re.search(text): + raise SystemExit(f"{path} must not contain a concrete Unix home path") + if re.search(r"ocid1\.[a-z]+\.oc1\.[a-z0-9]{8,}", text, flags=re.I): + raise SystemExit(f"{path} must not embed live-looking OCIDs") + print("PASS: tools contain no live identifiers or private key material") + + with tempfile.TemporaryDirectory(prefix="clean-room-json-") as tmp: + tmp_path = Path(tmp) + destructive = { + "resource_changes": [ + { + "address": "oci_core_instance.node", + "change": {"actions": ["delete", "create"]}, + } + ] + } + delete_only = { + "resource_changes": [ + { + "address": "oci_core_subnet.public", + "change": {"actions": ["delete"]}, + } + ] + } + create_only = { + "resource_changes": [ + { + "address": "oci_core_vcn.this", + "change": {"actions": ["create"]}, + } + ] + } + (tmp_path / "replace.json").write_text(json.dumps(destructive), encoding="utf-8") + (tmp_path / "delete.json").write_text(json.dumps(delete_only), encoding="utf-8") + (tmp_path / "create.json").write_text(json.dumps(create_only), encoding="utf-8") + + print("PASS: static clean-room automation contracts") + + +if __name__ == "__main__": + main() +PY + +# shellcheck source=../../tools/lib/clean-room-common.sh +# Dynamic ROOT path is not followed without shellcheck -x; the helper is linted separately. +# shellcheck disable=SC1091 +source "${ROOT}/tools/lib/clean-room-common.sh" + +if [[ "$(clean_room_terraform_zip_arch aarch64)" == "arm64" ]]; then + pass "arch map aarch64 -> arm64" +else + fail "arch map aarch64 -> arm64" +fi +if [[ "$(clean_room_terraform_zip_arch arm64)" == "arm64" ]]; then + pass "arch map arm64 -> arm64" +else + fail "arch map arm64 -> arm64" +fi +if [[ "$(clean_room_terraform_zip_arch x86_64)" == "amd64" ]]; then + pass "arch map x86_64 -> amd64" +else + fail "arch map x86_64 -> amd64" +fi +if [[ "$(clean_room_terraform_zip_arch amd64)" == "amd64" ]]; then + pass "arch map amd64 -> amd64" +else + fail "arch map amd64 -> amd64" +fi +if clean_room_terraform_zip_arch riscv64 >/dev/null 2>&1; then + fail "unsupported arch must fail" +else + pass "unsupported arch must fail" +fi + +if clean_room_terraform_version_ok "1.15.8"; then + pass "terraform 1.15.8 satisfies ~> 1.15.0" +else + fail "terraform 1.15.8 satisfies ~> 1.15.0" +fi +if clean_room_terraform_version_ok "1.14.9"; then + fail "terraform 1.14.9 must not satisfy ~> 1.15.0" +else + pass "terraform 1.14.9 must not satisfy ~> 1.15.0" +fi +if clean_room_terraform_version_ok "1.16.0"; then + fail "terraform 1.16.0 must not satisfy ~> 1.15.0" +else + pass "terraform 1.16.0 must not satisfy ~> 1.15.0" +fi + +TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/clean-room-automation.XXXXXX")" +HELPER_DIR="${TMP_ROOT}/helper" +mkdir -p "$HELPER_DIR" + +python3 - "$HELPER_DIR" <<'PY' +import json +from pathlib import Path +import sys + +root = Path(sys.argv[1]) +(root / "replace.json").write_text( + json.dumps( + { + "resource_changes": [ + { + "address": "oci_core_instance.node", + "change": {"actions": ["delete", "create"]}, + } + ] + } + ), + encoding="utf-8", +) +(root / "delete.json").write_text( + json.dumps( + { + "resource_changes": [ + { + "address": "oci_core_subnet.public", + "change": {"actions": ["delete"]}, + } + ] + } + ), + encoding="utf-8", +) +(root / "create.json").write_text( + json.dumps( + { + "resource_changes": [ + { + "address": "oci_core_vcn.this", + "change": {"actions": ["create"]}, + } + ] + } + ), + encoding="utf-8", +) +(root / "ok-recap.log").write_text( + "PLAY RECAP *********************************************************************\n" + "reference-node : ok=12 changed=0 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0\n", + encoding="utf-8", +) +(root / "changed-recap.log").write_text( + "PLAY RECAP *********************************************************************\n" + "reference-node : ok=12 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0\n", + encoding="utf-8", +) +(root / "blank.log").write_text( + "fatal: [reference-node]: FAILED! => {\n" + " 'msg': 'The scratch volume has no filesystem. Set scratch_storage_allow_format=true " + "only after verifying that this is the intended Terraform-managed scratch volume.'\n" + "}\n", + encoding="utf-8", +) +(root / "other.log").write_text("fatal: [reference-node]: FAILED! => ntp configuration failed\n", encoding="utf-8") +(root / "argo-ok.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "root"}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Healthy"}}, + }, + { + "metadata": {"name": "monitoring"}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Healthy"}}, + }, + ] + } + ), + encoding="utf-8", +) +(root / "argo-empty.json").write_text(json.dumps({"items": []}), encoding="utf-8") +(root / "argo-degraded.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "postgres"}, + "status": {"sync": {"status": "Synced"}, "health": {"status": "Degraded"}}, + } + ] + } + ), + encoding="utf-8", +) +(root / "argo-pending.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"name": "root"}, + "status": { + "sync": {"status": "OutOfSync"}, + "health": {"status": "Progressing"}, + }, + } + ] + } + ), + encoding="utf-8", +) +(root / "missing-recap.log").write_text("ok: all tasks completed\n", encoding="utf-8") +(root / "malformed-recap.log").write_text( + "PLAY RECAP *********************************************************************\n" + "reference-node completed without counters\n", + encoding="utf-8", +) +(root / "pods-ok.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"namespace": "postgres"}, + "status": { + "phase": "Succeeded", + "containerStatuses": [ + {"ready": False, "state": {"terminated": {"reason": "Completed"}}} + ], + }, + }, + { + "metadata": {"namespace": "mlflow"}, + "status": { + "phase": "Running", + "containerStatuses": [{"ready": True, "state": {"running": {}}}], + }, + }, + ] + } + ), + encoding="utf-8", +) +(root / "pods-crash.json").write_text( + json.dumps( + { + "items": [ + { + "metadata": {"namespace": "monitoring"}, + "status": { + "phase": "Running", + "containerStatuses": [ + { + "ready": False, + "state": {"waiting": {"reason": "CrashLoopBackOff"}}, + } + ], + }, + } + ] + } + ), + encoding="utf-8", +) +PY + +if clean_room_plan_is_destructive "${HELPER_DIR}/replace.json" >/dev/null; then + fail "replace plan must be destructive" +else + pass "replace plan is rejected" +fi +if clean_room_plan_is_destructive "${HELPER_DIR}/delete.json" >/dev/null; then + fail "delete plan must be destructive" +else + pass "delete plan is rejected" +fi +if clean_room_plan_is_destructive "${HELPER_DIR}/create.json" >/dev/null; then + pass "create-only plan is non-destructive" +else + fail "create-only plan is non-destructive" +fi + +if clean_room_parse_play_recap "${HELPER_DIR}/ok-recap.log" 1 >/dev/null; then + pass "PLAY RECAP changed=0 is accepted" +else + fail "PLAY RECAP changed=0 is accepted" +fi +if clean_room_parse_play_recap "${HELPER_DIR}/changed-recap.log" 1 >/dev/null; then + fail "PLAY RECAP changed>0 must fail" +else + pass "PLAY RECAP changed>0 must fail" +fi +set +e +clean_room_parse_play_recap "${HELPER_DIR}/missing-recap.log" 1 >/dev/null +missing_recap_rc=$? +clean_room_parse_play_recap "${HELPER_DIR}/malformed-recap.log" 1 >/dev/null +malformed_recap_rc=$? +set -e +if [[ "$missing_recap_rc" -ne 0 ]]; then + pass "missing PLAY RECAP fails closed" +else + fail "missing PLAY RECAP fails closed (rc=${missing_recap_rc})" +fi +if [[ "$malformed_recap_rc" -ne 0 ]]; then + pass "malformed PLAY RECAP fails closed" +else + fail "malformed PLAY RECAP fails closed (rc=${malformed_recap_rc})" +fi + +if clean_room_is_blank_scratch_gate "${HELPER_DIR}/blank.log"; then + pass "blank-scratch fail-closed message is detected" +else + fail "blank-scratch fail-closed message is detected" +fi +if clean_room_is_blank_scratch_gate "${HELPER_DIR}/other.log"; then + fail "unrelated Ansible failure must not look like FORMAT gate" +else + pass "unrelated Ansible failure must not look like FORMAT gate" +fi + +if clean_room_eval_argo_json "${HELPER_DIR}/argo-ok.json" >/dev/null; then + pass "Synced+Healthy Applications pass" +else + fail "Synced+Healthy Applications pass" +fi +set +e +clean_room_eval_argo_json "${HELPER_DIR}/argo-empty.json" >/dev/null +empty_rc=$? +clean_room_eval_argo_json "${HELPER_DIR}/argo-degraded.json" >/dev/null +degraded_rc=$? +set -e +if [[ "$empty_rc" -eq 2 ]]; then + pass "empty Application set fails" +else + fail "empty Application set fails (rc=${empty_rc})" +fi +if [[ "$degraded_rc" -eq 3 ]]; then + pass "Degraded Application fails immediately" +else + fail "Degraded Application fails immediately (rc=${degraded_rc})" +fi +set +e +clean_room_eval_argo_json "${HELPER_DIR}/argo-pending.json" >/dev/null +pending_rc=$? +set -e +if [[ "$pending_rc" -eq 1 ]]; then + pass "non-converged Applications wait rather than pass" +else + fail "non-converged Applications wait rather than pass (rc=${pending_rc})" +fi + +if clean_room_eval_pods_json "${HELPER_DIR}/pods-ok.json" >/dev/null; then + pass "completed Jobs are treated as healthy" +else + fail "completed Jobs are treated as healthy" +fi +set +e +clean_room_eval_pods_json "${HELPER_DIR}/pods-crash.json" >/dev/null +crash_rc=$? +set -e +if [[ "$crash_rc" -eq 2 ]]; then + pass "CrashLoopBackOff pods fail" +else + fail "CrashLoopBackOff pods fail (rc=${crash_rc})" +fi + +printf 'APPLY\n' | clean_room_require_exact_input APPLY +pass "exact APPLY input is accepted" +if printf 'nope\n' | clean_room_require_exact_input APPLY; then + fail "non-APPLY input must be rejected" +else + pass "non-APPLY input must be rejected" +fi + +unset TMUX || true +tmux_out="$(clean_room_warn_tmux)" +assert_contains "tmux warning when TMUX is unset" "WARNING: tmux not detected." "$tmux_out" + +seed_fixture() { + local fx="$1" + mkdir -p \ + "${fx}/terraform" \ + "${fx}/ansible/extra-vars" \ + "${fx}/ansible/playbooks" \ + "${fx}/ansible/inventory" \ + "${fx}/docs" \ + "${fx}/tools" + cp "${ROOT}/terraform/versions.tf" "${fx}/terraform/versions.tf" + cp "${ROOT}/terraform/backend.hcl.example" "${fx}/terraform/backend.hcl.example" + cp "${ROOT}/terraform/terraform.tfvars.example" "${fx}/terraform/terraform.tfvars.example" + cp "${ROOT}/ansible/extra-vars/private-runtime.yml.example" \ + "${fx}/ansible/extra-vars/private-runtime.yml.example" + cp "${ROOT}/ansible/requirements.yml" "${fx}/ansible/requirements.yml" + cp "${ROOT}/ansible/playbooks/site.yml" "${fx}/ansible/playbooks/site.yml" + cp "${ROOT}/ansible/playbooks/private-runtime-config.yml" \ + "${fx}/ansible/playbooks/private-runtime-config.yml" + cp "${ROOT}/ansible/ansible.cfg" "${fx}/ansible/ansible.cfg" + cp "${ROOT}/docs/V2_CLEAN_ROOM_DEPLOYMENT.md" "${fx}/docs/V2_CLEAN_ROOM_DEPLOYMENT.md" +} + +write_completed_inputs() { + local fx="$1" + cat >"${fx}/terraform/backend.hcl" <<'EOF' +bucket = "example-state-bucket" +namespace = "examplenamespace" +region = "eu-frankfurt-1" +key = "tradingchassis/production/terraform.tfstate" +auth = "APIKey" +config_file_profile = "tradingchassis" +EOF + cat >"${fx}/terraform/terraform.tfvars" <<'EOF' +oci_auth = "APIKey" +oci_config_file_profile = "tradingchassis" +oci_region = "eu-frankfurt-1" +oci_compartment_id = "ocid1.compartment.oc1..example" +oci_tenancy_id = "ocid1.tenancy.oc1..example" +oci_vault_id = "ocid1.vault.oc1..example" +oci_vault_compartment_id = "ocid1.compartment.oc1..vault-example" +ssh_ingress_cidr = "203.0.113.10/32" +ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA contract-test@example.invalid" +EOF + cat >"${fx}/ansible/extra-vars/private-runtime.yml" <<'EOF' +private_runtime_config_vault_id: "ocid1.vault.oc1.eu-test-1..aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +private_runtime_config_oci_region: "eu-test-1" +EOF + chmod 600 \ + "${fx}/terraform/backend.hcl" \ + "${fx}/terraform/terraform.tfvars" \ + "${fx}/ansible/extra-vars/private-runtime.yml" +} + +seed_operator_home() { + local home="$1" + mkdir -p "${home}/bin" "${home}/.oci" "${home}/.ssh" "${home}/.venvs/tradingchassis-ansible/bin" + printf '%s\n' "[tradingchassis]" >"${home}/.oci/config" + printf '%s\n' "not-a-secret" >"${home}/.oci/tradingchassis_api_key.pem" + printf '%s\n' "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA contract-test@example.invalid" \ + >"${home}/.ssh/tradingchassis.pub" + printf '%s\n' "not-a-secret-key" >"${home}/.ssh/tradingchassis" + chmod 600 "${home}/.oci/config" "${home}/.oci/tradingchassis_api_key.pem" "${home}/.ssh/tradingchassis" +} + +write_python312_stub() { + local dest="$1" + cat >"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "-c" ]]; then + if [[ "${2:-}" == *'%d.%d'* ]]; then + printf '%s\n' "3.12" + exit 0 + fi + exec python3 -c "${2}" +fi +if [[ "${1:-}" == "-m" && "${2:-}" == "venv" ]]; then + dest="${3:?}" + mkdir -p "${dest}/bin" + cat >"${dest}/bin/python" <<'PY' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "-c" ]]; then + if [[ "${2:-}" == *version_info[:2]* ]]; then + exit 0 + fi + exec python3 -c "${2}" +fi +if [[ "${1:-}" == "-m" && "${2:-}" == "pip" ]]; then + exit 0 +fi +exit 0 +PY + cat >"${dest}/bin/pip" <<'PIP' +#!/usr/bin/env bash +exit 0 +PIP + cat >"${dest}/bin/ansible-playbook" <<'AP' +#!/usr/bin/env bash +if [[ "${1:-}" == "--version" ]]; then + echo "ansible-playbook [core 2.21.2]" + exit 0 +fi +exit 0 +AP + cat >"${dest}/bin/ansible-galaxy" <<'AG' +#!/usr/bin/env bash +exit 0 +AG + chmod +x "${dest}/bin/python" "${dest}/bin/pip" "${dest}/bin/ansible-playbook" "${dest}/bin/ansible-galaxy" + exit 0 +fi +exec python3 "$@" +EOF + chmod +x "$dest" +} + +write_tf_stub() { + local dest="$1" + cat >"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +log="${TF_STUB_LOG:-/dev/null}" +printf '%s\n' "$*" >>"$log" +args=() +for arg in "$@"; do + case "$arg" in + -chdir=*) ;; + *) args+=("$arg") ;; + esac +done +set -- "${args[@]}" +case "${1:-}" in + version) + if [[ "${2:-}" == "-json" ]]; then + printf '%s\n' '{"terraform_version":"1.15.8"}' + else + printf '%s\n' "Terraform v1.15.8" + fi + exit 0 + ;; + init|validate) + exit 0 + ;; + plan) + count_file="${TF_STUB_PLAN_COUNT:-}" + n=1 + if [[ -n "$count_file" ]]; then + if [[ -f "$count_file" ]]; then + n="$(cat "$count_file")" + n=$((n + 1)) + fi + printf '%s\n' "$n" >"$count_file" + fi + out="" + for arg in "$@"; do + case "$arg" in + -out=*) out="${arg#-out=}" ;; + esac + done + if [[ -n "$out" ]]; then + printf 'stub-plan\n' >"$out" + fi + if [[ -n "$count_file" && "$n" -ge 2 ]]; then + exit "${TF_STUB_POST_PLAN_EXIT:-0}" + fi + exit "${TF_STUB_PLAN_EXIT:-0}" + ;; + apply) + printf 'apply\n' >>"${TF_STUB_APPLY_LOG:-/dev/null}" + exit 0 + ;; + show) + if [[ "${2:-}" == "-json" ]]; then + cat "${TF_STUB_PLAN_JSON:?}" + exit 0 + fi + printf '%s\n' "Plan: 1 to add, 0 to change, 0 to destroy." + exit 0 + ;; + output) + printf '%s\n' "192.0.2.10" + exit 0 + ;; + *) + exit 0 + ;; +esac +EOF + chmod +x "$dest" +} + +write_ansible_stub() { + local dest="$1" + cat >"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${ANSIBLE_STUB_LOG:?}" +if printf '%s' "$*" | grep -Fq 'scratch_storage_allow_format=true'; then + printf 'FORMAT_FLAG\n' >>"${ANSIBLE_STUB_LOG}" +fi +if printf '%s' "$*" | grep -Fq 'private-runtime-config.yml'; then + if [[ "${PRIV_RECAP:-ok}" == "missing" ]]; then + printf '%s\n' "ok: private-runtime finished without recap" + exit "${PRIV_EXIT:-0}" + fi + if [[ "${PRIV_RECAP:-ok}" == "malformed" ]]; then + cat <<'REC' +PLAY RECAP ********************************************************************* +reference-node completed without counters +REC + exit "${PRIV_EXIT:-0}" + fi + cat < { + "msg": "The scratch volume has no filesystem. Set scratch_storage_allow_format=true only after verifying that this is the intended Terraform-managed scratch volume." +} +BLANK + exit 2 + fi + if [[ "${SITE_MODE:-ok}" == "other-fail" ]]; then + printf '%s\n' "fatal: [reference-node]: FAILED! => ntp configuration failed" + exit 2 + fi + if [[ "${SITE_RECAP:-ok}" == "missing" ]]; then + printf '%s\n' "ok: site.yml finished without recap" + exit "${SITE_EXIT:-0}" + fi + if [[ "${SITE_RECAP:-ok}" == "malformed" ]]; then + cat <<'REC' +PLAY RECAP ********************************************************************* +reference-node completed without counters +REC + exit "${SITE_EXIT:-0}" + fi + cat <"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${SSH_STUB_LOG:?}" +remote="" +for arg in "$@"; do + remote="$arg" +done +case "$remote" in + true) + if [[ -f "${SSH_REBOOTED:-/tmp/does-not-exist}" && ! -f "${SSH_DROPPED:-/tmp/does-not-exist}" ]]; then + : >"${SSH_DROPPED}" + exit 1 + fi + exit 0 + ;; + *get\ applications*) + cat "${ARGO_JSON:?}" + exit 0 + ;; + *get\ pods*) + cat "${PODS_JSON:?}" + exit 0 + ;; + *boot_id*) + rebooted=0 + if [[ -f "${SSH_REBOOTED:-/tmp/does-not-exist}" ]]; then + rebooted=1 + fi + case "${SSH_BOOT_MODE:-ok}" in + empty-before) + if [[ "$rebooted" -eq 0 ]]; then + printf '\n' + exit 0 + fi + printf '%s\n' "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + exit 0 + ;; + fail-before) + if [[ "$rebooted" -eq 0 ]]; then + exit 1 + fi + printf '%s\n' "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + exit 0 + ;; + unchanged) + printf '%s\n' "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + exit 0 + ;; + empty-after) + if [[ "$rebooted" -eq 1 ]]; then + printf '\n' + exit 0 + fi + printf '%s\n' "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + exit 0 + ;; + *) + if [[ "$rebooted" -eq 1 ]]; then + printf '%s\n' "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + else + printf '%s\n' "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + fi + exit 0 + ;; + esac + ;; + *mountpoint*) + exit "${MOUNT_RC:-0}" + ;; + *microk8s\ status*) + printf '%s\n' "microk8s is running" + exit 0 + ;; + *reboot*) + : >"${SSH_REBOOTED:?}" + exit 255 + ;; + *) + exit 0 + ;; +esac +EOF + chmod +x "$dest" +} + +write_oci_stub() { + local dest="$1" + cat >"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${OCI_STUB_LOG:-/dev/null}" +if printf '%s' "$*" | grep -Fq 'instance_obo_user'; then + echo "refusing instance_obo_user" >&2 + exit 1 +fi +if printf '%s' "$*" | grep -Fq -- '--auth api_key'; then + exit 0 +fi +exit 1 +EOF + chmod +x "$dest" +} + +write_curl_stub() { + local dest="$1" + cat >"$dest" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${CURL_STUB_LOG:?}" +out="" +url="" +while [[ $# -gt 0 ]]; do + case "$1" in + -o|--output) + out="$2" + shift 2 + ;; + --proto|--proto-redir) + shift 2 + ;; + --tlsv1.2|-fsSL|-s|-f|-L|-S) + shift + ;; + *) + url="$1" + shift + ;; + esac +done +if [[ -z "$out" || -z "$url" ]]; then + exit 1 +fi +if [[ "$url" == *SHA256SUMS ]]; then + cp "${CURL_SUMS:?}" "$out" + exit 0 +fi +if [[ "$url" == *.zip ]]; then + cp "${CURL_ZIP:?}" "$out" + exit 0 +fi +exit 1 +EOF + chmod +x "$dest" +} + +# --- bootstrap: tmux warning does not fail --- +fx="${TMP_ROOT}/boot-tmux" +home="${TMP_ROOT}/home-tmux" +mkdir -p "$fx" "$home" "${TMP_ROOT}/stubs-tmux" +seed_fixture "$fx" +seed_operator_home "$home" +write_python312_stub "${TMP_ROOT}/stubs-tmux/python3.12" +write_tf_stub "${home}/bin/terraform" +write_ansible_stub "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +printf '%s\n' '#!/usr/bin/env bash' 'echo ansible-playbook [core 2.21.2]' >"${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +chmod +x "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +# recreate a venv python that claims 3.12 so reuse path works +mkdir -p "${home}/.venvs/tradingchassis-ansible/bin" +cat >"${home}/.venvs/tradingchassis-ansible/bin/python" <<'EOF' +#!/usr/bin/env bash +if [[ "${1:-}" == "-c" && "${2:-}" == *version_info[:2]* ]]; then + exit 0 +fi +if [[ "${1:-}" == "-m" && "${2:-}" == "pip" ]]; then + exit 0 +fi +exit 0 +EOF +cat >"${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +chmod +x "${home}/.venvs/tradingchassis-ansible/bin/python" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +write_curl_stub "${TMP_ROOT}/stubs-tmux/curl" +export CURL_STUB_LOG="${TMP_ROOT}/curl-tmux.log" +: >"$CURL_STUB_LOG" +unset TMUX || true +set +e +out="$( + HOME="$home" PATH="${TMP_ROOT}/stubs-tmux:${home}/bin:${PATH}" \ + "${ROOT}/tools/bootstrap-cloud-shell" --root "$fx" 2>&1 +)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]]; then + pass "bootstrap succeeds when TMUX is unset" +else + fail "bootstrap succeeds when TMUX is unset (rc=${rc})" + printf '%s\n' "$out" +fi +assert_contains "bootstrap prints tmux warning" "WARNING: tmux not detected." "$out" +assert_contains "bootstrap does not start deployment" "Deployment has NOT begun." "$out" +assert_not_contains "bootstrap did not download Terraform when present" "releases.hashicorp.com" "$out" + +# --- bootstrap: create vs preserve operator files --- +fx="${TMP_ROOT}/boot-files" +home="${TMP_ROOT}/home-files" +mkdir -p "${TMP_ROOT}/stubs-files" +seed_fixture "$fx" +seed_operator_home "$home" +write_python312_stub "${TMP_ROOT}/stubs-files/python3.12" +write_tf_stub "${home}/bin/terraform" +mkdir -p "${home}/.venvs/tradingchassis-ansible/bin" +cat >"${home}/.venvs/tradingchassis-ansible/bin/python" <<'EOF' +#!/usr/bin/env bash +if [[ "${1:-}" == "-c" && "${2:-}" == *version_info[:2]* ]]; then exit 0; fi +if [[ "${1:-}" == "-m" && "${2:-}" == "pip" ]]; then exit 0; fi +exit 0 +EOF +printf '%s\n' '#!/usr/bin/env bash' 'echo ansible-playbook [core 2.21.2]' \ + >"${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +printf '%s\n' '#!/usr/bin/env bash' 'exit 0' \ + >"${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" +chmod +x \ + "${home}/.venvs/tradingchassis-ansible/bin/python" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" +printf '%s\n' "UNIQUE_OPERATOR_MARKER" >"${fx}/terraform/backend.hcl" +HOME="$home" PATH="${TMP_ROOT}/stubs-files:${home}/bin:${PATH}" \ + "${ROOT}/tools/bootstrap-cloud-shell" --root "$fx" >/dev/null +if grep -Fq "UNIQUE_OPERATOR_MARKER" "${fx}/terraform/backend.hcl"; then + pass "bootstrap does not overwrite existing operator files" +else + fail "bootstrap does not overwrite existing operator files" +fi +if [[ -f "${fx}/terraform/terraform.tfvars" && -f "${fx}/ansible/extra-vars/private-runtime.yml" ]]; then + pass "bootstrap creates missing operator input files from examples" +else + fail "bootstrap creates missing operator input files from examples" +fi + +# --- bootstrap: checksum mismatch is fatal --- +fx="${TMP_ROOT}/boot-sum" +home="${TMP_ROOT}/home-sum" +mkdir -p "${TMP_ROOT}/stubs-sum" "$home" +seed_fixture "$fx" +seed_operator_home "$home" +rm -f "${home}/bin/terraform" +write_python312_stub "${TMP_ROOT}/stubs-sum/python3.12" +arch="$(clean_room_terraform_zip_arch)" +zip_name="terraform_1.15.8_linux_${arch}.zip" +python3 - "${TMP_ROOT}/${zip_name}" "${TMP_ROOT}/bad.SHA256SUMS" "${TMP_ROOT}/good.SHA256SUMS" <<'PY' +import hashlib +import sys +import zipfile +from pathlib import Path + +zip_path = Path(sys.argv[1]) +with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("terraform", "#!/bin/sh\necho Terraform v1.15.8\n") +digest = hashlib.sha256(zip_path.read_bytes()).hexdigest() +Path(sys.argv[2]).write_text(f"{'0'*64} {zip_path.name}\n", encoding="utf-8") +Path(sys.argv[3]).write_text(f"{digest} {zip_path.name}\n", encoding="utf-8") +PY +write_curl_stub "${TMP_ROOT}/stubs-sum/curl" +export CURL_STUB_LOG="${TMP_ROOT}/curl-sum.log" +export CURL_SUMS="${TMP_ROOT}/bad.SHA256SUMS" +export CURL_ZIP="${TMP_ROOT}/${zip_name}" +mkdir -p "${home}/.venvs/tradingchassis-ansible/bin" +cat >"${home}/.venvs/tradingchassis-ansible/bin/python" <<'EOF' +#!/usr/bin/env bash +if [[ "${1:-}" == "-c" && "${2:-}" == *version_info[:2]* ]]; then exit 0; fi +if [[ "${1:-}" == "-m" && "${2:-}" == "pip" ]]; then exit 0; fi +exit 0 +EOF +printf '%s\n' '#!/usr/bin/env bash' 'echo ansible-playbook [core 2.21.2]' \ + >"${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" +printf '%s\n' '#!/usr/bin/env bash' 'exit 0' \ + >"${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" +chmod +x \ + "${home}/.venvs/tradingchassis-ansible/bin/python" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" \ + "${home}/.venvs/tradingchassis-ansible/bin/ansible-galaxy" +set +e +out="$( + HOME="$home" PATH="${TMP_ROOT}/stubs-sum:${PATH}" \ + "${ROOT}/tools/bootstrap-cloud-shell" --root "$fx" 2>&1 +)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "checksum mismatch"; then + pass "Terraform checksum mismatch is fatal" +else + fail "Terraform checksum mismatch is fatal (rc=${rc})" + printf '%s\n' "$out" +fi +if [[ -e "${home}/bin/terraform" ]]; then + fail "checksum failure must not install Terraform" +else + pass "checksum failure must not install Terraform" +fi + +# --- deploy/verify integration stubs --- +prepare_runtime() { + local prefix="$1" + local fx="${TMP_ROOT}/${prefix}-fx" + local home="${TMP_ROOT}/${prefix}-home" + local stubs="${TMP_ROOT}/${prefix}-stubs" + rm -rf "$fx" "$home" "$stubs" + mkdir -p "$fx" "$home" "$stubs" + seed_fixture "$fx" + write_completed_inputs "$fx" + seed_operator_home "$home" + write_python312_stub "${stubs}/python3.12" + write_tf_stub "${home}/bin/terraform" + write_ansible_stub "${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook" + cat >"${home}/.venvs/tradingchassis-ansible/bin/python" <<'EOF' +#!/usr/bin/env bash +if [[ "${1:-}" == "-c" && "${2:-}" == *version_info[:2]* ]]; then exit 0; fi +exit 0 +EOF + printf '%s\n' '#!/usr/bin/env bash' 'echo ansible-playbook [core 2.21.2]' \ + >"${home}/.venvs/tradingchassis-ansible/bin/ansible-playbook.version" + write_ssh_stub "${stubs}/ssh" + write_oci_stub "${stubs}/oci" + printf '%s\n' '#!/usr/bin/env bash' 'exit 0' >"${stubs}/ssh-keygen" + chmod +x "${home}/.venvs/tradingchassis-ansible/bin/python" "${stubs}/ssh-keygen" + cp "${HELPER_DIR}/create.json" "${TMP_ROOT}/${prefix}-create.json" + cp "${HELPER_DIR}/replace.json" "${TMP_ROOT}/${prefix}-replace.json" + cp "${HELPER_DIR}/argo-ok.json" "${TMP_ROOT}/${prefix}-argo.json" + cp "${HELPER_DIR}/pods-ok.json" "${TMP_ROOT}/${prefix}-pods.json" + printf '%s' "$fx $home $stubs" +} + +run_deploy_env() { + local fx="$1" home="$2" stubs="$3" + HOME="$home" \ + PATH="${home}/bin:${home}/.venvs/tradingchassis-ansible/bin:${stubs}:${PATH}" \ + TF_STUB_LOG="${home}/tf.log" \ + TF_STUB_APPLY_LOG="${home}/apply.log" \ + TF_STUB_PLAN_COUNT="${home}/plan.count" \ + TF_STUB_PLAN_EXIT="${TF_STUB_PLAN_EXIT:-0}" \ + TF_STUB_POST_PLAN_EXIT="${TF_STUB_POST_PLAN_EXIT:-0}" \ + TF_STUB_PLAN_JSON="${TF_STUB_PLAN_JSON:-}" \ + ANSIBLE_STUB_LOG="${home}/ansible.log" \ + SSH_STUB_LOG="${home}/ssh.log" \ + OCI_STUB_LOG="${home}/oci.log" \ + ARGO_JSON="${home}/argo.json" \ + PODS_JSON="${home}/pods.json" \ + SSH_REBOOTED="${home}/rebooted" \ + SSH_DROPPED="${home}/dropped" \ + SITE_MODE="${SITE_MODE:-ok}" \ + SITE_CHANGED="${SITE_CHANGED:-0}" \ + PRIV_CHANGED="${PRIV_CHANGED:-0}" \ + CLEAN_ROOM_SSH_ATTEMPTS=2 \ + CLEAN_ROOM_SSH_DELAY=0 \ + CLEAN_ROOM_ARGO_ATTEMPTS=2 \ + CLEAN_ROOM_ARGO_DELAY=0 \ + CLEAN_ROOM_WORKLOAD_ATTEMPTS=2 \ + CLEAN_ROOM_WORKLOAD_DELAY=0 \ + "${ROOT}/tools/deploy-clean-room" --root "$fx" +} + +read -r fx home stubs <<<"$(prepare_runtime nochg2)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +set +e +out="$(TF_STUB_PLAN_EXIT=0 TF_STUB_PLAN_JSON="${HELPER_DIR}/create.json" SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]] && printf '%s' "$out" | grep -Fq "skipping APPLY"; then + pass "no-change Terraform plan skips APPLY" +else + fail "no-change Terraform plan skips APPLY (rc=${rc})" + printf '%s\n' "$out" +fi +if [[ -s "${home}/apply.log" ]]; then + fail "no-change plan must not apply" +else + pass "no-change plan must not apply" +fi +if grep -Fq FORMAT_FLAG "${home}/ansible.log" 2>/dev/null; then + fail "successful site.yml must not pass FORMAT" +else + pass "successful site.yml must not pass FORMAT" +fi +if grep -Fq "private-runtime-config.yml" "${home}/ansible.log"; then + pass "private-runtime runs after successful site.yml" +else + fail "private-runtime runs after successful site.yml" +fi + +# changed plan requires APPLY; other input prevents apply +read -r fx home stubs <<<"$(prepare_runtime applyno)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +set +e +out="$(printf 'nope\n' | TF_STUB_PLAN_EXIT=2 TF_STUB_PLAN_JSON="${HELPER_DIR}/create.json" SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "APPLY was not confirmed"; then + pass "non-APPLY input prevents terraform apply" +else + fail "non-APPLY input prevents terraform apply (rc=${rc})" + printf '%s\n' "$out" +fi +if [[ -s "${home}/apply.log" ]]; then + fail "refused APPLY must not create an apply record" +else + pass "refused APPLY must not create an apply record" +fi + +# APPLY confirmed + post-apply no-change +read -r fx home stubs <<<"$(prepare_runtime applyyes)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +: >"${home}/plan.count" +set +e +out="$(printf 'APPLY\n' | TF_STUB_PLAN_EXIT=2 TF_STUB_POST_PLAN_EXIT=0 TF_STUB_PLAN_JSON="${HELPER_DIR}/create.json" SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]] && grep -Fq apply "${home}/apply.log"; then + pass "exact APPLY applies the saved plan" +else + fail "exact APPLY applies the saved plan (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "post-apply Terraform plan has no changes"; then + pass "successful apply requires post-apply no-change" +else + fail "successful apply requires post-apply no-change" + printf '%s\n' "$out" +fi + +# destructive plan rejected before apply +read -r fx home stubs <<<"$(prepare_runtime destroy)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +set +e +out="$(printf 'APPLY\n' | TF_STUB_PLAN_EXIT=2 TF_STUB_PLAN_JSON="${HELPER_DIR}/replace.json" SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "delete or replace"; then + pass "destructive Terraform plan is rejected before apply" +else + fail "destructive Terraform plan is rejected before apply (rc=${rc})" + printf '%s\n' "$out" +fi +if [[ -s "${home}/apply.log" ]]; then + fail "destructive plan must not apply" +else + pass "destructive plan must not apply" +fi + +# blank scratch offers FORMAT; unrelated failure does not +read -r fx home stubs <<<"$(prepare_runtime blank)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'FORMAT\n' | TF_STUB_PLAN_EXIT=0 SITE_MODE=blank run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]] && grep -Fq FORMAT_FLAG "${home}/ansible.log"; then + pass "blank-scratch fail-closed error offers FORMAT and reruns with the flag" +else + fail "blank-scratch FORMAT path (rc=${rc})" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime otherans)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=other-fail run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "FORMAT was not offered"; then + pass "unrelated Ansible error does not offer FORMAT" +else + fail "unrelated Ansible error does not offer FORMAT (rc=${rc})" + printf '%s\n' "$out" +fi +if grep -Fq FORMAT_FLAG "${home}/ansible.log" 2>/dev/null; then + fail "unrelated Ansible error must not pass FORMAT flag" +else + pass "unrelated Ansible error must not pass FORMAT flag" +fi + +read -r fx home stubs <<<"$(prepare_runtime fmtno)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'nope\n' | TF_STUB_PLAN_EXIT=0 SITE_MODE=blank run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "FORMAT was not confirmed"; then + pass "non-FORMAT input stops without formatting" +else + fail "non-FORMAT input stops without formatting (rc=${rc})" + printf '%s\n' "$out" +fi + +# rerun with existing scratch never supplies format flag +read -r fx home stubs <<<"$(prepare_runtime rerun)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]] && ! grep -Fq FORMAT_FLAG "${home}/ansible.log"; then + pass "rerun with existing scratch never supplies FORMAT flag" +else + fail "rerun with existing scratch never supplies FORMAT flag (rc=${rc})" + printf '%s\n' "$out" +fi + +# empty Argo set fails +read -r fx home stubs <<<"$(prepare_runtime argoempty)" +cp "${HELPER_DIR}/argo-empty.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Eq 'empty or unhealthy|no Argo Applications'; then + pass "empty Application set fails deploy wait" +else + fail "empty Application set fails deploy wait (rc=${rc})" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime argodeg)" +cp "${HELPER_DIR}/argo-degraded.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]]; then + pass "Degraded Applications fail deploy wait" +else + fail "Degraded Applications fail deploy wait" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime argotime)" +cp "${HELPER_DIR}/argo-pending.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "before timeout"; then + pass "Argo timeout fails deploy wait" +else + fail "Argo timeout fails deploy wait (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room deployment convergence completed"; then + fail "Argo timeout must not report deploy PASS" +else + pass "Argo timeout must not report deploy PASS" +fi + +# post-apply remaining changes stop before Ansible +read -r fx home stubs <<<"$(prepare_runtime postdrift)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +: >"${home}/ansible.log" +: >"${home}/plan.count" +set +e +out="$(printf 'APPLY\n' | TF_STUB_PLAN_EXIT=2 TF_STUB_POST_PLAN_EXIT=2 TF_STUB_PLAN_JSON="${HELPER_DIR}/create.json" SITE_MODE=ok run_deploy_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "stopping before Ansible"; then + pass "post-apply Terraform changes stop deploy before Ansible" +else + fail "post-apply Terraform changes stop deploy before Ansible (rc=${rc})" + printf '%s\n' "$out" +fi +if grep -Fq apply "${home}/apply.log"; then + pass "post-apply drift still applied the saved plan" +else + fail "post-apply drift still applied the saved plan" +fi +if grep -Eq 'site.yml|private-runtime-config.yml' "${home}/ansible.log"; then + fail "post-apply drift must not continue into Ansible" +else + pass "post-apply drift must not continue into Ansible" +fi + +run_verify_env() { + local fx="$1" home="$2" stubs="$3" + HOME="$home" \ + PATH="${home}/bin:${home}/.venvs/tradingchassis-ansible/bin:${stubs}:${PATH}" \ + TF_STUB_LOG="${home}/tf.log" \ + TF_STUB_APPLY_LOG="${home}/apply.log" \ + TF_STUB_PLAN_COUNT="${home}/plan.count" \ + TF_STUB_PLAN_EXIT="${TF_STUB_PLAN_EXIT:-0}" \ + TF_STUB_PLAN_JSON="${TF_STUB_PLAN_JSON:-}" \ + ANSIBLE_STUB_LOG="${home}/ansible.log" \ + SSH_STUB_LOG="${home}/ssh.log" \ + OCI_STUB_LOG="${home}/oci.log" \ + ARGO_JSON="${home}/argo.json" \ + PODS_JSON="${home}/pods.json" \ + SSH_REBOOTED="${home}/rebooted" \ + SSH_DROPPED="${home}/dropped" \ + SSH_BOOT_MODE="${SSH_BOOT_MODE:-ok}" \ + SITE_MODE="${SITE_MODE:-ok}" \ + SITE_CHANGED="${SITE_CHANGED:-0}" \ + SITE_RECAP="${SITE_RECAP:-ok}" \ + PRIV_CHANGED="${PRIV_CHANGED:-0}" \ + PRIV_RECAP="${PRIV_RECAP:-ok}" \ + CLEAN_ROOM_SSH_ATTEMPTS=2 \ + CLEAN_ROOM_SSH_DELAY=0 \ + CLEAN_ROOM_ARGO_ATTEMPTS=2 \ + CLEAN_ROOM_ARGO_DELAY=0 \ + CLEAN_ROOM_WORKLOAD_ATTEMPTS=2 \ + CLEAN_ROOM_WORKLOAD_DELAY=0 \ + CLEAN_ROOM_REBOOT_DROP_ATTEMPTS=3 \ + CLEAN_ROOM_REBOOT_DROP_DELAY=0 \ + CLEAN_ROOM_REBOOT_RETURN_ATTEMPTS=3 \ + CLEAN_ROOM_REBOOT_RETURN_DELAY=0 \ + CLEAN_ROOM_MICROK8S_TIMEOUT=1 \ + "${ROOT}/tools/verify-clean-room" --root "$fx" +} + +# terraform drift fails verify without apply +read -r fx home stubs <<<"$(prepare_runtime vdrift)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +set +e +out="$(TF_STUB_PLAN_EXIT=2 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "Terraform drift detected"; then + pass "Terraform drift fails verification instead of applying" +else + fail "Terraform drift fails verification instead of applying (rc=${rc})" + printf '%s\n' "$out" +fi +if [[ -s "${home}/apply.log" ]]; then + fail "verify must not apply Terraform" +else + pass "verify must not apply Terraform" +fi + +read -r fx home stubs <<<"$(prepare_runtime vplanerr)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/apply.log" +set +e +out="$(TF_STUB_PLAN_EXIT=1 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "terraform plan failed"; then + pass "Terraform plan error fails verification without a drift message" +else + fail "Terraform plan error fails verification without a drift message (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "Terraform drift detected"; then + fail "plan error must not be reported as drift" +else + pass "plan error must not be reported as drift" +fi +if [[ -s "${home}/apply.log" ]]; then + fail "plan error must not apply Terraform" +else + pass "plan error must not apply Terraform" +fi + +# private-runtime changed>0 fails; site.yml changed=0 required +read -r fx home stubs <<<"$(prepare_runtime vpriv)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 PRIV_CHANGED=2 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not idempotent"; then + pass "private-runtime changed>0 fails acceptance" +else + fail "private-runtime changed>0 fails acceptance (rc=${rc})" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime vsitechg)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_CHANGED=4 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not idempotent"; then + pass "site.yml changed>0 fails acceptance" +else + fail "site.yml changed>0 fails acceptance (rc=${rc})" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime vprivrecap)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 PRIV_RECAP=missing run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not idempotent"; then + pass "missing private-runtime PLAY RECAP fails verification" +else + fail "missing private-runtime PLAY RECAP fails verification (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "missing private-runtime recap must not report acceptance PASS" +else + pass "missing private-runtime recap must not report acceptance PASS" +fi + +read -r fx home stubs <<<"$(prepare_runtime vsiterecap)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 SITE_RECAP=malformed run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "not idempotent"; then + pass "malformed site.yml PLAY RECAP fails verification" +else + fail "malformed site.yml PLAY RECAP fails verification (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "malformed site.yml recap must not report acceptance PASS" +else + pass "malformed site.yml recap must not report acceptance PASS" +fi + +# decline reboot: no reboot, no final PASS +read -r fx home stubs <<<"$(prepare_runtime vnoreboot)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'nope\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "acceptance is incomplete"; then + pass "declining REBOOT does not report final PASS" +else + fail "declining REBOOT does not report final PASS (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "declined reboot must not print acceptance PASS" +else + pass "declined reboot must not print acceptance PASS" +fi +if grep -Fq "sudo reboot" "${home}/ssh.log"; then + fail "declined REBOOT must not issue reboot" +else + pass "declined REBOOT must not issue reboot" +fi +if grep -Fq 'scratch_storage_allow_format=true' "${home}/ansible.log"; then + fail "verify site.yml must never receive FORMAT flag" +else + pass "verify site.yml must never receive FORMAT flag" +fi + +# full verify PASS with REBOOT +read -r fx home stubs <<<"$(prepare_runtime vpass)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'REBOOT\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -eq 0 ]] && printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + pass "REBOOT confirmation plus post-reboot checks can PASS" +else + fail "REBOOT confirmation plus post-reboot checks can PASS (rc=${rc})" + printf '%s\n' "$out" +fi +if grep -Fq "sudo reboot" "${home}/ssh.log"; then + pass "confirmed REBOOT issues remote reboot" +else + fail "confirmed REBOOT issues remote reboot" +fi + +# unhealthy pods fail verify +read -r fx home stubs <<<"$(prepare_runtime vcrash)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-crash.json" "${home}/pods.json" +set +e +out="$(TF_STUB_PLAN_EXIT=0 run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "unhealthy"; then + pass "unhealthy pods fail verification" +else + fail "unhealthy pods fail verification (rc=${rc})" + printf '%s\n' "$out" +fi + +read -r fx home stubs <<<"$(prepare_runtime vbootempty)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/ssh.log" +set +e +out="$(printf 'REBOOT\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 SSH_BOOT_MODE=empty-before run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "pre-reboot boot identity"; then + pass "empty pre-reboot boot ID fails before REBOOT" +else + fail "empty pre-reboot boot ID fails before REBOOT (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "empty pre-reboot boot ID must not report acceptance PASS" +else + pass "empty pre-reboot boot ID must not report acceptance PASS" +fi +if grep -Fq "sudo reboot" "${home}/ssh.log"; then + fail "empty pre-reboot boot ID must not issue reboot" +else + pass "empty pre-reboot boot ID must not issue reboot" +fi + +read -r fx home stubs <<<"$(prepare_runtime vbootfail)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +: >"${home}/ssh.log" +set +e +out="$(printf 'REBOOT\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 SSH_BOOT_MODE=fail-before run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "pre-reboot boot identity"; then + pass "unavailable pre-reboot boot ID fails before REBOOT" +else + fail "unavailable pre-reboot boot ID fails before REBOOT (rc=${rc})" + printf '%s\n' "$out" +fi +if grep -Fq "sudo reboot" "${home}/ssh.log"; then + fail "unavailable pre-reboot boot ID must not issue reboot" +else + pass "unavailable pre-reboot boot ID must not issue reboot" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "unavailable pre-reboot boot ID must not report acceptance PASS" +else + pass "unavailable pre-reboot boot ID must not report acceptance PASS" +fi + +read -r fx home stubs <<<"$(prepare_runtime vbootunch)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'REBOOT\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 SSH_BOOT_MODE=unchanged run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "did not change after reboot"; then + pass "unchanged boot ID fails verification" +else + fail "unchanged boot ID fails verification (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "unchanged boot ID must not report acceptance PASS" +else + pass "unchanged boot ID must not report acceptance PASS" +fi +if grep -Fq "sudo reboot" "${home}/ssh.log"; then + pass "unchanged boot ID still issued reboot" +else + fail "unchanged boot ID still issued reboot" +fi + +read -r fx home stubs <<<"$(prepare_runtime vbootafter)" +cp "${HELPER_DIR}/argo-ok.json" "${home}/argo.json" +cp "${HELPER_DIR}/pods-ok.json" "${home}/pods.json" +set +e +out="$(printf 'REBOOT\n' | TF_STUB_PLAN_EXIT=0 SITE_CHANGED=0 PRIV_CHANGED=0 SSH_BOOT_MODE=empty-after run_verify_env "$fx" "$home" "$stubs" 2>&1)" +rc=$? +set -e +if [[ "$rc" -ne 0 ]] && printf '%s' "$out" | grep -Fq "post-reboot boot identity"; then + pass "empty post-reboot boot ID fails verification" +else + fail "empty post-reboot boot ID fails verification (rc=${rc})" + printf '%s\n' "$out" +fi +if printf '%s' "$out" | grep -Fq "PASS: clean-room acceptance completed"; then + fail "empty post-reboot boot ID must not report acceptance PASS" +else + pass "empty post-reboot boot ID must not report acceptance PASS" +fi + +echo +echo "Summary: PASS=${PASS_COUNT} FAIL=${FAIL_COUNT}" +if [[ "$FAIL_COUNT" -gt 0 ]]; then + exit 1 +fi +exit 0 diff --git a/tests/unit/test_cloud_shell_execution_contracts.sh b/tests/unit/test_cloud_shell_execution_contracts.sh index 97ac8e6..d8187ea 100755 --- a/tests/unit/test_cloud_shell_execution_contracts.sh +++ b/tests/unit/test_cloud_shell_execution_contracts.sh @@ -112,6 +112,11 @@ def main() -> None: runbook = (ROOT / "docs/V2_CLEAN_ROOM_DEPLOYMENT.md").read_text(encoding="utf-8") helper = (ROOT / "tools/check-cloud-shell-readiness").read_text(encoding="utf-8") + if "python3.12" not in helper: + raise SystemExit("check-cloud-shell-readiness must require python3.12") + if re.search(r"(?m)^require_cmd python3$", helper): + raise SystemExit("check-cloud-shell-readiness must not treat generic python3 as the Ansible interpreter") + print("PASS: check-cloud-shell-readiness requires python3.12") for label, text in ( ("terraform/backend.hcl.example", backend_example), diff --git a/tools/bootstrap-cloud-shell b/tools/bootstrap-cloud-shell new file mode 100755 index 0000000..33b69c8 --- /dev/null +++ b/tools/bootstrap-cloud-shell @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Prepare a cloned repository's control-node environment up to the operator-input gate. +# Does not clone this repository. Does not deploy infrastructure. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=lib/clean-room-common.sh +# Dynamic SCRIPT_DIR path is not followed without shellcheck -x; the helper is linted separately. +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/clean-room-common.sh" + +usage() { + cat <<'EOF' +Usage: tools/bootstrap-cloud-shell [--root DIR] + +Prepare an already-cloned TradingChassis/infrastructure control node: + user-local Terraform ~> 1.15.0 (canonical 1.15.8) + dedicated python3.12 Ansible venv (ansible-core==2.21.2) + repository-pinned Ansible collections + operator input files from committed examples when missing + +Does not clone this repository. +Does not run terraform init/plan/apply, Ansible, SSH, or kubectl. +Does not overwrite existing operator input files. +Does not start tmux. + +After success, complete the private input files, then run tools/deploy-clean-room. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --root) + ROOT="$(cd "$2" && pwd)" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +require_cmd() { + if command -v "$1" >/dev/null 2>&1; then + clean_room_pass "command available: $1" + else + clean_room_die "missing command: $1" + fi +} + +copy_example_if_absent() { + local src="$1" + local dest="$2" + if [[ -e "$dest" ]]; then + clean_room_info "preserving existing operator file: ${dest#"$ROOT"/}" + return 0 + fi + if [[ ! -f "$src" ]]; then + clean_room_die "missing committed example: ${src#"$ROOT"/}" + fi + umask 077 + cp "$src" "$dest" + chmod 600 "$dest" + clean_room_pass "created ${dest#"$ROOT"/} from example (operator must complete it)" +} + +ensure_user_terraform() { + mkdir -p "$CLEAN_ROOM_TF_BIN_DIR" + local tf_bin="${CLEAN_ROOM_TF_BIN_DIR}/terraform" + export PATH="${CLEAN_ROOM_TF_BIN_DIR}:${PATH}" + + if [[ -e "$tf_bin" ]]; then + if [[ ! -x "$tf_bin" ]]; then + clean_room_die "incompatible Terraform path exists and is not executable: $tf_bin (move it manually)" + fi + local existing="" + existing="$("$tf_bin" version -json 2>/dev/null | "$(clean_room_python)" -c 'import json,sys; print(json.load(sys.stdin)["terraform_version"])' 2>/dev/null || true)" + if [[ -z "$existing" ]]; then + existing="$("$tf_bin" version 2>/dev/null | head -n1 | sed -n 's/^Terraform v//p' || true)" + fi + if clean_room_terraform_version_ok "$existing"; then + clean_room_pass "reusing user-local Terraform ${existing} at \$HOME/bin/terraform" + return 0 + fi + clean_room_die "incompatible Terraform ${existing:-unknown} exists at \$HOME/bin/terraform; move it manually before bootstrap" + fi + + local arch + if ! arch="$(clean_room_terraform_zip_arch)"; then + clean_room_die "unsupported architecture for Terraform zip: $(uname -m)" + fi + local os + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + if [[ "$os" != "linux" ]]; then + clean_room_die "Terraform bootstrap downloads the linux zip; use a Linux control node" + fi + + local version="$CLEAN_ROOM_TF_CANONICAL_VERSION" + local zip_name="terraform_${version}_${os}_${arch}.zip" + local base_url="https://releases.hashicorp.com/terraform/${version}" + local tmp + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap 'rm -rf "'"$tmp"'"' RETURN + + clean_room_info "downloading Terraform ${version} (${os}_${arch})" + curl --proto '=https' --proto-redir '=https' --tlsv1.2 -fsSL \ + "${base_url}/terraform_${version}_SHA256SUMS" \ + -o "${tmp}/SHA256SUMS" + curl --proto '=https' --proto-redir '=https' --tlsv1.2 -fsSL \ + "${base_url}/${zip_name}" \ + -o "${tmp}/${zip_name}" + + if ! grep -E " ${zip_name}\$" "${tmp}/SHA256SUMS" >"${tmp}/SHA256SUMS.selected"; then + clean_room_die "SHA256SUMS has no entry for ${zip_name}" + fi + if ! (cd "$tmp" && sha256sum -c SHA256SUMS.selected); then + clean_room_die "Terraform checksum mismatch; refusing to install" + fi + + mkdir -p "${tmp}/extract" + unzip -o -q "${tmp}/${zip_name}" -d "${tmp}/extract" + if [[ ! -f "${tmp}/extract/terraform" ]]; then + clean_room_die "Terraform archive did not contain a terraform binary" + fi + install -m 0755 "${tmp}/extract/terraform" "$tf_bin" + trap - RETURN + rm -rf "$tmp" + + local installed="" + installed="$("$tf_bin" version -json 2>/dev/null | "$(clean_room_python)" -c 'import json,sys; print(json.load(sys.stdin)["terraform_version"])' 2>/dev/null || true)" + if [[ -z "$installed" ]]; then + installed="$("$tf_bin" version | head -n1 | sed -n 's/^Terraform v//p')" + fi + if ! clean_room_terraform_version_ok "$installed"; then + clean_room_die "installed Terraform ${installed:-unknown} does not satisfy ~> 1.15.0" + fi + clean_room_pass "installed Terraform ${installed} at \$HOME/bin/terraform" +} + +venv_interpreter_is_python312() { + local venv_python="$1" + if [[ ! -x "$venv_python" ]]; then + return 1 + fi + "$venv_python" -c 'import sys; raise SystemExit(0 if sys.version_info[:2] == (3, 12) else 1)' +} + +ensure_ansible_venv() { + local venv="$CLEAN_ROOM_VENV_DIR" + local venv_python="${venv}/bin/python" + mkdir -p "$(dirname "$venv")" + + if [[ -e "$venv" ]]; then + if ! venv_interpreter_is_python312 "$venv_python"; then + clean_room_die "existing venv at \$HOME/.venvs/tradingchassis-ansible is not Python 3.12; recreate it manually" + fi + clean_room_pass "reusing dedicated Ansible venv at \$HOME/.venvs/tradingchassis-ansible" + else + python3.12 -m venv "$venv" + if ! venv_interpreter_is_python312 "$venv_python"; then + clean_room_die "created venv is not Python 3.12" + fi + clean_room_pass "created dedicated Ansible venv with python3.12" + fi + + "${venv}/bin/python" -m pip install --disable-pip-version-check \ + "ansible-core==${CLEAN_ROOM_ANSIBLE_CORE_PIN}" >/dev/null + if [[ ! -x "${venv}/bin/ansible-playbook" ]]; then + clean_room_die "ansible-playbook missing from dedicated venv" + fi + local ap_path + ap_path="$(PATH="${venv}/bin:${PATH}" command -v ansible-playbook)" + if [[ "$ap_path" != *".venvs/tradingchassis-ansible/"* ]]; then + clean_room_die "ansible-playbook does not resolve inside the dedicated venv" + fi + local ver + ver="$("${venv}/bin/ansible-playbook" --version 2>/dev/null | head -n1 || true)" + if printf '%s\n' "$ver" | grep -Eq 'ansible-playbook[[:space:]]+2\.9\.'; then + clean_room_die "Cloud Shell system Ansible 2.9 is unsupported" + fi + if ! printf '%s\n' "$ver" | grep -Fq "$CLEAN_ROOM_ANSIBLE_CORE_PIN"; then + clean_room_die "dedicated venv must provide ansible-core==${CLEAN_ROOM_ANSIBLE_CORE_PIN}" + fi + clean_room_pass "ansible-playbook is ${CLEAN_ROOM_ANSIBLE_CORE_PIN} in the dedicated venv" + + if [[ ! -f "${ROOT}/ansible/requirements.yml" ]]; then + clean_room_die "missing ansible/requirements.yml" + fi + "${venv}/bin/ansible-galaxy" collection install -r "${ROOT}/ansible/requirements.yml" >/dev/null + clean_room_pass "Ansible collections installed from ansible/requirements.yml" +} + +if [[ ! -f "${ROOT}/terraform/versions.tf" || ! -f "${ROOT}/ansible/playbooks/site.yml" ]]; then + clean_room_die "run this tool from a TradingChassis/infrastructure clone (--root)" +fi + +clean_room_warn_tmux + +require_cmd git +require_cmd curl +require_cmd unzip +require_cmd sha256sum +require_cmd python3.12 +require_cmd install +if command -v oci >/dev/null 2>&1; then + clean_room_pass "command available: oci" +else + clean_room_warn "oci CLI not found (expected in OCI Cloud Shell; required later by deploy)" +fi + +py_mm="$(python3.12 -c 'import sys; print("%d.%d" % sys.version_info[:2])')" +if [[ "$py_mm" != "3.12" ]]; then + clean_room_die "python3.12 must be Python 3.12.x" +fi + +ensure_user_terraform +ensure_ansible_venv +copy_example_if_absent "${ROOT}/terraform/backend.hcl.example" "${ROOT}/terraform/backend.hcl" +copy_example_if_absent "${ROOT}/terraform/terraform.tfvars.example" "${ROOT}/terraform/terraform.tfvars" +copy_example_if_absent \ + "${ROOT}/ansible/extra-vars/private-runtime.yml.example" \ + "${ROOT}/ansible/extra-vars/private-runtime.yml" + +export PATH="${CLEAN_ROOM_TF_BIN_DIR}:${CLEAN_ROOM_VENV_DIR}/bin:${PATH}" +"${SCRIPT_DIR}/check-cloud-shell-readiness" --root "$ROOT" + +cat </dev/null 2>&1; then + py_mm="$(python3.12 -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null || true)" + if [[ "$py_mm" == "3.12" ]]; then + pass "python3.12 is Python 3.12.x" + else + fail "python3.12 must be Python 3.12.x (found ${py_mm:-unknown})" + fi +fi if command -v oci >/dev/null 2>&1; then pass "command available: oci" else warn "oci CLI not found (expected in OCI Cloud Shell)" fi +readiness_python() { + if command -v python3.12 >/dev/null 2>&1; then + command -v python3.12 + elif command -v python3 >/dev/null 2>&1; then + command -v python3 + else + return 1 + fi +} + if command -v terraform >/dev/null 2>&1; then - tf_ver="$(terraform version -json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["terraform_version"])' 2>/dev/null || true)" + tf_ver="" + readiness_py="$(readiness_python || true)" + if [[ -n "$readiness_py" ]]; then + tf_ver="$(terraform version -json 2>/dev/null | "$readiness_py" -c 'import json,sys; print(json.load(sys.stdin)["terraform_version"])' 2>/dev/null || true)" + fi if [[ -z "$tf_ver" ]]; then tf_ver="$(terraform version | head -n1 | sed -n 's/^Terraform v//p')" fi # required_version ~> 1.15.0 means >=1.15.0,<1.16.0 - if python3 - "$tf_ver" <<'PY' + if [[ -n "$readiness_py" ]] && "$readiness_py" - "$tf_ver" <<'PY' import sys -ver = sys.argv[1].strip() -parts = [int(p) for p in ver.split(".")[:3]] +ver = sys.argv[1].strip().lstrip("v") +parts = [] +for token in ver.split("."): + digits = "".join(ch for ch in token if ch.isdigit()) + if digits == "": + break + parts.append(int(digits)) + if len(parts) == 3: + break while len(parts) < 3: parts.append(0) major, minor, patch = parts @@ -130,6 +159,15 @@ check_file() { fi } +placeholder_issue() { + local label="$1" + if [[ "$STRICT" -eq 1 ]]; then + fail "$label still contains placeholders" + else + warn "$label still contains placeholders" + fi +} + check_file "${ROOT}/terraform/backend.hcl.example" "backend example" check_file "${ROOT}/terraform/terraform.tfvars.example" "tfvars example" check_file "${ROOT}/ansible/extra-vars/private-runtime.yml.example" "private-runtime example" @@ -138,8 +176,8 @@ check_file "${ROOT}/docs/V2_CLEAN_ROOM_DEPLOYMENT.md" "clean-room runbook" if [[ -f "${ROOT}/terraform/backend.hcl" ]]; then pass "operator backend.hcl present" - if grep -E '<|>' "${ROOT}/terraform/backend.hcl" >/dev/null 2>&1; then - fail "terraform/backend.hcl still contains placeholders" + if grep -Eq '<|>' "${ROOT}/terraform/backend.hcl"; then + placeholder_issue "terraform/backend.hcl" else pass "terraform/backend.hcl has no angle-bracket placeholders" fi @@ -163,6 +201,11 @@ fi if [[ -f "${ROOT}/terraform/terraform.tfvars" ]]; then pass "operator terraform.tfvars present" + if grep -Eq '<|>' "${ROOT}/terraform/terraform.tfvars"; then + placeholder_issue "terraform/terraform.tfvars" + else + pass "terraform.tfvars has no angle-bracket placeholders" + fi if grep -Eq '^[[:space:]]*oci_auth[[:space:]]*=[[:space:]]*"APIKey"' "${ROOT}/terraform/terraform.tfvars"; then pass "terraform.tfvars selects APIKey auth" else @@ -176,6 +219,27 @@ else fi fi +if [[ -f "${ROOT}/ansible/extra-vars/private-runtime.yml" ]]; then + pass "operator private-runtime.yml present" + if grep -Eq '<|>' "${ROOT}/ansible/extra-vars/private-runtime.yml"; then + placeholder_issue "ansible/extra-vars/private-runtime.yml" + else + pass "private-runtime.yml has no angle-bracket placeholders" + fi + if grep -Eq '^[[:space:]]*private_runtime_config_vault_id:' "${ROOT}/ansible/extra-vars/private-runtime.yml" \ + && grep -Eq '^[[:space:]]*private_runtime_config_oci_region:' "${ROOT}/ansible/extra-vars/private-runtime.yml"; then + pass "private-runtime.yml declares required keys" + else + fail "private-runtime.yml must set private_runtime_config_vault_id and private_runtime_config_oci_region" + fi +else + if [[ "$STRICT" -eq 1 ]]; then + fail "operator private-runtime.yml missing" + else + warn "operator private-runtime.yml missing (copy from private-runtime.yml.example)" + fi +fi + oci_config="${HOME}/.oci/config" api_key="${HOME}/.oci/tradingchassis_api_key.pem" if [[ -f "$oci_config" ]]; then diff --git a/tools/deploy-clean-room b/tools/deploy-clean-room new file mode 100755 index 0000000..1eb8be3 --- /dev/null +++ b/tools/deploy-clean-room @@ -0,0 +1,365 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# State-aware clean-room deployment. Inspects live Terraform/Ansible/K8s state +# on each run. Never infers prior APPLY/FORMAT approval from an internal step file. +# Do not execute this tool from implementation/CI against live infrastructure. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=lib/clean-room-common.sh +# Dynamic SCRIPT_DIR path is not followed without shellcheck -x; the helper is linted separately. +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/clean-room-common.sh" + +SSH_ATTEMPTS="${CLEAN_ROOM_SSH_ATTEMPTS:-30}" +SSH_DELAY="${CLEAN_ROOM_SSH_DELAY:-10}" +ARGO_ATTEMPTS="${CLEAN_ROOM_ARGO_ATTEMPTS:-60}" +ARGO_DELAY="${CLEAN_ROOM_ARGO_DELAY:-15}" +WORKLOAD_ATTEMPTS="${CLEAN_ROOM_WORKLOAD_ATTEMPTS:-40}" +WORKLOAD_DELAY="${CLEAN_ROOM_WORKLOAD_DELAY:-15}" + +TF_DIR="" +ANSIBLE_PLAYBOOK="" +ANSIBLE_CONFIG_FILE="" +INVENTORY="" +PRIVATE_VARS="" +PLAN_FILE="" +ANSIBLE_LOG="" +INSTANCE_IP="" + +usage() { + cat <<'EOF' +Usage: tools/deploy-clean-room [--root DIR] + +Deploy TradingChassis V2 from current Terraform/Ansible/Kubernetes state. + +Gates that always require exact operator input when they apply: + APPLY Terraform mutation (skipped when the plan has no changes) + FORMAT blank scratch filesystem creation (skipped when the volume is already formatted) + +Does not reboot. +Does not destroy or replace Terraform resources. +Does not start tmux. +Does not remember APPLY/FORMAT from a previous invocation. + +After success, run tools/verify-clean-room for acceptance. +EOF +} + +cleanup() { + if [[ -n "${PLAN_FILE:-}" && -f "${PLAN_FILE:-}" ]]; then + rm -f "$PLAN_FILE" + fi + if [[ -n "${ANSIBLE_LOG:-}" && -f "${ANSIBLE_LOG:-}" ]]; then + rm -f "$ANSIBLE_LOG" + fi +} +trap cleanup EXIT + +while [[ $# -gt 0 ]]; do + case "$1" in + --root) + ROOT="$(cd "$2" && pwd)" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +TF_DIR="${ROOT}/terraform" +ANSIBLE_CONFIG_FILE="${ROOT}/ansible/ansible.cfg" +INVENTORY="${ROOT}/ansible/inventory/local.yml" +PRIVATE_VARS="${ROOT}/ansible/extra-vars/private-runtime.yml" +ANSIBLE_PLAYBOOK="${CLEAN_ROOM_VENV_DIR}/bin/ansible-playbook" + +require_operator_inputs() { + local path="$1" + local label="$2" + if [[ ! -f "$path" ]]; then + clean_room_die "missing operator input: $label" + fi + if clean_room_has_angle_placeholders "$path"; then + clean_room_die "$label still contains placeholders" + fi + clean_room_pass "$label is present without placeholders" +} + +tf() { + terraform -chdir="$TF_DIR" "$@" +} + +remote() { + ssh -n \ + -i "$CLEAN_ROOM_SSH_KEY" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=accept-new \ + -o ConnectTimeout=10 \ + "${CLEAN_ROOM_SSH_USER}@${INSTANCE_IP}" \ + "$@" +} + +wait_for_ssh() { + local attempt=1 + while (( attempt <= SSH_ATTEMPTS )); do + if remote true >/dev/null 2>&1; then + clean_room_pass "SSH is ready" + return 0 + fi + clean_room_info "waiting for SSH (${attempt}/${SSH_ATTEMPTS})" + sleep "$SSH_DELAY" + attempt=$((attempt + 1)) + done + clean_room_die "SSH was not ready before the bounded timeout" +} + +run_playbook() { + local log="$1" + local rc=0 + shift + set +e + ANSIBLE_CONFIG="$ANSIBLE_CONFIG_FILE" "$ANSIBLE_PLAYBOOK" "$@" >"$log" 2>&1 + rc=$? + cat "$log" + set +e + return "$rc" +} + +wait_for_argo() { + local tmp + tmp="$(mktemp)" + local attempt=1 + while (( attempt <= ARGO_ATTEMPTS )); do + if ! remote "sudo microk8s kubectl -n argocd get applications -o json" >"$tmp" 2>/dev/null; then + clean_room_info "waiting for Argo Applications (${attempt}/${ARGO_ATTEMPTS})" + sleep "$ARGO_DELAY" + attempt=$((attempt + 1)) + continue + fi + set +e + clean_room_eval_argo_json "$tmp" + local rc=$? + set -e + case "$rc" in + 0) + rm -f "$tmp" + return 0 + ;; + 1) + clean_room_info "Argo Applications not yet Synced+Healthy (${attempt}/${ARGO_ATTEMPTS})" + ;; + 2|3) + rm -f "$tmp" + clean_room_die "Argo Application set is empty or unhealthy" + ;; + *) + rm -f "$tmp" + clean_room_die "Argo Application evaluation failed" + ;; + esac + sleep "$ARGO_DELAY" + attempt=$((attempt + 1)) + done + rm -f "$tmp" + clean_room_die "Argo Applications did not become Synced and Healthy before timeout" +} + +wait_for_workloads() { + local tmp + tmp="$(mktemp)" + local attempt=1 + while (( attempt <= WORKLOAD_ATTEMPTS )); do + if ! remote "sudo microk8s kubectl get pods -A -o json" >"$tmp" 2>/dev/null; then + clean_room_info "waiting for Kubernetes workloads (${attempt}/${WORKLOAD_ATTEMPTS})" + sleep "$WORKLOAD_DELAY" + attempt=$((attempt + 1)) + continue + fi + set +e + clean_room_eval_pods_json "$tmp" + local rc=$? + set -e + case "$rc" in + 0) + rm -f "$tmp" + return 0 + ;; + 1) + clean_room_info "Kubernetes workloads not ready (${attempt}/${WORKLOAD_ATTEMPTS})" + ;; + 2) + rm -f "$tmp" + clean_room_die "Kubernetes workloads are unhealthy" + ;; + *) + rm -f "$tmp" + clean_room_die "Kubernetes workload evaluation failed" + ;; + esac + sleep "$WORKLOAD_DELAY" + attempt=$((attempt + 1)) + done + rm -f "$tmp" + clean_room_die "Kubernetes workloads were not healthy before timeout" +} + +if [[ ! -f "${ROOT}/terraform/versions.tf" || ! -f "${ROOT}/ansible/playbooks/site.yml" ]]; then + clean_room_die "run this tool from a TradingChassis/infrastructure clone (--root)" +fi + +clean_room_warn_tmux + +export PATH="${CLEAN_ROOM_TF_BIN_DIR}:${CLEAN_ROOM_VENV_DIR}/bin:${PATH}" + +if [[ ! -x "$ANSIBLE_PLAYBOOK" ]]; then + clean_room_die "dedicated Ansible venv is missing; run tools/bootstrap-cloud-shell" +fi +if [[ "$ANSIBLE_PLAYBOOK" != *".venvs/tradingchassis-ansible/"* ]]; then + clean_room_die "refusing system Ansible; expected \$HOME/.venvs/tradingchassis-ansible" +fi + +"${SCRIPT_DIR}/check-cloud-shell-readiness" --strict --root "$ROOT" + +require_operator_inputs "${TF_DIR}/backend.hcl" "terraform/backend.hcl" +require_operator_inputs "${TF_DIR}/terraform.tfvars" "terraform/terraform.tfvars" +require_operator_inputs "$PRIVATE_VARS" "ansible/extra-vars/private-runtime.yml" + +if ! command -v oci >/dev/null 2>&1; then + clean_room_die "oci CLI is required for the APIKey identity preflight" +fi +clean_room_info "running read-only OCI APIKey identity preflight" +oci iam region list \ + --config-file "${HOME}/.oci/config" \ + --profile "$CLEAN_ROOM_OCI_PROFILE" \ + --auth api_key >/dev/null +clean_room_pass "OCI APIKey identity preflight succeeded" + +PLAN_FILE="$(mktemp)" +clean_room_info "terraform init" +tf init -input=false -backend-config=backend.hcl >/dev/null +clean_room_info "terraform validate" +tf validate >/dev/null +clean_room_pass "terraform validate succeeded" + +set +e +tf plan -input=false -detailed-exitcode -out="$PLAN_FILE" +plan_rc=$? +set -e +case "$plan_rc" in + 0) + clean_room_pass "Terraform plan has no changes; skipping APPLY" + ;; + 1) + clean_room_die "terraform plan failed" + ;; + 2) + show_json="$(mktemp)" + tf show -json "$PLAN_FILE" >"$show_json" + set +e + destructive_out="$(clean_room_plan_is_destructive "$show_json" 2>&1)" + destructive_rc=$? + set -e + rm -f "$show_json" + if [[ "$destructive_rc" -eq 2 ]]; then + printf '%s\n' "$destructive_out" + clean_room_die "Terraform plan contains delete or replace actions; refusing to apply" + fi + if [[ "$destructive_rc" -ne 0 ]]; then + clean_room_die "unable to inspect Terraform plan JSON; refusing to apply" + fi + tf show "$PLAN_FILE" + cat <<'EOF' + +TERRAFORM APPLY GATE +Type APPLY to apply this saved plan. Any other input stops without applying. +EOF + if ! clean_room_require_exact_input "APPLY"; then + clean_room_die "APPLY was not confirmed; Terraform was not changed" + fi + clean_room_info "applying the saved Terraform plan" + tf apply -input=false "$PLAN_FILE" + set +e + tf plan -input=false -detailed-exitcode >/dev/null + post_rc=$? + set -e + if [[ "$post_rc" -ne 0 ]]; then + clean_room_die "post-apply Terraform plan is not no-change; stopping before Ansible" + fi + clean_room_pass "post-apply Terraform plan has no changes" + ;; + *) + clean_room_die "unexpected terraform plan exit code ${plan_rc}" + ;; +esac + +"${SCRIPT_DIR}/render-ansible-inventory" --terraform-dir "$TF_DIR" --output "$INVENTORY" >/dev/null +clean_room_pass "rendered Ansible inventory" + +INSTANCE_IP="$(tf output -raw instance_public_ip)" +if [[ -z "$INSTANCE_IP" ]]; then + clean_room_die "Terraform output instance_public_ip is empty" +fi +wait_for_ssh + +ANSIBLE_LOG="$(mktemp)" +site_args=(-i "$INVENTORY" "${ROOT}/ansible/playbooks/site.yml") +set +e +run_playbook "$ANSIBLE_LOG" "${site_args[@]}" +site_rc=$? +set -e +if [[ "$site_rc" -eq 0 ]]; then + clean_room_pass "site.yml converged without format authorization" +else + if clean_room_is_blank_scratch_gate "$ANSIBLE_LOG"; then + cat <<'EOF' + +SCRATCH FORMAT GATE +This authorizes filesystem creation on the Ansible-discovered Terraform-managed +blank scratch candidate. Type FORMAT to continue. Any other input stops. +EOF + if ! clean_room_require_exact_input "FORMAT"; then + clean_room_die "FORMAT was not confirmed; scratch was not formatted" + fi + set +e + run_playbook "$ANSIBLE_LOG" "${site_args[@]}" -e scratch_storage_allow_format=true + format_rc=$? + set -e + if [[ "$format_rc" -ne 0 ]]; then + clean_room_die "site.yml failed after authorized FORMAT" + fi + clean_room_pass "site.yml converged with one-time FORMAT authorization" + else + clean_room_die "site.yml failed; FORMAT was not offered because this is not the blank-scratch gate" + fi +fi + +clean_room_info "running private-runtime-config.yml" +set +e +run_playbook "$ANSIBLE_LOG" \ + -i "$INVENTORY" \ + -e "@${PRIVATE_VARS}" \ + "${ROOT}/ansible/playbooks/private-runtime-config.yml" +private_rc=$? +set -e +if [[ "$private_rc" -ne 0 ]]; then + clean_room_die "private-runtime-config.yml failed" +fi +clean_room_pass "private-runtime-config.yml succeeded" + +wait_for_argo +wait_for_workloads + +cat <<'EOF' + +PASS: clean-room deployment convergence completed. +Next: tools/verify-clean-room +EOF diff --git a/tools/lib/clean-room-common.sh b/tools/lib/clean-room-common.sh new file mode 100755 index 0000000..f5e4a37 --- /dev/null +++ b/tools/lib/clean-room-common.sh @@ -0,0 +1,297 @@ +#!/usr/bin/env bash +# Shared helpers for TradingChassis clean-room operator tools. +# Source only. Does not mutate infrastructure by itself. +# shellcheck shell=bash +# Shared CLEAN_ROOM_* constants are consumed by scripts that source this helper. +# They appear unused when this file is linted standalone. +# shellcheck disable=SC2034 + +CLEAN_ROOM_TF_CANONICAL_VERSION="1.15.8" +CLEAN_ROOM_ANSIBLE_CORE_PIN="2.21.2" +CLEAN_ROOM_VENV_DIR="${HOME}/.venvs/tradingchassis-ansible" +CLEAN_ROOM_TF_BIN_DIR="${HOME}/bin" +CLEAN_ROOM_SSH_USER="ubuntu" +CLEAN_ROOM_SSH_KEY="${HOME}/.ssh/tradingchassis" +CLEAN_ROOM_OCI_PROFILE="tradingchassis" +CLEAN_ROOM_SCRATCH_MOUNT="/mnt/scratch" +CLEAN_ROOM_BLANK_SCRATCH_MSG="The scratch volume has no filesystem. Set scratch_storage_allow_format=true" + +clean_room_info() { printf 'INFO: %s\n' "$1"; } +clean_room_pass() { printf 'PASS: %s\n' "$1"; } +clean_room_warn() { printf 'WARN: %s\n' "$1"; } +clean_room_fail() { printf 'FAIL: %s\n' "$1"; } + +clean_room_die() { + clean_room_fail "$1" + exit "${2:-1}" +} + +clean_room_python() { + if command -v python3.12 >/dev/null 2>&1; then + command -v python3.12 + return 0 + fi + if command -v python3 >/dev/null 2>&1; then + command -v python3 + return 0 + fi + return 1 +} + +clean_room_warn_tmux() { + if [[ -z "${TMUX:-}" ]]; then + cat <<'EOF' +WARNING: tmux not detected. +Running the clean-room workflow inside tmux is recommended for Cloud Shell +browser/network disconnect resilience. +EOF + else + clean_room_info "tmux session detected" + fi +} + +clean_room_terraform_zip_arch() { + local machine="${1:-$(uname -m)}" + case "$machine" in + aarch64|arm64) + printf '%s\n' arm64 + ;; + x86_64|amd64) + printf '%s\n' amd64 + ;; + *) + return 1 + ;; + esac +} + +clean_room_terraform_version_ok() { + local ver="${1:-}" + local py + py="$(clean_room_python)" || return 1 + "$py" - "$ver" <<'PY' +import sys + +ver = sys.argv[1].strip().lstrip("v") +parts = [] +for token in ver.split("."): + digits = "".join(ch for ch in token if ch.isdigit()) + if digits == "": + break + parts.append(int(digits)) + if len(parts) == 3: + break +while len(parts) < 3: + parts.append(0) +major, minor, patch = parts +ok = (major, minor, patch) >= (1, 15, 0) and (major, minor) < (1, 16) +sys.exit(0 if ok else 1) +PY +} + +clean_room_has_angle_placeholders() { + local path="$1" + grep -Eq '<|>' "$path" +} + +clean_room_require_exact_input() { + local expected="$1" + local reply="" + if ! IFS= read -r reply; then + reply="" + fi + if [[ "$reply" != "$expected" ]]; then + return 1 + fi + return 0 +} + +clean_room_is_blank_scratch_gate() { + local log="$1" + grep -Fq "$CLEAN_ROOM_BLANK_SCRATCH_MSG" "$log" +} + +clean_room_plan_is_destructive() { + local json_file="$1" + local py + py="$(clean_room_python)" || return 2 + "$py" - "$json_file" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, encoding="utf-8") as handle: + plan = json.load(handle) + +destructive = [] +for change in plan.get("resource_changes") or []: + actions = list((change.get("change") or {}).get("actions") or []) + if "delete" in actions: + address = change.get("address") or "unknown" + destructive.append(f"{address} actions={actions}") + +if destructive: + for line in destructive: + print(line) + sys.exit(2) +sys.exit(0) +PY +} + +clean_room_parse_play_recap() { + local log="$1" + local require_zero_changed="${2:-1}" + local py + py="$(clean_room_python)" || return 2 + "$py" - "$log" "$require_zero_changed" <<'PY' +import re +import sys + +path = sys.argv[1] +require_zero_changed = sys.argv[2] != "0" +text = open(path, encoding="utf-8", errors="replace").read() +recap = False +hosts = [] +pattern = re.compile( + r"^(\S+)\s+:\s+ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)" +) +for line in text.splitlines(): + if line.startswith("PLAY RECAP"): + recap = True + continue + if not recap: + continue + match = pattern.match(line.strip()) + if match: + hosts.append( + { + "host": match.group(1), + "ok": int(match.group(2)), + "changed": int(match.group(3)), + "unreachable": int(match.group(4)), + "failed": int(match.group(5)), + } + ) + +if not hosts: + print("FAIL: PLAY RECAP not found or not parseable") + sys.exit(2) + +errors = [] +for host in hosts: + if host["unreachable"] != 0: + errors.append(f"{host['host']} unreachable={host['unreachable']}") + if host["failed"] != 0: + errors.append(f"{host['host']} failed={host['failed']}") + if require_zero_changed and host["changed"] != 0: + errors.append(f"{host['host']} changed={host['changed']}") + +if errors: + print("FAIL: " + "; ".join(errors)) + sys.exit(1) +print("PASS: PLAY RECAP ok changed=0 unreachable=0 failed=0") +sys.exit(0) +PY +} + +clean_room_eval_argo_json() { + local json_file="$1" + local py + py="$(clean_room_python)" || return 2 + "$py" - "$json_file" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1], encoding="utf-8")) +items = data.get("items") +if not isinstance(items, list) or not items: + print("FAIL: no Argo Applications found") + sys.exit(2) + +immediate = [] +pending = [] +for item in items: + name = ((item.get("metadata") or {}).get("name")) or "unnamed" + status = item.get("status") or {} + sync = ((status.get("sync") or {}).get("status")) or "" + health = ((status.get("health") or {}).get("status")) or "" + if health in {"Degraded", "Missing", "Unknown"} or sync in {"Unknown"}: + immediate.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}") + continue + if sync != "Synced" or health != "Healthy": + pending.append(f"{name} sync={sync or 'unset'} health={health or 'unset'}") + +if immediate: + print("FAIL: " + "; ".join(immediate)) + sys.exit(3) +if pending: + print("WAIT: " + "; ".join(pending)) + sys.exit(1) +print("PASS: all Argo Applications are Synced and Healthy") +sys.exit(0) +PY +} + +clean_room_eval_pods_json() { + local json_file="$1" + local py + py="$(clean_room_python)" || return 2 + "$py" - "$json_file" <<'PY' +import json +import sys + +UNHEALTHY_WAITING = { + "CrashLoopBackOff", + "ImagePullBackOff", + "ErrImagePull", + "CreateContainerError", + "InvalidImageName", +} + +data = json.load(open(sys.argv[1], encoding="utf-8")) +items = data.get("items") +if not isinstance(items, list) or not items: + print("WAIT: no Pods found") + sys.exit(1) + +unhealthy = [] +pending = [] +for item in items: + metadata = item.get("metadata") or {} + namespace = metadata.get("namespace") or "unknown" + status = item.get("status") or {} + phase = status.get("phase") or "" + if phase == "Succeeded": + continue + reasons = [] + for cs in status.get("containerStatuses") or []: + waiting = ((cs.get("state") or {}).get("waiting") or {}) + reason = waiting.get("reason") or "" + if reason: + reasons.append(reason) + last_waiting = ((cs.get("lastState") or {}).get("waiting") or {}) + last_reason = last_waiting.get("reason") or "" + if last_reason in UNHEALTHY_WAITING: + reasons.append(last_reason) + if phase == "Failed" or any(reason in UNHEALTHY_WAITING for reason in reasons): + shown = ",".join(reasons) if reasons else phase + unhealthy.append(f"namespace={namespace} phase={phase} reason={shown}") + continue + if phase == "Running": + statuses = status.get("containerStatuses") or [] + if statuses and all(cs.get("ready") for cs in statuses): + continue + pending.append(f"namespace={namespace} phase={phase} reason=not-ready") + continue + pending.append(f"namespace={namespace} phase={phase or 'unset'}") + +if unhealthy: + print("FAIL: " + "; ".join(unhealthy)) + sys.exit(2) +if pending: + print("WAIT: " + "; ".join(pending)) + sys.exit(1) +print("PASS: Kubernetes workloads are healthy") +sys.exit(0) +PY +} diff --git a/tools/validate-safe b/tools/validate-safe index 8fe9459..a4905e8 100755 --- a/tools/validate-safe +++ b/tools/validate-safe @@ -69,6 +69,10 @@ run_syntax_checks() { syntax_check_file "$CHECKER" syntax_check_file "${ROOT}/tools/validate-safe" + syntax_check_file "${ROOT}/tools/bootstrap-cloud-shell" + syntax_check_file "${ROOT}/tools/deploy-clean-room" + syntax_check_file "${ROOT}/tools/verify-clean-room" + syntax_check_file "${ROOT}/tools/lib/clean-room-common.sh" syntax_check_file "$UNIT_TEST" syntax_check_file "$METADATA_UNIT_TEST" @@ -112,6 +116,12 @@ run_shellcheck_if_present() { done shopt -u nullglob targets+=("$CHECKER" "${ROOT}/tools/validate-safe" "$UNIT_TEST" "$METADATA_UNIT_TEST") + targets+=( + "${ROOT}/tools/bootstrap-cloud-shell" + "${ROOT}/tools/deploy-clean-room" + "${ROOT}/tools/verify-clean-room" + "${ROOT}/tools/lib/clean-room-common.sh" + ) if shellcheck "${targets[@]}"; then pass "shellcheck" diff --git a/tools/verify-clean-room b/tools/verify-clean-room new file mode 100755 index 0000000..70fbd7d --- /dev/null +++ b/tools/verify-clean-room @@ -0,0 +1,370 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# Clean-room acceptance against an already-deployed environment. +# Verification never repairs Terraform drift and never formats scratch. +# Do not execute this tool from implementation/CI against live infrastructure. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +# shellcheck source=lib/clean-room-common.sh +# Dynamic SCRIPT_DIR path is not followed without shellcheck -x; the helper is linted separately. +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/clean-room-common.sh" + +SSH_ATTEMPTS="${CLEAN_ROOM_SSH_ATTEMPTS:-30}" +SSH_DELAY="${CLEAN_ROOM_SSH_DELAY:-10}" +ARGO_ATTEMPTS="${CLEAN_ROOM_ARGO_ATTEMPTS:-60}" +ARGO_DELAY="${CLEAN_ROOM_ARGO_DELAY:-15}" +WORKLOAD_ATTEMPTS="${CLEAN_ROOM_WORKLOAD_ATTEMPTS:-40}" +WORKLOAD_DELAY="${CLEAN_ROOM_WORKLOAD_DELAY:-15}" +REBOOT_DROP_ATTEMPTS="${CLEAN_ROOM_REBOOT_DROP_ATTEMPTS:-30}" +REBOOT_DROP_DELAY="${CLEAN_ROOM_REBOOT_DROP_DELAY:-2}" +REBOOT_RETURN_ATTEMPTS="${CLEAN_ROOM_REBOOT_RETURN_ATTEMPTS:-60}" +REBOOT_RETURN_DELAY="${CLEAN_ROOM_REBOOT_RETURN_DELAY:-5}" +MICROK8S_TIMEOUT="${CLEAN_ROOM_MICROK8S_TIMEOUT:-180}" + +TF_DIR="" +ANSIBLE_PLAYBOOK="" +ANSIBLE_CONFIG_FILE="" +INVENTORY="" +PRIVATE_VARS="" +ANSIBLE_LOG="" +INSTANCE_IP="" +BOOT_ID_BEFORE="" + +usage() { + cat <<'EOF' +Usage: tools/verify-clean-room [--root DIR] + +Accept an already-deployed TradingChassis V2 clean-room environment. + +Requires: + Terraform no-change plan (does not apply) + SSH, scratch mount, Argo Synced+Healthy, healthy workloads + private-runtime-config.yml changed=0 + site.yml changed=0 without scratch format authorization + exact REBOOT confirmation + post-reboot SSH, MicroK8s, scratch, Argo, and workload health + +Does not start tmux. +Does not repair drift. +Does not pass scratch_storage_allow_format=true. +EOF +} + +cleanup() { + if [[ -n "${ANSIBLE_LOG:-}" && -f "${ANSIBLE_LOG:-}" ]]; then + rm -f "$ANSIBLE_LOG" + fi +} +trap cleanup EXIT + +while [[ $# -gt 0 ]]; do + case "$1" in + --root) + ROOT="$(cd "$2" && pwd)" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +TF_DIR="${ROOT}/terraform" +ANSIBLE_CONFIG_FILE="${ROOT}/ansible/ansible.cfg" +INVENTORY="${ROOT}/ansible/inventory/local.yml" +PRIVATE_VARS="${ROOT}/ansible/extra-vars/private-runtime.yml" +ANSIBLE_PLAYBOOK="${CLEAN_ROOM_VENV_DIR}/bin/ansible-playbook" + +tf() { + terraform -chdir="$TF_DIR" "$@" +} + +remote() { + ssh -n \ + -i "$CLEAN_ROOM_SSH_KEY" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=accept-new \ + -o ConnectTimeout=10 \ + "${CLEAN_ROOM_SSH_USER}@${INSTANCE_IP}" \ + "$@" +} + +wait_for_ssh() { + local attempts="${1:-$SSH_ATTEMPTS}" + local delay="${2:-$SSH_DELAY}" + local attempt=1 + while (( attempt <= attempts )); do + if remote true >/dev/null 2>&1; then + clean_room_pass "SSH is ready" + return 0 + fi + clean_room_info "waiting for SSH (${attempt}/${attempts})" + sleep "$delay" + attempt=$((attempt + 1)) + done + clean_room_die "SSH was not ready before the bounded timeout" +} + +read_boot_id() { + local raw="" + if ! raw="$(remote "cat /proc/sys/kernel/random/boot_id")"; then + return 1 + fi + raw="${raw//$'\r'/}" + raw="${raw//$'\n'/}" + raw="${raw#"${raw%%[![:space:]]*}"}" + raw="${raw%"${raw##*[![:space:]]}"}" + if [[ -z "$raw" ]]; then + return 1 + fi + printf '%s\n' "$raw" +} + +run_playbook() { + local log="$1" + local rc=0 + shift + set +e + ANSIBLE_CONFIG="$ANSIBLE_CONFIG_FILE" "$ANSIBLE_PLAYBOOK" "$@" >"$log" 2>&1 + rc=$? + cat "$log" + set +e + return "$rc" +} + +require_zero_change_playbook() { + local label="$1" + shift + ANSIBLE_LOG="$(mktemp)" + set +e + run_playbook "$ANSIBLE_LOG" "$@" + local rc=$? + set -e + if [[ "$rc" -ne 0 ]]; then + clean_room_die "${label} failed" + fi + set +e + clean_room_parse_play_recap "$ANSIBLE_LOG" 1 + local recap_rc=$? + set -e + rm -f "$ANSIBLE_LOG" + ANSIBLE_LOG="" + if [[ "$recap_rc" -ne 0 ]]; then + clean_room_die "${label} is not idempotent (require changed=0 unreachable=0 failed=0)" + fi + clean_room_pass "${label} PLAY RECAP changed=0" +} + +eval_argo_now() { + local tmp rc=0 + tmp="$(mktemp)" + if ! remote "sudo microk8s kubectl -n argocd get applications -o json" >"$tmp" 2>/dev/null; then + rm -f "$tmp" + return 1 + fi + set +e + clean_room_eval_argo_json "$tmp" + rc=$? + rm -f "$tmp" + set +e + return "$rc" +} + +wait_for_argo() { + local attempt=1 + while (( attempt <= ARGO_ATTEMPTS )); do + set +e + eval_argo_now + local rc=$? + set -e + case "$rc" in + 0) + return 0 + ;; + 1) + clean_room_info "Argo Applications not yet Synced+Healthy (${attempt}/${ARGO_ATTEMPTS})" + ;; + 2|3) + clean_room_die "Argo Application set is empty or unhealthy" + ;; + *) + clean_room_info "waiting for Argo Applications (${attempt}/${ARGO_ATTEMPTS})" + ;; + esac + sleep "$ARGO_DELAY" + attempt=$((attempt + 1)) + done + clean_room_die "Argo Applications did not become Synced and Healthy before timeout" +} + +eval_pods_now() { + local tmp rc=0 + tmp="$(mktemp)" + if ! remote "sudo microk8s kubectl get pods -A -o json" >"$tmp" 2>/dev/null; then + rm -f "$tmp" + return 1 + fi + set +e + clean_room_eval_pods_json "$tmp" + rc=$? + rm -f "$tmp" + set +e + return "$rc" +} + +wait_for_workloads() { + local attempt=1 + while (( attempt <= WORKLOAD_ATTEMPTS )); do + set +e + eval_pods_now + local rc=$? + set -e + case "$rc" in + 0) + return 0 + ;; + 1) + clean_room_info "Kubernetes workloads not ready (${attempt}/${WORKLOAD_ATTEMPTS})" + ;; + 2) + clean_room_die "Kubernetes workloads are unhealthy" + ;; + *) + clean_room_info "waiting for Kubernetes workloads (${attempt}/${WORKLOAD_ATTEMPTS})" + ;; + esac + sleep "$WORKLOAD_DELAY" + attempt=$((attempt + 1)) + done + clean_room_die "Kubernetes workloads were not healthy before timeout" +} + +verify_scratch_mount() { + if ! remote "mountpoint -q ${CLEAN_ROOM_SCRATCH_MOUNT}"; then + clean_room_die "scratch mount ${CLEAN_ROOM_SCRATCH_MOUNT} is not mounted" + fi + clean_room_pass "scratch mount is present" +} + +verify_microk8s() { + if ! remote "sudo microk8s status --wait-ready --timeout ${MICROK8S_TIMEOUT}"; then + clean_room_die "MicroK8s is not ready" + fi + clean_room_pass "MicroK8s is ready" +} + +if [[ ! -f "${ROOT}/terraform/versions.tf" || ! -f "${ROOT}/ansible/playbooks/site.yml" ]]; then + clean_room_die "run this tool from a TradingChassis/infrastructure clone (--root)" +fi + +clean_room_warn_tmux + +export PATH="${CLEAN_ROOM_TF_BIN_DIR}:${CLEAN_ROOM_VENV_DIR}/bin:${PATH}" + +if [[ ! -x "$ANSIBLE_PLAYBOOK" ]]; then + clean_room_die "dedicated Ansible venv is missing; run tools/bootstrap-cloud-shell" +fi + +"${SCRIPT_DIR}/check-cloud-shell-readiness" --strict --root "$ROOT" + +if [[ ! -f "${TF_DIR}/backend.hcl" || ! -f "${TF_DIR}/terraform.tfvars" || ! -f "$PRIVATE_VARS" ]]; then + clean_room_die "operator input files are missing" +fi +if clean_room_has_angle_placeholders "${TF_DIR}/backend.hcl" \ + || clean_room_has_angle_placeholders "${TF_DIR}/terraform.tfvars" \ + || clean_room_has_angle_placeholders "$PRIVATE_VARS"; then + clean_room_die "operator input files still contain placeholders" +fi + +clean_room_info "terraform init (non-mutating)" +tf init -input=false -backend-config=backend.hcl >/dev/null +set +e +tf plan -input=false -detailed-exitcode >/dev/null +plan_rc=$? +set -e +case "$plan_rc" in + 0) + clean_room_pass "Terraform plan has no changes" + ;; + 2) + clean_room_die "Terraform drift detected; verification does not apply changes" + ;; + *) + clean_room_die "terraform plan failed; verification does not apply changes" + ;; +esac + +"${SCRIPT_DIR}/render-ansible-inventory" --terraform-dir "$TF_DIR" --output "$INVENTORY" >/dev/null +INSTANCE_IP="$(tf output -raw instance_public_ip)" +if [[ -z "$INSTANCE_IP" ]]; then + clean_room_die "Terraform output instance_public_ip is empty" +fi +wait_for_ssh +verify_scratch_mount +wait_for_argo +wait_for_workloads + +require_zero_change_playbook "private-runtime-config.yml" \ + -i "$INVENTORY" \ + -e "@${PRIVATE_VARS}" \ + "${ROOT}/ansible/playbooks/private-runtime-config.yml" + +require_zero_change_playbook "site.yml" \ + -i "$INVENTORY" \ + "${ROOT}/ansible/playbooks/site.yml" + +if ! BOOT_ID_BEFORE="$(read_boot_id)"; then + clean_room_die "could not read pre-reboot boot identity" +fi +cat <<'EOF' + +REBOOT GATE +Type REBOOT to reboot the node and continue acceptance. Any other input stops +without reboot and does not report clean-room acceptance PASS. +EOF +if ! clean_room_require_exact_input "REBOOT"; then + clean_room_die "REBOOT was not confirmed; clean-room acceptance is incomplete" +fi + +clean_room_info "issuing remote reboot" +set +e +remote "sudo reboot" +set -e + +drop_attempt=1 +while (( drop_attempt <= REBOOT_DROP_ATTEMPTS )); do + if ! remote true >/dev/null 2>&1; then + break + fi + sleep "$REBOOT_DROP_DELAY" + drop_attempt=$((drop_attempt + 1)) +done + +wait_for_ssh "$REBOOT_RETURN_ATTEMPTS" "$REBOOT_RETURN_DELAY" +if ! BOOT_ID_AFTER="$(read_boot_id)"; then + clean_room_die "could not read post-reboot boot identity" +fi +if [[ "$BOOT_ID_BEFORE" == "$BOOT_ID_AFTER" ]]; then + clean_room_die "host boot identity did not change after reboot" +fi +clean_room_pass "host rebooted" + +verify_microk8s +verify_scratch_mount +wait_for_argo +wait_for_workloads + +cat <<'EOF' + +PASS: clean-room acceptance completed. +EOF