fix(ci): key rollback-aws-fargate on the Fargate Terraform state

.github/workflows/ci.yml

@@ -824,6 +824,40 @@ jobs:
       - name: Run RDS scope self-tests
         run: bash scripts/test-rds-deletion-protection-scope.sh
 
+  # Assert that every job applying a `compute_platform` writes the Terraform
+  # state namespace that platform owns: `compute_platform=lambda` into
+  # github-<env>/, `compute_platform=fargate` into github-fargate-<env>/. The
+  # AWS environment is one Terraform root applied twice into two state objects,
+  # nothing in Terraform ties the backend key to the platform, and a job that
+  # pairs them wrongly initialises, plans and applies cleanly while rewriting
+  # the other platform's stack and recording it in the wrong state file. That
+  # was #1811, where rollback.yml's Fargate rollback keyed on the Lambda
+  # namespace, so both rollback jobs wrote the same object. A workflow run
+  # proves nothing about this, so the pairing is asserted as text.
+  # Both directions are asserted, and the positive one first: the seven real
+  # pairings are named and checked before any absence, since a scan that
+  # recognizes no state-writing job has no violations either. The negative half
+  # is checked over a GLOB of .github/workflows and scripts/, not a list of
+  # known files, so a job added later is covered without anyone naming it.
+  # Fast (shell only), so it always runs.
+  aws-tfstate-platform-key:
+    name: AWS Terraform state namespace per platform
+    runs-on: ubuntu-latest
+    # Same shape as ecr-delete-selection above: this job checks out the tree and
+    # runs a shell script against it, so the repository-default read/write token
+    # is narrowed to `contents: read`.
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          persist-credentials: false
+
+      - name: Run state namespace self-tests
+        run: bash scripts/test-aws-tfstate-platform-key.sh
+
   # Assert that no Terraform file declares an azurerm_key_vault_access_policy
   # resource. This project's only Key Vault sets enable_rbac_authorization =
   # true, and an RBAC-enabled vault ignores access policies entirely, so such a
@@ -863,6 +897,7 @@ jobs:
       - gcp-secret-scope
       - ecr-delete-selection
       - rds-deletion-protection-scope
+      - aws-tfstate-platform-key
       - azure-kv-access-policy
     if: always()
 

.github/workflows/rollback.yml

@@ -273,19 +273,21 @@ jobs:
     timeout-minutes: 30
     needs: validate
     if: inputs.cloud == 'aws-fargate'
-    # NOTE the group is `aws-tfstate-*`, NOT `aws-fargate-tfstate-*`. Despite
-    # the job name, the state key this job builds below is
-    # `github-<environment>/terraform.tfstate` -- the LAMBDA namespace -- not
-    # `github-fargate-<environment>/` as deploy-aws-fargate.yml uses. The group
-    # must name the object this job actually locks, or it would serialize
-    # against a state file it never touches while writing one unguarded, which
-    # is #1806 reproduced in a new place. That namespace mismatch is a real
-    # pre-existing defect (a Fargate rollback applies into the Lambda state),
-    # tracked in #1811 rather than changed here, because moving the key changes
-    # which infrastructure a rollback rewrites. When #1811 lands, this group
-    # moves to `aws-fargate-tfstate-*` in the same commit as the key.
+    # `terraform apply` against
+    # s3://<bucket>/github-fargate-<environment>/terraform.tfstate, the same
+    # object deploy-aws-fargate.yml, cleanup-staging.yml and
+    # destroy-fargate-dev.yml write, so it takes the same concurrency group
+    # (#1806). Until #1811 this job built the key from the LAMBDA namespace
+    # (`github-<environment>/`) and so carried `aws-tfstate-*` to match; the
+    # group and the key move together, because a group that does not name the
+    # object the job locks serializes against a state file it never touches
+    # while writing one unguarded. `inputs.environment` is a required `choice`
+    # constrained to dev|staging|prod, so the suffix is never empty; it is the
+    # same value this job interpolates into the state key below.
+    # scripts/test-aws-tfstate-platform-key.sh is what keeps the key and the
+    # `compute_platform` this job applies from drifting apart again.
     concurrency:
-      group: aws-tfstate-${{ inputs.environment }}
+      group: aws-fargate-tfstate-${{ inputs.environment }}
       cancel-in-progress: false
     permissions:
       id-token: write
@@ -343,7 +345,7 @@ jobs:
           IMAGE_URI: ${{ needs.validate.outputs.image_uri }}
         run: |
           set -euo pipefail
-          printf '%s\nkey = "github-%s/terraform.tfstate"\n' "$TF_BACKEND" "$ENVIRONMENT" > /tmp/backend.tfbackend
+          printf '%s\nkey = "github-fargate-%s/terraform.tfstate"\n' "$TF_BACKEND" "$ENVIRONMENT" > /tmp/backend.tfbackend
           cd terraform/environments/aws
           terraform init -backend-config=/tmp/backend.tfbackend
 

scripts/lib/code-scan-awk.sh

@@ -2,13 +2,14 @@
 # code-scan-awk.sh
 #
 # Shared awk helper functions for the guard suites that scan workflow and shell
-# sources for destructive commands: test-ecr-delete-selection.sh and
-# test-rds-deletion-protection-scope.sh. Sourced, not executed; it defines one
-# variable, AWK_CODE_FUNCS, to be prepended to an awk program.
+# sources for what a step actually runs: test-ecr-delete-selection.sh,
+# test-rds-deletion-protection-scope.sh and test-aws-tfstate-platform-key.sh.
+# Sourced, not executed; it defines one variable, AWK_CODE_FUNCS, to be
+# prepended to an awk program.
 #
 # Shared rather than copied because these functions encode the rule that
-# separates code that RUNS a command from prose that only mentions it, and both
-# suites are wrong in the same way if that rule drifts in one of them. A guard
+# separates code that RUNS a command from prose that only mentions it, and every
+# suite is wrong in the same way if that rule drifts in one of them. A guard
 # that fires on a comment constrains what may be WRITTEN about a command, which
 # is the "the string is present somewhere" mistake the selector these suites
 # guard exists to remove, one level up.
@@ -52,40 +53,53 @@
 
 # build_swept_scripts SCRIPTS_DIR
 #
-# Sets SWEPT_SCRIPTS to every `*.sh` directly under SCRIPTS_DIR and under
-# SCRIPTS_DIR/lib, excluding the guard suites themselves.
+# Sets SWEPT_SCRIPTS to every `*.sh` anywhere under SCRIPTS_DIR, at any depth,
+# excluding the three guard suites themselves.
 #
-# Globbed rather than named file by file, in both suites, because naming the two
+# Discovered rather than named file by file, in every suite, because naming the
 # scripts already known to be guarded is the same defect the suites exist to
 # catch, one level up: a NEW script running the dangerous command without the
 # selector is invisible to a sweep that only ever opens the files someone
 # remembered to list, which is how a guard fails to reach a sibling site.
 #
-# The guard suites are excluded by basename because each carries both its
-# dangerous command and the selector as fixture data and inside awk programs, so
-# sweeping them reports a suite as a violation of itself. Matching on basename
-# rather than on a path fragment keeps the exclusion from exempting a real
-# script that merely sits beside them.
+# RECURSIVE, via `find`, and this is the second half of that same defect. The
+# original form globbed `$dir/*.sh` and `$dir/lib/*.sh`, which names two
+# directories the way the thing above names two files: a script added at
+# `scripts/aws/rollback.sh` was never opened by ANY of the three suites, while
+# each went on reporting coverage of "every script under scripts/". Raised by
+# review on this suite and fixed here in the shared helper rather than locally,
+# because the RDS and ECR guards call this same function and carried the
+# identical blind spot.
 #
-# `nullglob` so a pattern matching nothing expands to nothing rather than to the
-# literal pattern text. Without it an unmatched glob becomes a nonexistent path,
-# the sweep bails out early, and it covers no scripts at all. Callers must still
-# assert SWEPT_SCRIPTS is non-empty and contains the script that actually runs
-# their command: an empty swept set satisfies every "no violations" reading.
+# `find -print0` with `read -d ''` rather than a `**` glob: `globstar` is bash 4,
+# and this must run on the bash 3.2 that ships with macOS. `sort -z` so the swept
+# order is deterministic across platforms, since `find` order is not defined.
+# The loop runs in the current shell (process substitution, not a pipe), because
+# a pipeline subshell would build the array and then discard it.
+#
+# The guard suites are excluded by basename because each carries the very
+# pattern it looks for as fixture data and inside awk programs, so sweeping them
+# reports a suite as a violation of itself. Matching on basename rather than on
+# a path fragment keeps the exclusion from exempting a real script that merely
+# sits beside them.
+#
+# A directory with no `*.sh` yields an empty SWEPT_SCRIPTS rather than a
+# nonexistent path. Callers must still assert SWEPT_SCRIPTS is non-empty and
+# contains the script that actually runs their command: an empty swept set
+# satisfies every "no violations" reading.
 #
 # Returns through a global because bash 3.2, which this must run on, has no
 # namerefs.
 build_swept_scripts() {
   local dir="$1" candidate
   SWEPT_SCRIPTS=()
-  shopt -s nullglob
-  for candidate in "$dir"/*.sh "$dir"/lib/*.sh; do
+  while IFS= read -r -d '' candidate; do
     case "$(basename "$candidate")" in
-      test-rds-deletion-protection-scope.sh | test-ecr-delete-selection.sh) continue ;;
+      test-rds-deletion-protection-scope.sh | test-ecr-delete-selection.sh | \
+        test-aws-tfstate-platform-key.sh) continue ;;
     esac
     SWEPT_SCRIPTS+=("$candidate")
-  done
-  shopt -u nullglob
+  done < <(find "$dir" -type f -name '*.sh' -print0 2>/dev/null | sort -z)
 }
 
 # shellcheck disable=SC2034  # read by the suites that source this file