fix(override): make approved overrides unblock app repos and split FAILED from HIGH

.github/workflows/approve-override.yml

@@ -2,21 +2,26 @@ name: Approve Override
 
 # Workflow dispatch — only runs after approval from the 'override-approval'
 # GitHub environment (requires board member review).
+#
+# Only plan_key and reason are needed: the repo and run come from the plan
+# key. For a platform run the failed jobs are re-run here; github.token cannot
+# touch app repos, so for an app run the job prints the run URL and the
+# developer clicks "Re-run failed jobs" (docs/apply-gate.md#requesting-an-override).
 
 on:
   workflow_dispatch:
     inputs:
       plan_key:
-        description: "S3 plan key (from the Slack alert)"
+        description: "S3 plan key, plans/javaBin/<repo>/<run_id>/tfplan (from the blocked run's log or the Slack alert)"
         required: true
         type: string
       repo:
-        description: "Repository (e.g. javaBin/moresleep)"
-        required: true
+        description: "Optional check: repository (e.g. javaBin/moresleep); must match the plan key"
+        required: false
         type: string
       run_id:
-        description: "Failed apply run ID to retrigger"
-        required: true
+        description: "Optional check: blocked run ID; must match the plan key"
+        required: false
         type: string
       reason:
         description: "Reason for override"
@@ -42,6 +47,15 @@ jobs:
         with:
           sparse-checkout: scripts
 
+      # Before signing: a plan key that names no run is refused here.
+      - name: Resolve blocked run
+        id: target
+        env:
+          PLAN_KEY: ${{ inputs.plan_key }}
+          REPO: ${{ inputs.repo }}
+          RUN_ID: ${{ inputs.run_id }}
+        run: sh scripts/override-target.sh resolve
+
       - name: Configure AWS credentials via OIDC
         uses: aws-actions/configure-aws-credentials@v6
         with:
@@ -60,14 +74,15 @@ jobs:
       - name: Notify Slack
         env:
           SSM_WEBHOOK_PARAM: /javabin/slack/platform-override-alerts-webhook
-          REPO: ${{ inputs.repo }}
+          REPO: ${{ steps.target.outputs.repo }}
+          RUN_URL: ${{ steps.target.outputs.run_url }}
           PLAN_KEY: ${{ inputs.plan_key }}
           REASON: ${{ inputs.reason }}
-        run: sh scripts/notify-slack.sh "Risk Override Approved" "*Repo:* ${REPO}\n*Plan:* \`${PLAN_KEY}\`\n*By:* ${GITHUB_ACTOR}\n*Reason:* ${REASON}"
+        run: sh scripts/notify-slack.sh "Risk Override Approved" "*Repo:* ${REPO}\n*Plan:* \`${PLAN_KEY}\`\n*By:* ${GITHUB_ACTOR}\n*Reason:* ${REASON}\n*Next:* Re-run failed jobs on the blocked run (automatic for javaBin/platform)" "$RUN_URL" "Blocked run"
 
       - name: Retrigger failed apply
         env:
           GH_TOKEN: ${{ github.token }}
-          RUN_ID: ${{ inputs.run_id }}
-          REPO: ${{ inputs.repo }}
-        run: gh run rerun "$RUN_ID" --repo "$REPO" --failed
+          RUN_ID: ${{ steps.target.outputs.run_id }}
+          REPO: ${{ steps.target.outputs.repo }}
+        run: sh scripts/override-target.sh retrigger

CLAUDE.md

@@ -152,7 +152,7 @@ terraform/state/
   tf-apply.yml         SHA verify + apply via project role
   eb-deploy.yml        Elastic Beanstalk deploy (transition)
   ecs-deploy.yml       Register revision pinned to the image digest, roll the service, check rolloutState
-  approve-override.yml Risk gate override (board members)
+  approve-override.yml Risk gate override (platform team; docs/apply-gate.md#requesting-an-override)
   expand-terraform.yml Validate app.yaml, generate terraform/, commit it
 ```
 
@@ -194,7 +194,9 @@ terraform/state/
 | `scripts/provision-atlassian.py` | CI: resolve atlassian.yaml to java.no emails, invoke team-provisioner `sync_atlassian` (report-only unless `atlassian_sync_apply`) |
 | `scripts/sync-members.py` | CI: sync approved heroes from Google Sheets into members.yaml |
 | `scripts/review-plan.py` | CI: LLM plan review via Bedrock (platform CI; also bundled into the plan-review Lambda) |
-| `scripts/invoke-plan-review.sh` | CI: app tf-plan → plan-review Lambda |
+| `scripts/invoke-plan-review.sh` | CI: app tf-plan → plan-review Lambda; retries FAILED (service) reviews, says FAILED vs HIGH (tests: `scripts/tests/`) |
+| `scripts/override-help.sh` | CI: what a blocked apply tells the developer (HIGH: request override; FAILED: re-run all jobs) |
+| `scripts/override-target.sh` | CI (approve-override): repo + run from the plan key; re-runs platform runs, prints the re-run step for app repos |
 | `scripts/notify-slack.sh` | CI: generic Slack webhook notification (via `slack_post.py`) |
 | `scripts/notify-high-risk.sh` | CI: blocked-apply alert (risk HIGH or review FAILED) with plan key + override link (via `slack_post.py`) |
 | `scripts/slack_post.py` | CI: builds escaped, size-limited Block Kit, checks Slack's reply, falls back to plain text (tests: `scripts/tests/`) |

docs/apply-gate.md

@@ -86,7 +86,7 @@ other registered team's `apps/{team}/` prefix. The list is built from
 | LOW | Auto-approve, credentials issued |
 | MEDIUM | Auto-approve, credentials issued |
 | HIGH | Blocked until override token exists and HMAC is valid |
-| FAILED | Blocked (review script error) |
+| FAILED | Blocked: the review service gave no verdict (Bedrock or Lambda failure). Not a risk finding; re-run all jobs or override |
 
 ## Override Tokens
 
@@ -109,13 +109,66 @@ An override token is an HMAC-signed JSON file stored in S3 alongside the plan:
 
 ## Override Flow
 
-1. Apply blocks on HIGH risk → Slack alert with plan key and override link
-2. Admin triggers `approve-override.yml` in the platform repo
-   - Protected by GitHub environment `override-approval` (requires board member review)
+1. Apply blocks on HIGH or FAILED → Terraform Apply fails with what to do
+   (see below), and a Slack alert with the plan key, run link and override link
+   goes to the override channel
+2. A platform team member dispatches `approve-override.yml` in the platform repo
+   with the plan key and a reason
+   - Protected by GitHub environment `override-approval` (requires review by the
+     `javaBin/platform` team)
+   - The repo and run are read from the plan key
+     (`plans/javaBin/<repo>/<run_id>/tfplan`); the optional `repo` and `run_id`
+     inputs only have to match it. A malformed key fails before anything is signed
 3. Override workflow assumes `javabin-ci-override-approver` OIDC role
 4. Invokes gate Lambda `sign` action → Lambda computes HMAC, writes `override.json` to S3
-5. Workflow retriggers the failed apply run via `gh run rerun --failed`
-6. Apply re-runs → gate Lambda finds `override.json` → verifies HMAC → issues credentials
+5. The blocked run is re-run:
+   - javaBin/platform runs: the workflow runs `gh run rerun --failed` itself
+   - app repos: the workflow's `github.token` only covers javaBin/platform (no
+     platform GitHub App has `actions: write`), so the job summary and Slack
+     message give the run URL and the developer clicks **Re-run failed jobs**
+6. Apply re-runs → gate Lambda finds `override.json` → verifies HMAC → issues
+   credentials. The gate checks for an override on every run, so no other state
+   has to change. Docker build and ECS deploy, which wait for a successful
+   apply, run after it.
+
+## Requesting an override
+
+A HIGH or FAILED review blocks Terraform Apply, and with it the image build
+and ECS deploy of the same run. The Terraform Apply job prints which case it
+is, and the plan key.
+
+**HIGH — the review found HIGH risk.** Read the review in the Terraform Plan
+job log or the PR comment. If the change is intended:
+
+1. Ask the platform team (GitHub team `javaBin/platform`, or the override Slack
+   channel where the alert was posted) for an override. Send the plan key
+   (`plans/javaBin/<repo>/<run_id>/tfplan`), the run URL, and why the change is
+   safe.
+2. They run [approve-override.yml](https://github.com/javaBin/platform/actions/workflows/approve-override.yml)
+   with that plan key and a reason, or:
+
+   ```bash
+   gh workflow run approve-override.yml -R javaBin/platform \
+     -f plan_key=plans/javaBin/<repo>/<run_id>/tfplan -f reason="..."
+   ```
+
+   A `javaBin/platform` team member must then approve the `override-approval`
+   environment.
+3. When it has finished, open your blocked run and click **Re-run failed jobs**.
+
+**FAILED — the review service failed.** This is not a risk finding: Bedrock
+or the plan-review Lambda did not return a verdict. `invoke-plan-review.sh`
+already retries (3 invokes by default, each with 3 Bedrock attempts) before
+giving up. Click **Re-run all jobs** on the run later; that re-plans and
+re-reviews. **Re-run failed jobs** does not help, since it re-reads the same
+FAILED verdict. If the service stays down, request an override as for HIGH,
+then click **Re-run failed jobs**.
+
+The override is for that plan key (HMAC over the key) and expires with the
+plan bucket's lifecycle. A push of a new commit makes a new run and plan key,
+which is reviewed afresh. Known gap: **Re-run all jobs** re-plans the same
+commit under the same key (the key holds the run ID, not the attempt), and an
+`override.json` signed for an earlier attempt is still accepted.
 
 ## S3 Plan Artifact Layout
 

docs/ci-workflow.md

@@ -46,7 +46,7 @@ Monday 08:00   ──► securityhub-summary Lambda ──► #javabin-infra-ale
 
 - **LOW/MEDIUM** → auto-apply
 - **HIGH** → blocked, Slack notification sent, board member must run approve-override workflow
-- **FAILED** → blocked (review script error)
+- **FAILED** → blocked (the review could not run; a service failure, not a risk finding). See [Requesting an override](apply-gate.md#requesting-an-override)
 
 ## Plan Artifact Security
 

docs/lambda-functions.md

@@ -329,7 +329,7 @@ Idempotent: skips resources that already have a `created-by` tag (preserves orig
 **Trigger:** Invoked by app `tf-plan` (via the `javabin-ci-app-broker` OIDC role) after the plan is uploaded
 **Purpose:** LLM risk review of app Terraform plans with platform credentials. Team CI roles have no Bedrock access (developer boundary), and any code in a plan job runs with them.
 
-**Flow:** Verify the job's GitHub OIDC token (aud `javabin-plan-review`) → plan must be under `plans/javaBin/<repo>/` → read `plan-output.txt` → review with Bedrock using `scripts/review-plan.py` (bundled as `review_plan.py`), up to 3 attempts with backoff → write `risk.json` (read by apply-gate) and `review.json` → on HIGH/FAILED for pushes to main, post to the override Slack channel → return risk + markdown for the PR comment.
+**Flow:** Verify the job's GitHub OIDC token (aud `javabin-plan-review`) → plan must be under `plans/javaBin/<repo>/` → read `plan-output.txt` → review with Bedrock using `scripts/review-plan.py` (bundled as `review_plan.py`), up to 3 attempts with backoff → write `risk.json` (read by apply-gate) and `review.json` → on HIGH/FAILED for pushes to main, post to the override Slack channel (HIGH as a risk finding, FAILED as a service failure with no verdict; FAILED is not posted while the caller passes `retrying: true`) → return risk + markdown for the PR comment.
 
 Team roles are denied writing `risk.json`, `review.json` and `override.json` in the plan bucket. `risk.json` is written as FAILED before the review starts, so a crash or timeout still leaves a verdict the gate (and a board override) can act on. The plan text is uploaded by the plan job itself, so the review catches mistakes; it is not a control against code in the job forging its own plan text.
 

docs/reusable-workflows.md

@@ -174,7 +174,7 @@ Docker BuildKit build + push to ECR repo `{team}-{app_name}`. Uses GitHub Action
 
 ### tf-plan.yml
 
-Plans the committed `terraform/` at the branch head (after expand), uploads the plan to S3 with its SHA256, has the `javabin-plan-review` Lambda review it (Bedrock Claude Haiku 5.5, platform credentials; it writes `risk.json`) and posts plan + review to the PR. A review that cannot run after retries is `FAILED`: re-run the workflow, or request an override if it persists. `scripts/check-alb-priorities.py` checks whether a listener rule the plan creates or moves would take an ALB priority another rule already holds (read-only `describe-rules`); on a conflict the plan comment still gets posted, with the conflict at the top, and the last step fails the job.
+Plans the committed `terraform/` at the branch head (after expand), uploads the plan to S3 with its SHA256, has the `javabin-plan-review` Lambda review it (Bedrock Claude Haiku 5.5, platform credentials; it writes `risk.json`) and posts plan + review to the PR. A review the service could not run is `FAILED`, which is a service failure, not a risk finding: `invoke-plan-review.sh` retries it (`REVIEW_INVOKE_ATTEMPTS`, default 3) and the log says which case it is (`Plan review service failed` or `Plan review found HIGH risk`). Both block the apply; see [Requesting an override](apply-gate.md#requesting-an-override). `scripts/check-alb-priorities.py` checks whether a listener rule the plan creates or moves would take an ALB priority another rule already holds (read-only `describe-rules`); on a conflict the plan comment still gets posted, with the conflict at the top, and the last step fails the job.
 
 **AWS role:** `javabin-ci-app-broker` (OIDC) → broker Lambda → `javabin-ci-plan-{team}` (read-only: refresh, state read, state lock, plan upload; tf-plan runs on any branch, so it cannot change resources); the review step uses the broker OIDC role to invoke `javabin-plan-review`
 Repos that bring their own `terraform/` without `backend.tf`/`providers.tf` get them from `scripts/ensure-tf-boilerplate.sh`, with state key `apps/<team>/<repo>` and the team the broker verified (not a GitHub API lookup with the run's token). tf-apply generates the same files with the same team.
@@ -183,7 +183,7 @@ Repos that bring their own `terraform/` without `backend.tf`/`providers.tf` get
 
 ### tf-apply.yml
 
-Checks out the commit tf-plan planned, verifies the plan SHA256 and asks the apply-gate Lambda for credentials (HIGH/FAILED risk needs a board-member override), then applies. Before `terraform init` it restores `.terraform.lock.hcl` from the saved plan (Terraform stores the lock file in the plan, and the SHA256 check covers it), so apply installs exactly the provider versions the plan used even though app repos gitignore the lock file and pin the AWS provider only to `~> 5.0`.
+Checks out the commit tf-plan planned, verifies the plan SHA256 and asks the apply-gate Lambda for credentials (HIGH/FAILED risk needs an override approved by the platform team), then applies. Before `terraform init` it restores `.terraform.lock.hcl` from the saved plan (Terraform stores the lock file in the plan, and the SHA256 check covers it), so apply installs exactly the provider versions the plan used even though app repos gitignore the lock file and pin the AWS provider only to `~> 5.0`. A blocked apply prints the plan key and what to do; docker-build and ecs-deploy wait for it. See [Requesting an override](apply-gate.md#requesting-an-override).
 
 **AWS role:** `javabin-ci-apply-gate` (OIDC) → apply-gate Lambda → `javabin-ci-team-{team}`
 
@@ -206,7 +206,7 @@ Transitional Elastic Beanstalk deploy via EB CLI. Will be removed per-repo when
 
 ### approve-override.yml
 
-`workflow_dispatch` — only board members can trigger. Writes a signed override for a blocked plan. Posts confirmation to Slack.
+`workflow_dispatch` behind the `override-approval` environment (reviewed by the `javaBin/platform` team). Inputs: `plan_key` and `reason`; the repo and run come from the plan key. Writes a signed override for the blocked plan and posts to Slack. Platform runs are re-run automatically; for an app repo the job prints the run URL and the developer clicks **Re-run failed jobs** (`github.token` cannot re-run another repo's workflows).
 
 **AWS role:** `javabin-ci-override-approver`
 

scripts/invoke-apply-gate.sh

@@ -56,6 +56,7 @@ echo "Risk level: $RISK"
 
 if [ "$APPROVED" != "true" ]; then
   echo "Apply blocked: ${REASON:-risk gate denied}"
+  sh "$(dirname "$0")/override-help.sh" "$RISK" "$PLAN_KEY"
 
   # Notify Slack about the block
   if [ -n "$SSM_WEBHOOK_PARAM" ]; then