fix: add custody-aware stall recovery

AGENTS.md

@@ -106,6 +106,7 @@ state/               volatile runtime signals; gitignored
   <id>.hermes-turnend-token <id>.hermes-session <id>.hermes-started   firstmate-owned Hermes hook registry token plus the task's stable session id and per-turn start acknowledgement; removed by teardown
   <id>.omp-ext.ts <id>.omp-ready <id>.omp-started <id>.omp-doorbell-ready <id>.omp-doorbell-failed   firstmate-generated OMP task extension plus its session-start and first-turn acknowledgement markers; .omp-ready publishes only after the inbox doorbell activates, and a lost handshake journals its reason to .omp-doorbell-failed (docs/architecture.md; bin/fm-task-inbox-lib.sh); removed by teardown
   <id>.inbox/          durable steering inbox: sequenced firstmate instruction records the worker acknowledges by moving them into its handled/ subdirectory; written by fm-send, re-rung and escalated by the watcher, removed by teardown (bin/fm-task-inbox-lib.sh)
+  <id>.stall-recovery  append-only custody and bounded-attempt verdict journal for watcher-triggered stall recovery (bin/fm-stall-recovery.sh); audit evidence, never recovery authority
   inbox/               trusted-local orchestrator notes written by bin/fm-inbox.sh; pending mode-0600 *.note records move to handled/ on acknowledgement, and failed wake publication leaves the note durable (docs/architecture.md)
   inbox-results/       trusted-local terminal result envelopes and delivery state written by bin/fm-inbox-result.sh; result, posting, receipt, failure, and retry-confirmation records remain mode-0600 across restarts (docs/architecture.md)
   <id>.meta          written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; optional grok_turnend_dir=, kimi_turnend_dir=, and devin_turnend_dir= persist harness registry ownership for teardown; optional prewalk_into= records an effective OMP Prewalk target; optional allow_project_omp_extensions=1 records explicit approval for tracked project extensions on an OMP launch (docs/configuration.md "OMP project extensions"); an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_herdr_session=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14)

bin/fm-control.sh

@@ -5,7 +5,7 @@
 # Usage: fm-control.sh <task-id> interrupt
 #        fm-control.sh <task-id> exit
 #        fm-control.sh <task-id> relaunch [--harness <name>] [--model <name>]
-#                                         [--effort <level>]
+#                                         [--effort <level>] [--lock-preheld]
 #                                         (--note <text> | --note-file <path>)
 #
 # Why this exists, and how it differs from fm-send.sh. bin/fm-send.sh is the
@@ -46,6 +46,17 @@
 #              inherits the local copy but none of the conversation; a
 #              secondmate reconciles its own home's records at startup, so its
 #              standing charter is never rewritten.
+#              --lock-preheld is the supervised-recovery handshake: the caller
+#              (bin/fm-stall-recovery.sh) already holds this task's lifecycle
+#              lock, so fm-control verifies the lock's recorded owner is its
+#              own parent process instead of acquiring it. The flag never
+#              bypasses the lock - without a live parent holding it, the
+#              command refuses.
+#              --stall-record <basename> rides the same handshake: inside the
+#              lock, immediately before the agent is touched, fm-control
+#              re-proves the named inbox record is still the oldest unhandled
+#              instruction and cancels with exit 3 when it was handled or
+#              superseded in flight. It requires --lock-preheld.
 #              Records a durable checkpoint and that note, exits the old agent,
 #              then delegates the launch to its single owner,
 #              bin/fm-spawn.sh --relaunch. A failure before publication keeps
@@ -142,6 +153,8 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}"
 . "$SCRIPT_DIR/fm-pr-lib.sh"
 # shellcheck source=bin/fm-wake-lib.sh
 . "$SCRIPT_DIR/fm-wake-lib.sh"
+# shellcheck source=bin/fm-task-inbox-lib.sh
+. "$SCRIPT_DIR/fm-task-inbox-lib.sh"
 
 POLL=${FM_CONTROL_POLL:-0.5}
 SETTLE_WAIT=${FM_CONTROL_SETTLE_WAIT:-5}
@@ -203,6 +216,8 @@ MODEL_SET=0
 EFFORT_SET=0
 NOTE=
 NOTE_SET=0
+LOCK_PREHELD=0
+STALL_RECORD=
 control_want_value=
 for control_arg in "$@"; do
   if [ -n "$control_want_value" ]; then
@@ -214,6 +229,7 @@ for control_arg in "$@"; do
       model) NEW_MODEL=$control_arg; MODEL_SET=1 ;;
       effort) NEW_EFFORT=$control_arg; EFFORT_SET=1 ;;
       note) NOTE=$control_arg; NOTE_SET=1 ;;
+      stall_record) STALL_RECORD=$control_arg ;;
       note_file)
         [ -f "$control_arg" ] || die "--note-file '$control_arg' is not a readable file"
         NOTE=$(cat "$control_arg")
@@ -238,6 +254,9 @@ for control_arg in "$@"; do
       NOTE=$(cat "${control_arg#--note-file=}")
       NOTE_SET=1
       ;;
+    --lock-preheld) LOCK_PREHELD=1 ;;
+    --stall-record) control_want_value=stall_record ;;
+    --stall-record=*) STALL_RECORD=${control_arg#--stall-record=} ;;
     *) die "unexpected argument '$control_arg'" ;;
   esac
 done
@@ -247,9 +266,14 @@ if [ -n "$control_want_value" ]; then
 fi
 
 if [ "$VERB" != relaunch ]; then
-  [ "$HARNESS_SET" = 0 ] && [ "$MODEL_SET" = 0 ] && [ "$EFFORT_SET" = 0 ] && [ "$NOTE_SET" = 0 ] \
-    || die "--harness, --model, --effort, and --note apply to 'relaunch' only"
+  [ "$HARNESS_SET" = 0 ] && [ "$MODEL_SET" = 0 ] && [ "$EFFORT_SET" = 0 ] && [ "$NOTE_SET" = 0 ] && [ "$LOCK_PREHELD" = 0 ] && [ -z "$STALL_RECORD" ] \
+    || die "--harness, --model, --effort, --note, --lock-preheld, and --stall-record apply to 'relaunch' only"
 fi
+# The stall-record re-check is only meaningful inside the supervised-recovery
+# handshake: without --lock-preheld there is no proof the caller serialized
+# its own custody checks with this invocation.
+[ -z "$STALL_RECORD" ] || [ "$LOCK_PREHELD" = 1 ] \
+  || die "--stall-record requires --lock-preheld"
 [ "$HARNESS_SET" = 0 ] || [ -n "$NEW_HARNESS" ] || die "--harness requires a non-empty value"
 [ "$MODEL_SET" = 0 ] || [ -n "$NEW_MODEL" ] || die "--model requires a non-empty value"
 [ "$EFFORT_SET" = 0 ] || [ -n "$NEW_EFFORT" ] || die "--effort requires a non-empty value"
@@ -275,9 +299,23 @@ ID=$RAW_ID
 fm_lease_guard "$ID" "lifecycle control (fm-control)"
 CONTROL_LOCK="$STATE/.control-$ID.lock"
 trap control_cleanup EXIT
-fm_lock_try_acquire "$CONTROL_LOCK" \
-  || die "another lifecycle action is already running for task $ID"
-CONTROL_LOCK_HELD=1
+if [ "$LOCK_PREHELD" = 1 ]; then
+  # Supervised-recovery handshake: the direct parent (bin/fm-stall-recovery.sh)
+  # holds this task's lifecycle lock across this invocation, so its custody and
+  # inbox-record re-checks are serialized with the relaunch itself. The flag
+  # never bypasses the lock: it is honored only when the lock exists, its
+  # recorded owner pid is this process's own parent, and that parent is alive.
+  # CONTROL_LOCK_HELD stays 0 so the cleanup trap never releases a lock this
+  # process does not own.
+  lock_owner_pid=$(cat "$CONTROL_LOCK/pid" 2>/dev/null || true)
+  if [ -z "$lock_owner_pid" ] || [ "$lock_owner_pid" != "$PPID" ] || ! fm_pid_alive "$lock_owner_pid"; then
+    die "--lock-preheld requires the caller to hold $CONTROL_LOCK as this process's live parent (owner: ${lock_owner_pid:-none}, parent: $PPID)"
+  fi
+else
+  fm_lock_try_acquire "$CONTROL_LOCK" \
+    || die "another lifecycle action is already running for task $ID"
+  CONTROL_LOCK_HELD=1
+fi
 META="$STATE/$ID.meta"
 if [ ! -f "$META" ]; then
   case "$RAW_ID" in
@@ -535,6 +573,26 @@ TARGET_HARNESS=$HARNESS
 TARGET_MODEL=
 TARGET_EFFORT=
 
+# stall_relaunch_cancel: cancel the supervised --stall-record relaunch cleanly
+# after the checkpoint/note work has run. Restores the worker's instructions
+# byte-exact (record_note appended the progress note), journals the named
+# cancellation phase, and exits with <code> for the supervised caller. Reads
+# do_relaunch's note_line through bash's dynamic scope; only ever called from
+# inside it. A restore failure is a hard error, not a clean cancel.
+stall_relaunch_cancel() {  # <phase> <message> <code>
+  local phase=$1 message=$2 code=$3
+  if [ -n "$RELAUNCH_BRIEF" ] && [ -f "$BRIEF_PRIOR" ]; then
+    if ! cp -p "$BRIEF_PRIOR" "$RELAUNCH_BRIEF" 2>/dev/null; then
+      RELAUNCH_ACTIVE=0
+      journal_write "failed:$phase" "rollback=instructions-restore-failed" "${CHECKPOINT_LINES[@]}" "$note_line" || true
+      die "relaunch cancelled ($phase), but restoring the original instructions failed"
+    fi
+  fi
+  journal_write "cancelled:$phase" "${CHECKPOINT_LINES[@]}" "$note_line" || true
+  echo "relaunch cancelled: $message" >&2
+  exit "$code"
+}
+
 journal_write() {  # <phase> [extra-line]...
   local phase=$1
   shift
@@ -795,9 +853,18 @@ record_note() {
 }
 
 do_relaunch() {
-  local exit_result state note_line
+  local exit_result state note_line stall_oldest
   local -a spawn_args
 
+  stall_relaunch_absence_guard() {
+    state=$(agent_state)
+    [ -n "$STALL_RECORD" ] || return 0
+    case "$state" in
+      dead|missing) ;;
+      *) die "supervised relaunch refused: endpoint state '$state' is not positively absent" ;;
+    esac
+  }
+
   require_state_verified_backend relaunch
   resolve_relaunch_profile
 
@@ -832,9 +899,33 @@ do_relaunch() {
   record_note
   journal_write noted "${CHECKPOINT_LINES[@]}" "$note_line"
 
+  # Final stall-record re-check, inside the lifecycle lock and immediately
+  # before the agent is touched: the caller's own pre-invocation proof cannot
+  # cover the gap to this point, so a record handled or superseded in flight
+  # must still cancel the relaunch here. Exit 3 is the dedicated "instruction
+  # resolved; nothing to do" code the supervised caller maps to recovered. The
+  # brief is restored byte-exact before the cancelled journal write so the
+  # rollback trap has nothing left to do and the worker's instructions stay
+  # untouched when no relaunch happened.
+  if [ -n "$STALL_RECORD" ]; then
+    if stall_oldest=$(fm_task_inbox_oldest_unhandled "$STATE" "$ID" 2>/dev/null); then
+      :
+    else
+      stall_oldest_rc=$?
+      [ "$stall_oldest_rc" -eq 1 ] \
+        && stall_relaunch_cancel record-resolved "stall record resolved (inbox empty; instruction handled)" 3
+      die "relaunch refused: stall inbox is unreadable inside the lifecycle lock"
+    fi
+    if [ -z "$stall_oldest" ]; then
+      stall_relaunch_cancel record-resolved "stall record resolved (inbox empty; instruction handled)" 3
+    elif [ "${stall_oldest##*/}" != "$STALL_RECORD" ]; then
+      stall_relaunch_cancel record-resolved "stall record $STALL_RECORD handled or superseded (${stall_oldest##*/} is now oldest)" 3
+    fi
+  fi
+
   journal_write stopping "${CHECKPOINT_LINES[@]}" "$note_line"
-  state=$(agent_state)
-  if [ "$state" = missing ]; then
+  stall_relaunch_absence_guard
+  if [ "$state" = dead ] || [ "$state" = missing ]; then
     # The recorded endpoint is authoritatively absent, so there is no agent to
     # stop: the exit phase is already complete and the launch below recreates
     # the endpoint in the checkpointed worktree. Only a proven-missing endpoint
@@ -856,6 +947,7 @@ do_relaunch() {
   # before arming the new one, so nothing to do here.
   RELAUNCH_TX="${BASHPID:-$$}.$(date -u +%Y%m%dT%H%M%SZ).$RANDOM"
   journal_write launching "${CHECKPOINT_LINES[@]}" "$note_line" "relaunch_tx=$RELAUNCH_TX"
+  stall_relaunch_absence_guard
   if [ "$KIND" = secondmate ] && [ "$PRIOR_HARNESS" = omp ]; then
     spawn_args=("$ID" --secondmate --harness "$TARGET_HARNESS")
   else
@@ -888,7 +980,7 @@ do_relaunch() {
   }
   RELAUNCH_AGENT_CONFIRMED=1
 
-  journal_write complete "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result"
+  journal_write complete "${CHECKPOINT_LINES[@]}" "$note_line" "exit_result=$exit_result" "relaunch_tx=$RELAUNCH_TX"
   RELAUNCH_ACTIVE=0
   echo "relaunched $ID harness=$TARGET_HARNESS from=$PRIOR_RECORDED_HARNESS model=$TARGET_MODEL effort=$TARGET_EFFORT backend=$BACKEND endpoint=$T worktree=$WT"
 }

bin/fm-spawn.sh

@@ -435,6 +435,14 @@ done
 [ "$TRACEPARENT_SET" -eq 0 ] || [ -n "$TRACEPARENT_ARG" ] || { echo "error: --traceparent requires a non-empty value" >&2; exit 1; }
 
 if [ "$RELAUNCH" -eq 1 ]; then
+  # A relaunch driven by fm-control carries its transaction id so a failure
+  # after this record is published still classifies the successor correctly.
+  # The value lands verbatim in the durable record, so a malformed token is
+  # refused rather than corrupting it.
+  case "${FM_CONTROL_RELAUNCH_TX:-}" in
+    '') ;;
+    *[!A-Za-z0-9._-]*) { echo "error: FM_CONTROL_RELAUNCH_TX is not a safe metadata token" >&2; exit 1; } ;;
+  esac
   RELAUNCH_ID=${POS[0]:-}
   [ -n "$RELAUNCH_ID" ] || { echo "error: --relaunch requires a task id" >&2; exit 1; }
   RELAUNCH_META="$STATE/$RELAUNCH_ID.meta"
@@ -4654,6 +4662,10 @@ SPAWN_META_LOCK_HELD=1
   echo "model=${MODEL:-default}"
   echo "effort=${EFFORT:-default}"
   echo "spawn_gen=$SPAWN_GEN"
+  # The relaunch transaction id lets fm-control classify a post-publish
+  # failure as "new record published" rather than "replacement never
+  # launched"; only a relaunch under fm-control writes it.
+  [ -z "${FM_CONTROL_RELAUNCH_TX:-}" ] || echo "control_relaunch_tx=$FM_CONTROL_RELAUNCH_TX"
   [ -z "${GROK_AUTH_DIR:-}" ] || echo "grok_turnend_dir=$GROK_AUTH_DIR"
   [ -z "${KIMI_AUTH_DIR:-}" ] || echo "kimi_turnend_dir=$KIMI_AUTH_DIR"
   [ -z "${DEVIN_AUTH_DIR:-}" ] || echo "devin_turnend_dir=$DEVIN_AUTH_DIR"
@@ -4922,6 +4934,32 @@ sleep 0.3
 if [ "$OMP_LAUNCH_TEMPLATE" -eq 1 ] && [ "$HARNESS" = omp ]; then
   LAUNCH="/bin/bash -c $(shell_quote "$LAUNCH")"
 fi
+if [ "$HARNESS" = omp ] && [ "$RELAUNCH" -eq 1 ]; then
+  # Generation reconciliation: every request.* receipt under the doorbell
+  # requests dir was written for the PRIOR incarnation's doorbell. The
+  # extension only reconciles .ambiguous/.awaiting-turn on activate, so a
+  # surviving .acked/.delivered/.unproven tombstone would suppress the
+  # replacement worker's doorbell for the same still-unhandled record - the
+  # exact delivered-no-turn stall this relaunch recovers from. This runs only
+  # here, after every refusal gate above has passed and immediately before
+  # the replacement launch is submitted, so a refused relaunch never deletes
+  # the prior generation's receipts; the inbox records themselves live under
+  # state/<id>.inbox and are untouched.
+  OMP_REQUESTS_DIR="$STATE/$ID.omp-doorbell-ready.requests"
+  if [ -d "$OMP_REQUESTS_DIR" ] && [ ! -L "$OMP_REQUESTS_DIR" ]; then
+    for request_artifact in "$OMP_REQUESTS_DIR"/request.*; do
+      [ -e "$request_artifact" ] || [ -L "$request_artifact" ] || continue
+      if [ -L "$request_artifact" ] || [ ! -f "$request_artifact" ]; then
+        echo "error: refusing OMP relaunch through unsafe request entry: $request_artifact" >&2
+        exit 1
+      fi
+      rm -f "$request_artifact" || {
+        echo "error: refusing OMP relaunch because a stale doorbell receipt could not be retired: $request_artifact" >&2
+        exit 1
+      }
+    done
+  fi
+fi
 if [ "$BACKEND" = herdr ]; then
   spawn_send_text_line "$T" "$LAUNCH" || {
     echo "error: Herdr launch pane did not reach a proven idle shell; refusing to submit $HARNESS" >&2