diff --git a/apps/decodex/src/agent/tracker_tool_bridge/review.rs b/apps/decodex/src/agent/tracker_tool_bridge/review.rs index f5ccd8af0..3e47fc68a 100644 --- a/apps/decodex/src/agent/tracker_tool_bridge/review.rs +++ b/apps/decodex/src/agent/tracker_tool_bridge/review.rs @@ -408,6 +408,12 @@ impl<'a> TrackerToolBridge<'a> { } } + pub(crate) fn has_tracker_exit_signal(&self) -> bool { + *self.manual_attention_requested.borrow() + || *self.manual_attention_comment_recorded.borrow() + || self.pending_review_completion.borrow().is_some() + } + pub(crate) fn finalized_completion_disposition( &self, ) -> crate::prelude::Result> { diff --git a/apps/decodex/src/orchestrator/daemon.rs b/apps/decodex/src/orchestrator/daemon.rs index 520545482..7f93a4003 100644 --- a/apps/decodex/src/orchestrator/daemon.rs +++ b/apps/decodex/src/orchestrator/daemon.rs @@ -7,6 +7,16 @@ enum RetryEntryRetentionDecision { Block, } +struct ChildExitRetrySchedule<'a> { + issue_id: &'a str, + run_id: &'a str, + attempt_number: i64, + continuation_initial_issue_state: Option, + dispatch_mode: IssueDispatchMode, + kind: RetryKind, + attempt: u32, +} + struct DaemonTickRuntimeContext<'a, T, I> { tracker: &'a T, project: &'a ServiceConfig, @@ -1114,7 +1124,28 @@ where return Ok(()); } + let recovered_phase_goal_continuation = if !exit_status.success() { + maybe_recover_child_exit_active_phase_goal_continuation( + &context, + &issue, + child, + initial_issue_state, + dispatch_mode, + )? + } else { + None + }; let (kind, attempt, continuation_initial_issue_state) = if continuation_pending { + ( + RetryKind::Continuation, + u32::try_from(run_attempt.attempt_number()).unwrap_or(u32::MAX).max(1), + Some(initial_issue_state.to_owned()), + ) + } else if recovered_phase_goal_continuation.is_some() { + context + .state_store + .update_run_status(run_attempt.run_id(), CONTINUATION_PENDING_RUN_STATUS)?; + ( RetryKind::Continuation, u32::try_from(run_attempt.attempt_number()).unwrap_or(u32::MAX).max(1), @@ -1125,10 +1156,8 @@ where return Ok(()); } else { - let retry_budget_attempts = - child_exit_retry_budget_attempt_count(&context, &issue, child)?; - let retry_budget_limit = - child_exit_retry_budget_limit(&context, &issue, child)?; + let retry_budget_attempts = child_exit_retry_budget_attempt_count(&context, &issue, child)?; + let retry_budget_limit = child_exit_retry_budget_limit(&context, &issue, child)?; if retry_budget_attempts >= retry_budget_limit { return terminalize_exhausted_child_exit_retry( @@ -1143,12 +1172,36 @@ where (RetryKind::Failure, retry_budget_attempts, None) }; - let delay = retry_delay(kind, attempt.max(1), context.workflow); + + queue_child_exit_retry( + context.retry_queue, + context.state_store, + context.workflow, + ChildExitRetrySchedule { + issue_id, + run_id: run_attempt.run_id(), + attempt_number: run_attempt.attempt_number(), + continuation_initial_issue_state, + dispatch_mode, + kind, + attempt, + }, + ) +} + +fn queue_child_exit_retry( + retry_queue: &mut RetryQueue, + state_store: &StateStore, + workflow: &WorkflowDocument, + schedule: ChildExitRetrySchedule<'_>, +) -> Result<()> { + let attempt = schedule.attempt.max(1); + let delay = retry_delay(schedule.kind, attempt, workflow); tracing::info!( - issue_id, - retry_kind = ?kind, - retry_attempt = attempt.max(1), + issue_id = schedule.issue_id, + retry_kind = ?schedule.kind, + retry_attempt = attempt, retry_delay_ms = delay.as_millis(), "Queued retry after control-plane child exit." ); @@ -1158,28 +1211,75 @@ where ); write_retry_schedule_for_run( - context.state_store, - issue_id, - run_attempt.run_id(), - run_attempt.attempt_number(), - kind, + state_store, + schedule.issue_id, + schedule.run_id, + schedule.attempt_number, + schedule.kind, retry_ready_at_unix_epoch, )?; - context.retry_queue.upsert(RetryEntry { - issue_id: issue_id.to_owned(), + retry_queue.upsert(RetryEntry { + issue_id: schedule.issue_id.to_owned(), #[cfg(test)] retry_project_slug: String::new(), - continuation_initial_issue_state, - dispatch_mode, - kind, - attempt: attempt.max(1), + continuation_initial_issue_state: schedule.continuation_initial_issue_state, + dispatch_mode: schedule.dispatch_mode, + kind: schedule.kind, + attempt, ready_at: Instant::now() + delay, }); Ok(()) } +fn maybe_recover_child_exit_active_phase_goal_continuation( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, + child: ChildRunRef<'_>, + initial_issue_state: &str, + dispatch_mode: IssueDispatchMode, +) -> Result> +where + T: IssueTracker, +{ + let worktree = child_exit_worktree_spec(context, issue)?; + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: initial_issue_state.to_owned(), + worktree, + #[cfg(test)] + retry_project_slug: String::new(), + dispatch_mode, + attempt_number: child.attempt_number, + run_id: child.run_id.to_owned(), + retry_budget_base: 0, + }; + let recovery = recover_active_phase_goal_continuation( + context.project, + context.workflow, + context.state_store, + &issue_run, + "child_exit_failed", + )?; + + if let Some(recovery) = &recovery { + tracing::warn!( + project_id = context.project.service_id(), + issue_id = issue.id, + issue = issue.identifier, + run_id = child.run_id, + attempt = child.attempt_number, + source_phase = recovery.source_phase.as_str(), + next_phase = recovery.next_phase.as_str(), + "Recovered active phase goal after child exit failure; scheduling continuation." + ); + } + + Ok(recovery) +} + fn child_exit_retry_retention_decision( context: &ChildExitRetryContext<'_, T>, issue: &TrackerIssue, diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 56af37d37..25075bbad 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -109,6 +109,11 @@ struct ArchitectureRecoveryStart { detail: String, } +struct PhaseGoalRecoveryContinuation { + source_phase: PhaseGoalKind, + next_phase: PhaseGoalKind, +} + struct CompletedAppServerRun<'a, T> where T: IssueTracker, @@ -1026,6 +1031,18 @@ where return Ok(IssueAppServerRunOutcome::Finalized(summary)); } + if !input.tracker_tool_bridge.has_tracker_exit_signal() + && let Some(summary) = maybe_continue_after_active_phase_goal_recovery( + input.project, + input.workflow, + input.state_store, + input.issue_run, + &error, + )? + { + return Ok(IssueAppServerRunOutcome::Finalized(summary)); + } + return Err(preserve_and_promote_app_server_run_failure( input.project, input.state_store, @@ -1129,6 +1146,185 @@ where Ok(Some(run_summary_from_issue_run(project.service_id(), issue_run))) } +fn maybe_continue_after_active_phase_goal_recovery( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + error: &Report, +) -> Result> { + let Some(recovery) = recover_active_phase_goal_continuation( + project, + workflow, + state_store, + issue_run, + phase_goal_recovery_source_error_class(error), + )? else { + return Ok(None); + }; + let mut summary = run_summary_from_issue_run(project.service_id(), issue_run); + + summary.continuation_pending = true; + + tracing::warn!( + project_id = project.service_id(), + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + source_phase = recovery.source_phase.as_str(), + next_phase = recovery.next_phase.as_str(), + error = %error, + "Recovered active phase goal after app-server failure; scheduling continuation." + ); + + Ok(Some(summary)) +} + +fn recover_active_phase_goal_continuation( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue_run: &IssueRunPlan, + source_error_class: &str, +) -> Result> { + if !worktree_has_tracked_changes(&issue_run.worktree.path) { + return Ok(None); + } + + let Some(source_phase) = latest_active_phase_goal_recovery_candidate( + project, + state_store, + issue_run, + )? else { + return Ok(None); + }; + let controller = RepoGatePhaseGoalController { + project, + workflow, + state_store, + issue_run, + }; + let transition = controller.validate_phase_goal_output(source_phase)?; + let next_phase = match transition { + PhaseGoalTransition::Continue(next_goal) => next_goal.phase, + PhaseGoalTransition::CompleteRun => return Ok(None), + }; + + record_phase_goal_recovery_continuation( + project, + state_store, + issue_run, + source_phase, + next_phase, + source_error_class, + )?; + + Ok(Some(PhaseGoalRecoveryContinuation { source_phase, next_phase })) +} + +fn phase_goal_recovery_source_error_class(error: &Report) -> &'static str { + retained_progress_source_error_class(error).unwrap_or("app_server_run_failed") +} + +fn latest_active_phase_goal_recovery_candidate( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, +) -> Result> { + let events = state_store.list_private_execution_events( + project.service_id(), + &issue_run.issue.id, + &issue_run.run_id, + issue_run.attempt_number, + )?; + + for event in events.iter().rev() { + match event.event_type() { + "phase_goal_completed" + | "phase_goal_next" + | "phase_goal_transition" + | "review_completion_intent" + | "terminal_finalize" => return Ok(None), + "phase_goal_set" | "phase_goal_status" => { + let Some(phase) = phase_goal_event_phase(event.payload()) else { + return Ok(None); + }; + let Some(status) = phase_goal_event_status(event.payload()) else { + return Ok(None); + }; + + return Ok(phase_goal_recovery_candidate_from_status(phase, status)); + }, + _ => {}, + } + } + + Ok(None) +} + +fn phase_goal_event_phase(payload: &Value) -> Option { + payload + .get("phase") + .and_then(Value::as_str) + .or_else(|| payload.get("payload")?.get("phase")?.as_str()) + .and_then(phase_goal_kind_from_str) +} + +fn phase_goal_event_status(payload: &Value) -> Option<&str> { + payload + .get("status") + .and_then(Value::as_str) + .or_else(|| payload.get("payload")?.get("status")?.as_str()) +} + +fn phase_goal_recovery_candidate_from_status( + phase: PhaseGoalKind, + status: &str, +) -> Option { + if status != "active" { + return None; + } + if matches!( + phase, + PhaseGoalKind::ImplementToValidationReady + | PhaseGoalKind::RepairValidationFailures + | PhaseGoalKind::RepairAcceptedReviewFindings + ) { + Some(phase) + } else { + None + } +} + +fn record_phase_goal_recovery_continuation( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + source_phase: PhaseGoalKind, + next_phase: PhaseGoalKind, + source_error_class: &str, +) -> Result<()> { + state_store.append_private_execution_event( + project.service_id(), + &issue_run.issue.id, + &issue_run.run_id, + issue_run.attempt_number, + "phase_goal_recovery", + json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": source_phase.as_str(), + "signal": "active_goal_recovered", + "payload": { + "nextPhase": next_phase.as_str(), + "sourceErrorClass": source_error_class, + }, + }), + )?; + + Ok(()) +} + fn phase_goal_kind_from_str(value: &str) -> Option { match value { "implement_to_validation_ready" => Some(PhaseGoalKind::ImplementToValidationReady), diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index 849992c99..9dc6b6150 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -1446,7 +1446,7 @@ fn project_attention_count( for candidate in snapshot .queued_candidates .iter() - .filter(|candidate| candidate.classification == "blocked" || candidate.attention.is_some()) + .filter(|candidate| queued_candidate_counts_as_attention(candidate)) { attention_keys.insert(operator_issue_attention_key( &candidate.issue_id, @@ -1456,19 +1456,14 @@ fn project_attention_count( for lane in snapshot .post_review_lanes .iter() - .filter(|lane| { - matches!( - lane.classification.as_str(), - "blocked" | "needs_review_repair" | "closeout_blocked" - ) - }) + .filter(|lane| post_review_lane_counts_as_attention(lane)) { attention_keys.insert(operator_issue_attention_key(&lane.issue_id, Some(&lane.issue_identifier))); } for lane in snapshot .history_lanes .iter() - .filter(|lane| history_lane_has_unresolved_attention(snapshot, lane, completed_state)) + .filter(|lane| history_lane_has_current_attention(snapshot, lane, completed_state)) { attention_keys.insert(history_lane_group_key(lane)); } @@ -1476,7 +1471,29 @@ fn project_attention_count( attention_keys.len() } -fn history_lane_has_unresolved_attention( +fn project_history_only_attention_count(snapshot: &OperatorStatusSnapshot) -> usize { + snapshot + .history_lanes + .iter() + .filter(|lane| { + history_ledger_outcome_requires_attention(&lane.ledger_outcome) + && !history_lane_has_current_attention_signal(snapshot, lane) + }) + .count() +} + +fn queued_candidate_counts_as_attention(candidate: &OperatorQueuedIssueStatus) -> bool { + candidate.classification == "blocked" || candidate.attention.is_some() +} + +fn post_review_lane_counts_as_attention(lane: &OperatorPostReviewLaneStatus) -> bool { + matches!( + lane.classification.as_str(), + "blocked" | "needs_review_repair" | "closeout_blocked" + ) +} + +fn history_lane_has_current_attention( snapshot: &OperatorStatusSnapshot, lane: &OperatorHistoryLaneStatus, completed_state: Option<&str>, @@ -1485,7 +1502,34 @@ fn history_lane_has_unresolved_attention( return false; } - !history_lane_attention_is_resolved_tracker_echo(snapshot, lane, completed_state) + history_lane_has_current_attention_signal(snapshot, lane) + && !history_lane_attention_is_resolved_tracker_echo(snapshot, lane, completed_state) +} + +fn history_lane_has_current_attention_signal( + snapshot: &OperatorStatusSnapshot, + lane: &OperatorHistoryLaneStatus, +) -> bool { + if lane.active_label_present == Some(true) || lane.needs_attention_label_present == Some(true) { + return true; + } + + let issue_key = history_lane_group_key(lane); + + snapshot.worktrees.iter().any(|worktree| { + operator_issue_attention_key(&worktree.issue_id, worktree.issue_identifier.as_deref()) + == issue_key + }) || snapshot.post_review_lanes.iter().any(|post_review_lane| { + post_review_lane_counts_as_attention(post_review_lane) + && operator_issue_attention_key( + &post_review_lane.issue_id, + Some(&post_review_lane.issue_identifier), + ) == issue_key + }) || snapshot.queued_candidates.iter().any(|candidate| { + queued_candidate_counts_as_attention(candidate) + && operator_issue_attention_key(&candidate.issue_id, Some(&candidate.issue_identifier)) + == issue_key + }) } fn history_lane_attention_is_resolved_tracker_echo( @@ -6706,6 +6750,7 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { output.push_str(&format!("Recovery worktrees: {}\n", recovery_worktrees.len())); output.push_str(&format!("Post-review lanes: {}\n", snapshot.post_review_lanes.len())); + append_rendered_attention_summary(&mut output, snapshot); append_rendered_execution_programs(&mut output, snapshot); output.push_str("\nRunning Lanes\n"); @@ -6759,6 +6804,27 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { output } +fn append_rendered_attention_summary(output: &mut String, snapshot: &OperatorStatusSnapshot) { + let current_attention_count = snapshot + .projects + .iter() + .find(|project| project.project_id == snapshot.project_id) + .or_else(|| snapshot.projects.first()) + .map_or_else(|| project_attention_count(snapshot, None), |project| project.attention_count); + let history_only_attention_count = project_history_only_attention_count(snapshot); + + output.push_str(&format!("Current attention: {current_attention_count}\n")); + output.push_str(&format!( + "History-only terminal attention: {history_only_attention_count}\n" + )); + + if current_attention_count == 0 && history_only_attention_count > 0 { + output.push_str( + "Current attention action: none; terminal attention rows below are Run Ledger history only.\n", + ); + } +} + fn append_rendered_execution_programs(output: &mut String, snapshot: &OperatorStatusSnapshot) { output.push_str("\nExecution Programs\n"); diff --git a/apps/decodex/src/orchestrator/tests/operator/status/history.rs b/apps/decodex/src/orchestrator/tests/operator/status/history.rs index 6d4ca7ca2..9aa5a3069 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/history.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/history.rs @@ -475,6 +475,158 @@ fn local_status_summary_counts_terminal_history_needs_attention_without_queue_ca assert!(rendered.contains("outcome: needs_attention")); assert!(rendered.contains("needs_attention_reason: Decodex retained validation-ready partial progress for manual review.")); assert!(rendered.contains("role: retained_attention")); + assert!(rendered.contains("Current attention: 1")); + assert!(rendered.contains("History-only terminal attention: 0")); +} + +#[test] +fn local_status_summary_ignores_history_only_terminal_attention_without_current_owner() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-pub-1549", + "PUB-1549", + "Todo", + &[], + Some(3), + "2026-06-12T01:56:00Z", + ); + let local_comments = retained_partial_progress_linear_execution_history_comments(&issue); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-mono-pub-1549", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember previous lane ownership"); + state_store + .record_run_attempt("pub-1549-attempt-1-1781240781", &issue.id, 1, "failed") + .expect("failed attempt should record"); + state_store + .clear_worktree(&issue.id) + .expect("historical lane should not have current retained ownership"); + + seed_local_linear_execution_events(&state_store, &local_comments); + + let snapshot = orchestrator::build_operator_state_snapshot_without_live_observers( + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(snapshot.active_runs.len(), 0); + assert_eq!(snapshot.queued_candidates.len(), 0); + assert_eq!(snapshot.post_review_lanes.len(), 0); + assert_eq!(snapshot.worktrees.len(), 0); + assert_eq!(snapshot.projects[0].active_run_count, 0); + assert_eq!(snapshot.projects[0].running_lane_count, 0); + assert_eq!(snapshot.projects[0].queued_candidate_count, 0); + assert_eq!(snapshot.projects[0].post_review_lane_count, 0); + assert_eq!(snapshot.projects[0].attention_count, 0); + assert_eq!(snapshot.projects[0].retained_worktree_count, 0); + assert_eq!(snapshot.projects[0].cleanup_blocked_count, 0); + assert_eq!(snapshot.projects[0].cleanup_pending_count, 0); + assert_eq!(lane.ledger_outcome.ledger_status, "present"); + assert_eq!(lane.ledger_outcome.final_outcome, "needs_attention"); + assert_eq!(lane.latest_run.status, "needs_attention"); + assert_eq!(lane.latest_run.phase, "needs_attention"); + assert!(!lane.latest_run.active_lease); + assert_eq!(snapshot_json["projects"][0]["attention_count"], 0); + assert_eq!(snapshot_json["worktrees"].as_array().map(Vec::len), Some(0)); + assert_eq!( + snapshot_json["history_lanes"][0]["latest_run"]["status"], + "needs_attention" + ); + assert!(rendered.contains("Current attention: 0")); + assert!(rendered.contains("History-only terminal attention: 1")); + assert!(rendered.contains( + "Current attention action: none; terminal attention rows below are Run Ledger history only." + )); + assert!(rendered.contains("outcome: needs_attention")); +} + +#[test] +fn live_status_counts_terminal_attention_when_current_attention_label_remains() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-pub-1550", + "PUB-1550", + "Todo", + &["decodex:needs-attention"], + Some(3), + "2026-06-12T02:16:00Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + + tracker.issue_comments.borrow_mut().insert( + issue.id.clone(), + vec![linear_execution_history_comment( + &issue, + "needs_attention", + "2026-06-12T02:20:00Z", + "manual-attention", + |record| { + record.summary = Some(String::from("Decodex run requires operator attention.")); + record.error_class = Some(String::from("human_attention_required")); + record.next_action = Some(String::from( + "resolve the blocker, clear needs-attention, then requeue if needed", + )); + record.blockers = Some(vec![String::from("manual blocker remains")]); + record.evidence = Some(vec![String::from("needs-attention label remains")]); + record.terminal_path = Some(String::from("manual_attention")); + }, + )], + ); + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-mono-pub-1550", + &config.worktree_root().join(&issue.identifier).display().to_string(), + ) + .expect("worktree should remember previous lane ownership"); + state_store + .record_run_attempt("pub-1550-attempt-1-1781241600", &issue.id, 1, "failed") + .expect("failed attempt should record"); + state_store + .clear_worktree(&issue.id) + .expect("current retained worktree should be absent"); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let lane = snapshot.history_lanes.first().expect("history lane should exist"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert!( + snapshot + .queued_candidates + .iter() + .all(|candidate| candidate.issue_identifier != "PUB-1550"), + "terminal attention queue echo should be suppressed" + ); + assert!(snapshot.worktrees.is_empty()); + assert_eq!(snapshot.projects[0].attention_count, 1); + assert_eq!(snapshot.projects[0].retained_worktree_count, 0); + assert_eq!(lane.issue_state.as_deref(), Some("Todo")); + assert_eq!(lane.needs_attention_label_present, Some(true)); + assert_eq!(lane.ledger_outcome.final_outcome, "needs_attention"); + assert!(rendered.contains("Current attention: 1")); + assert!(rendered.contains("History-only terminal attention: 0")); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs index 201931e16..8216dd808 100644 --- a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs +++ b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs @@ -819,6 +819,183 @@ fn schedule_retry_after_child_exit_records_continuation_retry_for_clean_exit() { assert_eq!(entry.attempt, 1); } +#[test] +fn schedule_retry_after_child_exit_recovers_active_phase_goal_before_terminal_attention() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-3"; + + commit_worktree_change(config.repo_root(), "ready.txt", "before\n", "add ready file"); + + fs::write(config.repo_root().join("ready.txt"), "after\n") + .expect("tracked diff should write"); + + for (attempt, recorded_run_id) in [(1, "run-1"), (2, "run-2"), (3, run_id)] { + state_store + .record_run_attempt(recorded_run_id, &issue.id, attempt, "failed") + .expect("failed run attempt should record"); + } + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-pub-101", + &config.repo_root().display().to_string(), + ) + .expect("worktree should record"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + run_id, + 3, + "phase_goal_set", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "payload": { + "phase": "implement_to_validation_ready", + "status": "active", + }, + }), + ) + .expect("phase goal event should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 3 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("active phase goal should recover into continuation"); + + let entry = retry_queue.entries.get(&issue.id).expect("retry entry should exist for the issue"); + let run_attempt = state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should still exist"); + let events = state_store + .list_private_execution_events(TEST_SERVICE_ID, &issue.id, run_id, 3) + .expect("private events should load"); + + assert_eq!(entry.kind, orchestrator::RetryKind::Continuation); + assert_eq!(entry.attempt, 3); + assert_eq!(run_attempt.status(), CONTINUATION_PENDING_RUN_STATUS); + assert!(tracker.comments.borrow().is_empty()); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_recovery" + && event.payload()["payload"]["sourceErrorClass"] == "child_exit_failed" + })); +} + +#[test] +fn schedule_retry_after_child_exit_respects_terminal_finalize_before_phase_goal_recovery() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_service_owned_issue("In Progress"); + let tracker = + FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-3"; + + commit_worktree_change(config.repo_root(), "ready.txt", "before\n", "add ready file"); + + fs::write(config.repo_root().join("ready.txt"), "after\n") + .expect("tracked diff should write"); + + for (attempt, recorded_run_id) in [(1, "run-1"), (2, "run-2"), (3, run_id)] { + state_store + .record_run_attempt(recorded_run_id, &issue.id, attempt, "failed") + .expect("failed run attempt should record"); + } + + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-pub-101", + &config.repo_root().display().to_string(), + ) + .expect("worktree should record"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + run_id, + 3, + "phase_goal_set", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "payload": { + "phase": "implement_to_validation_ready", + "status": "active", + }, + }), + ) + .expect("phase goal event should record"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + run_id, + 3, + "terminal_finalize", + serde_json::json!({ + "path": "manual_attention", + "mode": "normal", + "branch": "x/pubfi-pub-101", + "worktree_path": config.repo_root().display().to_string(), + }), + ) + .expect("terminal finalize event should record"); + + let exit_status = + Command::new("sh").args(["-c", "exit 1"]).status().expect("failure exit should run"); + let mut retry_queue = RetryQueue::default(); + + orchestrator::schedule_retry_after_child_exit( + ChildExitRetryContext { + retry_queue: &mut retry_queue, + tracker: &tracker, + project: &config, + workflow: &workflow, + state_store: &state_store, + }, + ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 3 }, + issue.project_slug.as_deref().expect("sample issue should carry a project slug"), + &issue.state.name, + IssueDispatchMode::Retry, + exit_status, + ) + .expect("terminalized child exit should keep the terminal path"); + + let events = state_store + .list_private_execution_events(TEST_SERVICE_ID, &issue.id, run_id, 3) + .expect("private events should load"); + + assert!(!retry_queue.entries.contains_key(&issue.id)); + assert!( + events.iter().all(|event| event.event_type() != "phase_goal_recovery"), + "terminal finalize intent must not be replaced by active phase-goal recovery" + ); +} + #[test] fn schedule_retry_after_child_exit_preserves_specific_retry_schedule_kind_for_failure_retry() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs index e336efef5..01e3bcf60 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs @@ -186,6 +186,80 @@ fn phase_goal_completion_runs_repo_gate_and_persists_handoff_phase() { })); } +#[test] +fn active_phase_goal_failure_recovery_continues_to_repair_instead_of_attention() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + let issue = sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: repo_root.to_path_buf(), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1"), + retry_budget_base: 0, + }; + + commit_worktree_change(repo_root, "ready.txt", "before\n", "add ready file"); + + fs::write(repo_root.join("ready.txt"), "after\n").expect("tracked diff should write"); + + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + &issue_run.run_id, + 1, + "phase_goal_set", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "payload": { + "phase": "implement_to_validation_ready", + "status": "active", + }, + }), + ) + .expect("phase goal event should record"); + + let summary = orchestrator::maybe_continue_after_active_phase_goal_recovery( + &config, + &workflow, + &state_store, + &issue_run, + &Report::msg("app server transport closed after local verification"), + ) + .expect("phase goal recovery should not fail") + .expect("dirty active phase goal should recover to continuation"); + let events = state_store + .list_private_execution_events(TEST_SERVICE_ID, &issue.id, &issue_run.run_id, 1) + .expect("private events should load"); + + assert!(summary.continuation_pending); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_transition" + && event.payload()["signal"] == "validation_fail" + && event.payload()["payload"]["disposition"] == "continue_repair" + })); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_next" + && event.payload()["phase"] == "repair_validation_failures" + })); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_recovery" + && event.payload()["payload"]["nextPhase"] == "repair_validation_failures" + })); +} + #[test] fn implementation_phase_goal_contract_requires_explicit_goal_completion() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 6ede39ee3..2bdc8cb7b 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -172,6 +172,12 @@ and boundary disposition, reason, attempted recovery, changed-surface count, and improvement-signal count. These summaries are safe operator readback; raw reviewer finding bodies, changed-surface payloads, retained diffs, logs, and transcripts remain hidden unless `--include-payload` is explicitly requested for local repair. +Private phase-goal evidence may also include `phase_goal_recovery`. That event means +Decodex found a still-active implementation or repair phase goal after an app-server +failure or child exit, ran the registered repo gate itself, persisted the next phase, +and scheduled continuation instead of writing `decodex:needs-attention`. It is a +runtime recovery handoff, not final issue success; the later `handoff_evidence` phase +still owns review, push, PR creation, and terminal finalize. ## State Ownership @@ -290,7 +296,7 @@ protocol activity durable outside the local operator surface. | `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. Repeated identical open dependency blockers surface as `dependency_program_stale` after the guardrail threshold so operators can distinguish a stale Execution Program/dependency plan from a newly blocked queue item. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after stalled reconciliation, failure writeback, or retries, the candidate is partial retained progress and not just a generic stalled or retry-budget hold. Human-required authority stops expose their compact decision request fields here: `phase = human_required`, reason, boundary, `decision_request_id`, and `next_action`. When queued attention still maps to a run/attempt, it also carries the same compact loop status used by running lanes. Running lanes are not repeated as normal intake work. | | `Review & Landing` | Retained PR lanes after review handoff. This section owns post-review repair, wait-for-review, ready-to-land, closeout, cleanup, and blocked retained-lane visibility. Retained lanes expose compact loop status for their bound handoff run/attempt so operators can see review repair checkpoint state, architecture recovery stops, and boundary/human-required disposition without direct SQLite inspection. | | `Recovery Worktrees` | Retained local worktrees that are not currently owned by `Running Lanes`, `Review & Landing`, or queued attention in `Intake Queue`. This is the cleanup or recovery inbox for recovered paths, retained PR leftovers, and cleanup-only local worktrees. Empty is the normal healthy state. | -| `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Raw local attempts and heartbeat details stay in debug expansion. | +| `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Terminal attention rows are history unless a current attention signal still exists. Raw local attempts and heartbeat details stay in debug expansion. | ## Private Evidence Readback @@ -461,8 +467,15 @@ Worktree visibility follows the owning dashboard section: - `retained_attention` in `Recovery Worktrees` means the durable Run Ledger final outcome for the same issue is `needs_attention` or `terminal_failure`. This is a human-required retained lane, not neutral cleanup hygiene. The project summary - `attention_count` includes it even when `queued_candidates` is empty and no active - or post-review lane currently owns the issue. + `attention_count` includes it because the retained worktree is current recovery + state even when `queued_candidates` is empty and no active or post-review lane + currently owns the issue. A terminal Run Ledger attention row without a retained + worktree, queued attention row, active or needs-attention tracker label, or blocked + post-review lane is history-only and must not inflate current attention. +- If private evidence shows `phase_goal_recovery` followed by a queued continuation, + the lane is not a retained-attention worktree even when the preceding child failed. + Treat it as Decodex-owned re-entry into the next phase unless a later terminal Run + Ledger row or current attention signal supersedes it. Every operator snapshot worktree row includes `ownership`, `ownership_reason`, `provenance`, and optional `recovery_next_action` fields that distinguish active-lane @@ -604,8 +617,8 @@ rate-limited, or unavailable. - When a terminal Run Ledger attention record exists for an issue that still has both the service queue label and `decodex:needs-attention`, the operator snapshot treats the queue label as stale echo state. The issue remains visible through Run Ledger - attention and retained-worktree ownership instead of appearing again as intake - backlog. + attention and any retained-worktree or tracker-label attention signal instead of + appearing again as intake backlog. ## Current Non-Goals diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 30882e34a..957ae552d 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -291,11 +291,24 @@ the repo gate and select the next phase. An `issue_progress_checkpoint`, final c text, or "await next phase" statement is evidence only; it is not a phase exit and must not be treated as a substitute for goal completion. +If an app-server run fails or a supervised child exits unsuccessfully while the latest +private phase-goal signal for that same run is still an `active` implementation or +repair phase, Decodex must run the registered repo gate before converting retained +tracked changes into human attention. When that runtime-owned gate returns a +continuation transition, Decodex records `phase_goal_recovery`, persists the next +phase goal (`handoff_evidence` after validation pass or `repair_validation_failures` +after continued-repair validation failure), marks the attempt as +`continuation_pending`, and schedules normal continuation re-entry. This recovery path +does not apply to `handoff_evidence`, explicit manual-attention terminal intent, +unsupported phase-goal app-server methods, repo-gate human-attention failures, or +runs with no current active phase-goal signal. + Phase-goal telemetry is local runtime evidence. It must distinguish -`goal_complete`, `validation_pass`, `validation_fail`, review `clean`, review -`findings`, terminal `review_handoff`, and terminal `manual_attention`. These signals -may appear in private execution events and operator protocol activity, but Linear -receives only the existing low-frequency lifecycle projections. +`goal_complete`, `validation_pass`, `validation_fail`, `active_goal_recovered`, +review `clean`, review `findings`, terminal `review_handoff`, and terminal +`manual_attention`. These signals may appear in private execution events and operator +protocol activity, but Linear receives only the existing low-frequency lifecycle +projections. ## Tracker write ownership @@ -750,7 +763,7 @@ After a process restart, recent-run history, active lease ownership, retained po - Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, optional progress diagnostics, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. `last_progress_at` is meaningful-work progress only: tool calls, file or diff changes, plan/model output, repo validation, PR/review/terminal lifecycle, or other explicit work events may refresh it, but account, rate-limit, phase-goal, passive status, warning, token-usage, heartbeat, or similar non-work protocol traffic must only refresh protocol liveness. When a lane remains in `model_execution` with fresh protocol activity but stale or missing work progress and the recent protocol events are only non-work traffic, status should expose `progress_diagnostic = "protocol_only_activity"` while preserving process and protocol liveness separately. - Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it. - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. -- If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. +- If retry budget or needs-attention recovery finds tracked changes in the retained worktree after active phase-goal recovery has no applicable continuation path, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. - A retryable runtime or app-server failure that leaves tracked worktree changes must stop as retained partial progress instead of writing only a generic retry comment. This does not apply to repo-gate continued-repair failures, because those failures @@ -758,16 +771,13 @@ After a process restart, recent-run history, active lease ownership, retained po loop-guardrail limits are reached. - If the durable Run Ledger final outcome is `needs_attention` or `terminal_failure`, operator status must count that issue in project-level - `attention_count` even when no active run, queued candidate, or post-review lane - currently owns it. A retained worktree for that same issue must be projected as - retained attention, not neutral cleanup-only hygiene, so monitors do not need to - parse `history_lanes` to discover a human-required terminal outcome. When live - tracker readback proves the issue is in the configured completed state, the service - queue, active, and needs-attention labels are absent, no retained worktree or - post-review lane owns the issue, and an explicit `recover merged-closeout` path has - or can write `closeout` plus `cleanup_complete` records after validating the merged - PR lineage, that stale terminal attention is a resolved history echo rather than - current project attention. + `attention_count` only when a current attention signal still exists: a retained + worktree, queued attention row, active or needs-attention tracker label, or a + blocked post-review lane. A retained worktree for that same issue must be projected + as retained attention, not neutral cleanup-only hygiene, so monitors do not need to + parse `history_lanes` to discover a human-required terminal outcome. A bare terminal + Run Ledger attention row with no current owner is history-only ledger evidence; it + must remain visible in `history_lanes` without inflating current project attention. - If that same issue still carries the service queue label plus the configured `needs_attention_label`, the terminal Run Ledger attention outcome must own the operator projection. Status must not also render the issue as an intake queue