From 7314bab2ec97a8be71f34bfd6e64309e7db0793f Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 12 Jun 2026 08:31:55 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Stop retryable runtime failures with retained progress","authority":"manual","related":["XY-941"]} --- apps/decodex/src/orchestrator/execution.rs | 24 ++++-- .../tests/recovery/terminal_failures.rs | 77 ++++++++++++++++++- docs/spec/runtime.md | 5 ++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 7cea87d34..e0cc5f792 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -2956,6 +2956,14 @@ where } else { retryable_failure_loop_guardrail_stop(project, state_store, issue_run, error)? }; + let retained_partial_progress = retained_partial_progress_error( + error, + issue_run, + &worktree_path, + ); + let retryable_retained_partial_progress = retained_partial_progress + .as_ref() + .filter(|_| retryable_failure_should_stop_for_retained_progress(error)); if let Some(review_policy_stop) = error.downcast_ref::() && review_policy_stop.reason == ReviewPolicyStopReason::Exhausted @@ -2988,15 +2996,13 @@ where }; } - if !requires_terminal_attention && retry_budget_attempts < max_attempts { + if !requires_terminal_attention + && retry_budget_attempts < max_attempts + && retryable_retained_partial_progress.is_none() + { return apply_retryable_failure_writeback(&failure_context, error, max_attempts); } - let retained_partial_progress = retained_partial_progress_error( - error, - issue_run, - &worktree_path, - ); let terminal_error = retained_partial_progress.as_ref().unwrap_or(error); let privacy_classifier = configured_public_projection_privacy_classifier(project)?; let outcome = apply_terminal_failure_writeback( @@ -3279,6 +3285,12 @@ fn retained_partial_progress_error( })) } +fn retryable_failure_should_stop_for_retained_progress(error: &Report) -> bool { + !error.downcast_ref::().is_some_and(|repo_gate_failure| { + repo_gate_failure.disposition() == RepoGateFailureDisposition::ContinueRepair + }) +} + fn retained_progress_should_defer_to_terminal_intent(error: &Report) -> bool { error.downcast_ref::().is_some() || error.downcast_ref::().is_some() diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index 3cf3a8869..bffeedfd1 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -232,7 +232,82 @@ fn terminal_failure_with_retained_tracked_changes_records_retained_partial_progr .is_some_and(|evidence| evidence .iter() .any(|item| item.contains("Source failure class `repo_gate_command_spawn_failed`"))), - "retained partial progress evidence should preserve the source failure class" + "retained partial progress evidence should preserve the source failure class" + ); +} + +#[test] +fn retryable_runtime_failure_with_retained_tracked_changes_records_retained_partial_progress() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join("PUB-102"); + + git_status_success( + config.repo_root(), + &["worktree", "add", "-b", "x/pubfi-pub-102", ".worktrees/PUB-102", "main"], + ); + + fs::write(worktree_path.join("README.md"), "retained validation-ready runtime patch\n") + .expect("tracked worktree file should change"); + + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-102"), + issue_identifier: issue.identifier.clone(), + path: worktree_path, + reused_existing: true, + }, + retry_project_slug: issue.project_slug.clone().expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-102-attempt-1-123"), + retry_budget_base: 0, + }; + let error = Report::msg("app-server run ended after a validation-ready checkpoint without a terminal path"); + + state_store + .record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed") + .expect("run attempt should record"); + + orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) + .expect("dirty generic runtime failure should retain partial progress"); + + let comments = tracker.comments.borrow(); + + assert!(comments.iter().any(|comment| { + comment.contains("decodex retained partial progress and needs attention") + && comment.contains("partial_progress_retained") + && comment.contains("finish validation and PR handoff or reset the patch manually") + })); + assert!( + comments + .iter() + .all(|comment| !comment.contains("retryable_execution_failure")), + "retained validation-ready work must not be hidden behind a generic retry comment" + ); + + let ledger_event = comments + .iter() + .find_map(|comment| records::parse_linear_execution_event_record(comment)) + .expect("retained generic runtime failure should write a Linear execution event"); + + assert_eq!(ledger_event.event_type, "needs_attention"); + assert_eq!(ledger_event.error_class.as_deref(), Some("partial_progress_retained")); + assert_eq!(ledger_event.terminal_path.as_deref(), Some("retained_partial_progress")); + assert!( + ledger_event + .evidence + .as_deref() + .is_some_and(|evidence| evidence + .iter() + .any(|item| item.contains("tracked worktree changes retained"))), + "retained progress evidence should identify the retained tracked patch" ); } diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index dbe6c9db3..b58f8ec44 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -744,6 +744,11 @@ After a process restart, recent-run history, active lease ownership, retained po - Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it. - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. - If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. +- A retryable runtime or app-server failure that leaves tracked worktree changes must + stop as retained partial progress instead of writing only a generic retry comment. + This does not apply to repo-gate continued-repair failures, because those failures + still authorize bounded automatic repair against the retained patch until retry or + loop-guardrail limits are reached. - If the durable Run Ledger final outcome is `needs_attention` or `terminal_failure`, operator status must count that issue in project-level `attention_count` even when no active run, queued candidate, or post-review lane