Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions apps/decodex/src/orchestrator/execution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2956,6 +2956,14 @@ where
} else {
retryable_failure_loop_guardrail_stop(project, state_store, issue_run, error)?
};
let retained_partial_progress = retained_partial_progress_error(
error,
issue_run,
&worktree_path,
);
let retryable_retained_partial_progress = retained_partial_progress
.as_ref()
.filter(|_| retryable_failure_should_stop_for_retained_progress(error));

if let Some(review_policy_stop) = error.downcast_ref::<ReviewPolicyStopRequested>()
&& review_policy_stop.reason == ReviewPolicyStopReason::Exhausted
Expand Down Expand Up @@ -2988,15 +2996,13 @@ where
};
}

if !requires_terminal_attention && retry_budget_attempts < max_attempts {
if !requires_terminal_attention
&& retry_budget_attempts < max_attempts
&& retryable_retained_partial_progress.is_none()
{
return apply_retryable_failure_writeback(&failure_context, error, max_attempts);
}

let retained_partial_progress = retained_partial_progress_error(
error,
issue_run,
&worktree_path,
);
let terminal_error = retained_partial_progress.as_ref().unwrap_or(error);
let privacy_classifier = configured_public_projection_privacy_classifier(project)?;
let outcome = apply_terminal_failure_writeback(
Expand Down Expand Up @@ -3279,6 +3285,12 @@ fn retained_partial_progress_error(
}))
}

fn retryable_failure_should_stop_for_retained_progress(error: &Report) -> bool {
!error.downcast_ref::<RepoGateFailure>().is_some_and(|repo_gate_failure| {
repo_gate_failure.disposition() == RepoGateFailureDisposition::ContinueRepair
})
}

fn retained_progress_should_defer_to_terminal_intent(error: &Report) -> bool {
error.downcast_ref::<ManualAttentionRequested>().is_some()
|| error.downcast_ref::<LoopGuardrailStopRequested>().is_some()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,82 @@ fn terminal_failure_with_retained_tracked_changes_records_retained_partial_progr
.is_some_and(|evidence| evidence
.iter()
.any(|item| item.contains("Source failure class `repo_gate_command_spawn_failed`"))),
"retained partial progress evidence should preserve the source failure class"
"retained partial progress evidence should preserve the source failure class"
);
}

#[test]
fn retryable_runtime_failure_with_retained_tracked_changes_records_retained_partial_progress() {
let (_temp_dir, config, workflow) = temp_project_layout();
let active_label = tracker::automation_active_label(TEST_SERVICE_ID);
let issue = sample_issue("In Progress", &[active_label.as_str()]);
let tracker = FakeTracker::new(vec![issue.clone()]);
let state_store = StateStore::open_in_memory().expect("state store should open");
let worktree_path = config.worktree_root().join("PUB-102");

git_status_success(
config.repo_root(),
&["worktree", "add", "-b", "x/pubfi-pub-102", ".worktrees/PUB-102", "main"],
);

fs::write(worktree_path.join("README.md"), "retained validation-ready runtime patch\n")
.expect("tracked worktree file should change");

let issue_run = IssueRunPlan {
issue: issue.clone(),
issue_state: String::from("In Progress"),
initial_issue_state: String::from("Todo"),
worktree: WorktreeSpec {
branch_name: String::from("x/pubfi-pub-102"),
issue_identifier: issue.identifier.clone(),
path: worktree_path,
reused_existing: true,
},
retry_project_slug: issue.project_slug.clone().expect("sample issue should carry a project slug"),
dispatch_mode: IssueDispatchMode::Normal,
attempt_number: 1,
run_id: String::from("pub-102-attempt-1-123"),
retry_budget_base: 0,
};
let error = Report::msg("app-server run ended after a validation-ready checkpoint without a terminal path");

state_store
.record_run_attempt(&issue_run.run_id, &issue.id, issue_run.attempt_number, "failed")
.expect("run attempt should record");

orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error)
.expect("dirty generic runtime failure should retain partial progress");

let comments = tracker.comments.borrow();

assert!(comments.iter().any(|comment| {
comment.contains("decodex retained partial progress and needs attention")
&& comment.contains("partial_progress_retained")
&& comment.contains("finish validation and PR handoff or reset the patch manually")
}));
assert!(
comments
.iter()
.all(|comment| !comment.contains("retryable_execution_failure")),
"retained validation-ready work must not be hidden behind a generic retry comment"
);

let ledger_event = comments
.iter()
.find_map(|comment| records::parse_linear_execution_event_record(comment))
.expect("retained generic runtime failure should write a Linear execution event");

assert_eq!(ledger_event.event_type, "needs_attention");
assert_eq!(ledger_event.error_class.as_deref(), Some("partial_progress_retained"));
assert_eq!(ledger_event.terminal_path.as_deref(), Some("retained_partial_progress"));
assert!(
ledger_event
.evidence
.as_deref()
.is_some_and(|evidence| evidence
.iter()
.any(|item| item.contains("tracked worktree changes retained"))),
"retained progress evidence should identify the retained tracked patch"
);
}

Expand Down
5 changes: 5 additions & 0 deletions docs/spec/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,11 @@ After a process restart, recent-run history, active lease ownership, retained po
- Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it.
- If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane.
- If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly.
- A retryable runtime or app-server failure that leaves tracked worktree changes must
stop as retained partial progress instead of writing only a generic retry comment.
This does not apply to repo-gate continued-repair failures, because those failures
still authorize bounded automatic repair against the retained patch until retry or
loop-guardrail limits are reached.
- If the durable Run Ledger final outcome is `needs_attention` or
`terminal_failure`, operator status must count that issue in project-level
`attention_count` even when no active run, queued candidate, or post-review lane
Expand Down