Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1023,7 +1023,7 @@ fn rejects_manual_attention_comments_with_private_or_unsupported_fields() {
}

#[test]
fn rejects_manual_attention_comment_with_runtime_owned_retryable_error_class() {
fn rejects_manual_attention_comment_with_runtime_owned_error_class() {
for error_class in [
"retryable_execution_failure",
"repo_gate_verify_failed",
Expand Down Expand Up @@ -1072,7 +1072,7 @@ fn rejects_manual_attention_comment_with_runtime_owned_retryable_error_class() {
assert!(matches!(
response.content_items.as_slice(),
[DynamicToolContentItem::InputText { text }]
if text.contains("cannot use runtime-owned retryable error class")
if text.contains("cannot use runtime-owned error class")
&& text.contains(error_class)
));
}
Expand Down
2 changes: 1 addition & 1 deletion apps/decodex/src/agent/tracker_tool_bridge/tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2214,7 +2214,7 @@ fn validate_manual_attention_error_class(error_class: &str) -> Result<(), String

if is_runtime_owned_manual_attention_error_class(error_class) {
return Err(format!(
"`{ISSUE_COMMENT_TOOL_NAME}` kind `{COMMENT_KIND_MANUAL_ATTENTION}` cannot use runtime-owned retryable error class `{error_class}`; keep repairing or retrying the lane, or use a human-owned blocker class only when automation cannot clear the blocker."
"`{ISSUE_COMMENT_TOOL_NAME}` kind `{COMMENT_KIND_MANUAL_ATTENTION}` cannot use runtime-owned error class `{error_class}`; keep repairing, retrying, or letting Decodex retain the lane, and use a human-owned blocker class only when automation cannot clear the blocker."
));
}

Expand Down
89 changes: 72 additions & 17 deletions apps/decodex/src/orchestrator/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ enum RetryEntryRetentionDecision {
Block,
}

enum ChildExitPhaseGoalRecovery {
None,
Continuation(PhaseGoalRecoveryContinuation),
Terminalized,
}

struct ChildExitRetrySchedule<'a> {
issue_id: &'a str,
run_id: &'a str,
Expand Down Expand Up @@ -1045,7 +1051,7 @@ where
}

fn schedule_retry_after_child_exit<T>(
context: ChildExitRetryContext<'_, T>,
mut context: ChildExitRetryContext<'_, T>,
child: ChildRunRef<'_>,
#[cfg(test)]
_retry_project_slug: &str,
Expand Down Expand Up @@ -1087,8 +1093,7 @@ where

return Ok(());
};
let continuation_pending =
exit_status.success() && run_attempt.status() == CONTINUATION_PENDING_RUN_STATUS;
let continuation_pending = exit_status.success() && run_attempt.status() == CONTINUATION_PENDING_RUN_STATUS;

if !exit_status.success() && run_attempt.status() != "failed" {
clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?;
Expand All @@ -1110,16 +1115,18 @@ where
return Ok(());
}

let recovered_phase_goal_continuation = if !exit_status.success() {
maybe_recover_child_exit_active_phase_goal_continuation(
&context,
&issue,
child,
initial_issue_state,
dispatch_mode,
)?
} else {
None
let recovered_phase_goal_continuation = match recover_child_exit_phase_goal(
&mut context,
&issue,
child,
issue_id,
initial_issue_state,
dispatch_mode,
exit_status.success(),
)? {
ChildExitPhaseGoalRecovery::None => None,
ChildExitPhaseGoalRecovery::Continuation(recovery) => Some(recovery),
ChildExitPhaseGoalRecovery::Terminalized => return Ok(()),
};
let (kind, attempt, continuation_initial_issue_state) = if continuation_pending {
(
Expand Down Expand Up @@ -1219,13 +1226,44 @@ fn queue_child_exit_retry(
Ok(())
}

fn recover_child_exit_phase_goal<T>(
context: &mut ChildExitRetryContext<'_, T>,
issue: &TrackerIssue,
child: ChildRunRef<'_>,
issue_id: &str,
initial_issue_state: &str,
dispatch_mode: IssueDispatchMode,
exit_success: bool,
) -> Result<ChildExitPhaseGoalRecovery>
where
T: IssueTracker,
{
if exit_success {
return Ok(ChildExitPhaseGoalRecovery::None);
}

let recovery = maybe_recover_child_exit_active_phase_goal_continuation(
context,
issue,
child,
initial_issue_state,
dispatch_mode,
)?;

if matches!(recovery, ChildExitPhaseGoalRecovery::Terminalized) {
clear_retry_schedule_and_release(context.retry_queue, context.state_store, issue_id)?;
}

Ok(recovery)
}

fn maybe_recover_child_exit_active_phase_goal_continuation<T>(
context: &ChildExitRetryContext<'_, T>,
issue: &TrackerIssue,
child: ChildRunRef<'_>,
initial_issue_state: &str,
dispatch_mode: IssueDispatchMode,
) -> Result<Option<PhaseGoalRecoveryContinuation>>
) -> Result<ChildExitPhaseGoalRecovery>
where
T: IssueTracker,
{
Expand All @@ -1242,13 +1280,28 @@ where
run_id: child.run_id.to_owned(),
retry_budget_base: 0,
};
let recovery = recover_active_phase_goal_continuation(
let recovery = match recover_active_phase_goal_continuation(
context.project,
context.workflow,
context.state_store,
&issue_run,
"child_exit_failed",
)?;
) {
Ok(recovery) => recovery,
Err(error) if run_failure_requires_terminal_attention(&error) => {
handle_failure(
context.tracker,
context.project,
context.workflow,
context.state_store,
&issue_run,
&error,
)?;

return Ok(ChildExitPhaseGoalRecovery::Terminalized);
},
Err(error) => return Err(error),
};

if let Some(recovery) = &recovery {
tracing::warn!(
Expand All @@ -1263,7 +1316,9 @@ where
);
}

Ok(recovery)
Ok(recovery.map_or(ChildExitPhaseGoalRecovery::None, |recovery| {
ChildExitPhaseGoalRecovery::Continuation(recovery)
}))
}

fn child_exit_retry_retention_decision<T>(
Expand Down
8 changes: 4 additions & 4 deletions apps/decodex/src/orchestrator/git_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ mod repo_gate_failure {
fn disposition(self) -> RepoGateFailureDisposition {
match self {
Self::CanonicalizeCommandFailed
| Self::VerifyCommandFailed
| Self::TrackedRewritesLeft => RepoGateFailureDisposition::ContinueRepair,
| Self::VerifyCommandFailed => RepoGateFailureDisposition::ContinueRepair,
Self::TrackedRewritesLeft => RepoGateFailureDisposition::NeedsHumanAttention,
Self::GitLockContention => RepoGateFailureDisposition::RetryAfterBackoff,
Self::CommandSpawnFailed | Self::CleanlinessCheckFailed => {
RepoGateFailureDisposition::NeedsHumanAttention
Expand All @@ -59,7 +59,7 @@ mod repo_gate_failure {
"additional agent repair is required before repo verification can pass; decodex will retry automatically"
},
Self::TrackedRewritesLeft => {
"additional agent repair is required to reconcile repo-gate tracked rewrites before handoff; decodex will retry automatically"
"automatic retry is stopped because the repo gate left tracked rewrites after completing; inspect the retained worktree manually"
},
Self::GitLockContention => {
"another Git process appears to hold `.git/index.lock`; decodex will wait briefly, refresh lane state, and retry automatically"
Expand All @@ -82,7 +82,7 @@ mod repo_gate_failure {
"inspect the worktree, repair the repo verification failure manually, {recovery_gate}"
),
Self::TrackedRewritesLeft => format!(
"inspect the worktree, reconcile the tracked rewrites left by the repo gate manually, {recovery_gate}"
"inspect the retained worktree, decide whether the tracked rewrites are in scope, then finish validation and PR handoff or reset the patch manually, {recovery_gate}"
),
Self::GitLockContention => format!(
"inspect the worktree for an active or stale `.git/index.lock` holder, clear the Git lock contention manually, {recovery_gate}"
Expand Down
Loading