Skip to content

XY-777: [TRADING-PROD-BLOCKER] Recover lane controls when active process loses active lease #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yvette-carlisle merged 1 commit into from
Jun 7, 2026
Merged

XY-777: [TRADING-PROD-BLOCKER] Recover lane controls when active process loses active lease #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 84 additions & 15 deletions apps/decodex/src/orchestrator/lane_control.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use std::{

use libc::{ESRCH, SIGKILL, SIGTERM, c_int, pid_t};
use serde::Serialize;
use serde_json::Value;

use crate::{
config::ServiceConfig,
Expand Down Expand Up @@ -393,8 +394,12 @@ pub(super) fn steer_lane_with_state(
}

fn soft_interrupt_allows_hard_fallback(soft: &LaneSoftInterruptReport) -> bool {
matches!(soft.status.as_str(), "pending" | "failed" | "unavailable")
&& soft.error_class.as_deref() != Some("lane_not_active")
match soft.status.as_str() {
"pending" | "failed" | "unavailable" =>
soft.error_class.as_deref() != Some("lane_not_active"),
"rejected" => soft.error_class.as_deref() == Some("active_lease_missing"),
_ => false,
}
}

fn load_lane_control_project(
Expand Down Expand Up @@ -510,6 +515,45 @@ fn soft_interrupt_available_for_run(run: &OperatorRunStatus) -> bool {
&& run.control_capability.as_ref().is_some_and(|capability| capability.status == "active")
}

fn lane_control_operator_context(run: &OperatorRunStatus) -> Value {
let control_capability = run.control_capability.as_ref().map(|capability| {
serde_json::json!({
"project_id": capability.project_id.as_str(),
"issue_id": capability.issue_id.as_str(),
"run_id": capability.run_id.as_str(),
"attempt_number": capability.attempt_number,
"thread_id": capability.thread_id.as_deref(),
"turn_id": capability.turn_id.as_deref(),
"transport": capability.transport.as_str(),
"channel_path": capability.channel_path.as_str(),
"status": capability.status.as_str(),
"published_at": capability.published_at.as_str(),
"updated_at": capability.updated_at.as_str(),
})
});

serde_json::json!({
"status": run.status.as_str(),
"attempt_status": run.attempt_status.as_str(),
"phase": run.phase.as_str(),
"wait_reason": run.wait_reason.as_deref(),
"current_operation": run.current_operation.as_str(),
"active_lease": run.active_lease,
"queue_lease_state": run.queue_lease_state.as_str(),
"execution_liveness": run.execution_liveness.as_str(),
"thread_status": run.thread_status.as_deref(),
"process_id": run.process_id,
"process_alive": run.process_alive,
"process_liveness_reason": run.process_liveness_reason.as_deref(),
"branch": run.branch_name.as_deref(),
"worktree_path": run.worktree_path.as_deref(),
"last_event_type": run.last_event_type.as_deref(),
"last_event_at": run.last_event_at.as_deref(),
"event_count": run.event_count,
"control_capability": control_capability,
})
}

fn attempt_lane_steer(
state_store: &StateStore,
project: &ServiceConfig,
Expand All @@ -518,6 +562,7 @@ fn attempt_lane_steer(
) -> Result<LaneSteerReport> {
let message_byte_count = request.message.len();
let message_line_count = lane_steer_message_line_count(request.message);
let context = lane_control_operator_context(run);
let metadata = serde_json::json!({
"expectedTurnId": request.expected_turn_id,
"messageByteCount": message_byte_count,
Expand All @@ -534,6 +579,7 @@ fn attempt_lane_steer(
action: "steer",
timeout_ms: Some(i64::try_from(request.wait_timeout.as_millis()).unwrap_or(i64::MAX)),
metadata: Some(&metadata),
context: Some(&context),
})?;

if receipt.outcome() != RUN_CONTROL_ACTION_ACCEPTED {
Expand Down Expand Up @@ -793,20 +839,14 @@ fn attempt_soft_lane_interrupt(
));
}

let receipt = state_store.resolve_run_control_action(RunControlActionRequest {
project_id: project.service_id(),
issue_id: &run.issue_id,
run_id: &run.run_id,
attempt_number: run.attempt_number,
thread_id: Some(thread_id),
turn_id: Some(turn_id),
let receipt = resolve_soft_interrupt_control_action(
state_store,
project,
run,
thread_id,
turn_id,
source,
action: "interrupt",
timeout_ms: Some(
i64::try_from(LANE_INTERRUPT_RESPONSE_WAIT.as_millis()).unwrap_or(i64::MAX),
),
metadata: None,
})?;
)?;

if receipt.outcome() != "accepted" {
return Ok(LaneSoftInterruptReport::from_control_rejection(&receipt));
Expand Down Expand Up @@ -886,6 +926,33 @@ fn attempt_soft_lane_interrupt(
}
}

fn resolve_soft_interrupt_control_action(
state_store: &StateStore,
project: &ServiceConfig,
run: &OperatorRunStatus,
thread_id: &str,
turn_id: &str,
source: &str,
) -> Result<RunControlActionReceipt> {
let context = lane_control_operator_context(run);

state_store.resolve_run_control_action(RunControlActionRequest {
project_id: project.service_id(),
issue_id: &run.issue_id,
run_id: &run.run_id,
attempt_number: run.attempt_number,
thread_id: Some(thread_id),
turn_id: Some(turn_id),
source,
action: "interrupt",
timeout_ms: Some(
i64::try_from(LANE_INTERRUPT_RESPONSE_WAIT.as_millis()).unwrap_or(i64::MAX),
),
metadata: None,
context: Some(&context),
})
}

fn attempt_hard_lane_interrupt(
state_store: &StateStore,
run: &OperatorRunStatus,
Expand Down Expand Up @@ -978,6 +1045,7 @@ fn record_hard_interrupt_control_fallback(
run: &OperatorRunStatus,
_reason: Option<&str>,
) -> Result<()> {
let context = lane_control_operator_context(run);
let receipt = state_store.resolve_run_control_action(RunControlActionRequest {
project_id: &run.project_id,
issue_id: &run.issue_id,
Expand All @@ -989,6 +1057,7 @@ fn record_hard_interrupt_control_fallback(
action: "interrupt",
timeout_ms: None,
metadata: None,
context: Some(&context),
})?;

state_store.record_run_control_action_outcome(
Expand Down
50 changes: 43 additions & 7 deletions apps/decodex/src/orchestrator/status.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ fn build_operator_status_snapshot_with_account_mode(
active_runs.iter().map(|run| run.run_id.clone()).collect::<HashSet<_>>();

for run in &recent_runs {
if !active_run_ids.contains(&run.run_id) && operator_run_has_live_process(run) {
if !active_run_ids.contains(&run.run_id) && operator_run_has_live_execution(run) {
active_run_ids.insert(run.run_id.clone());
active_runs.push(run.clone());
}
Expand Down Expand Up @@ -1050,12 +1050,16 @@ fn worktree_cleanup_key(worktree: &OperatorWorktreeStatus) -> String {
}

fn operator_run_counts_as_active(run: &OperatorRunStatus) -> bool {
(run.active_lease || operator_run_has_live_process(run))
(run.active_lease || operator_run_has_live_execution(run))
&& !matches!(run.phase.as_str(), "completed" | "failed" | "terminated")
}

fn operator_run_has_live_process(run: &OperatorRunStatus) -> bool {
matches!(run.status.as_str(), "starting" | "running") && run.process_alive == Some(true)
fn operator_run_has_live_execution(run: &OperatorRunStatus) -> bool {
matches!(run.status.as_str(), "starting" | "running")
&& matches!(
run.execution_liveness.as_str(),
"process_alive" | "thread_active" | "protocol_observed"
)
}

fn operator_run_counts_as_running(run: &OperatorRunStatus) -> bool {
Expand Down Expand Up @@ -4348,8 +4352,14 @@ fn operator_run_status(
let timing = operator_run_timing(&run, marker.as_ref(), now_unix_epoch);
let app_server_state = operator_run_app_server_state(&run, marker.as_ref());
let protocol_summary = operator_run_protocol_summary(&run, marker.as_ref());
let status =
operator_run_visible_status(run.status(), &app_server_state, &protocol_summary, &timing);
let marker_current_operation = marker.as_ref().and_then(RunActivityMarker::current_operation);
let status = operator_run_visible_status(
run.status(),
&app_server_state,
&protocol_summary,
&timing,
marker_current_operation,
);
let (retry_kind, retry_ready_at_unix_epoch) = visible_operator_run_retry_schedule(
&status,
marker.as_ref().and_then(RunActivityMarker::retry_kind),
Expand All @@ -4364,7 +4374,7 @@ fn operator_run_status(
);
let current_operation = classify_operator_run_operation(
&phase,
marker.as_ref().and_then(RunActivityMarker::current_operation),
marker_current_operation,
);
let suspected_stall = operator_run_is_suspected_stall(
&phase,
Expand Down Expand Up @@ -4706,6 +4716,7 @@ fn operator_run_visible_status(
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
timing: &OperatorRunTiming,
marker_current_operation: Option<&str>,
) -> String {
if attempt_status == "starting"
&& operator_run_has_app_server_execution_evidence(
Expand All @@ -4716,10 +4727,35 @@ fn operator_run_visible_status(
{
return String::from("running");
}
if matches!(attempt_status, "failed" | "interrupted" | "stalled")
&& operator_marker_operation_allows_terminal_status_promotion(marker_current_operation)
&& operator_run_has_live_execution_evidence(app_server_state, timing)
{
return String::from("running");
}

attempt_status.to_owned()
}

fn operator_marker_operation_allows_terminal_status_promotion(
marker_current_operation: Option<&str>,
) -> bool {
matches!(marker_current_operation, None | Some(RUN_OPERATION_AGENT_RUN))
}

fn operator_run_has_live_execution_evidence(
app_server_state: &OperatorRunAppServerState,
timing: &OperatorRunTiming,
) -> bool {
timing.process_alive == Some(true)
|| matches!(app_server_state.thread_status.as_deref(), Some("active"))
|| !app_server_state.thread_active_flags.is_empty()
|| timing.protocol_idle_for_seconds.is_some_and(|idle_for| {
u64::try_from(idle_for)
.is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs())
})
}

fn operator_run_has_app_server_execution_evidence(
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
Expand Down
Loading