Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 75 additions & 1 deletion apps/decodex/src/orchestrator/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ struct OperatorTerminalFinalizeProjection {

struct OperatorRunLifecycleProjection {
status: String,
status_projection_reason: Option<String>,
phase: String,
wait_reason: Option<String>,
current_operation: String,
Expand Down Expand Up @@ -1124,6 +1125,7 @@ fn apply_terminal_history_ledger_outcome_to_latest_run(lane: &mut OperatorHistor

lane.latest_run.status = final_outcome.clone();
lane.latest_run.attempt_status = final_outcome;
lane.latest_run.status_projection_reason = None;
lane.latest_run.phase = String::from(if requires_attention { "needs_attention" } else { "completed" });
lane.latest_run.wait_reason = None;
lane.latest_run.current_operation = String::from("ledger_outcome");
Expand Down Expand Up @@ -5504,6 +5506,7 @@ fn operator_run_status(
attempt_number: run.attempt_number(),
status: lifecycle.status,
attempt_status: run.status().to_owned(),
status_projection_reason: lifecycle.status_projection_reason,
phase: lifecycle.phase,
wait_reason,
current_operation: lifecycle.current_operation,
Expand Down Expand Up @@ -5574,6 +5577,18 @@ fn operator_run_lifecycle_projection(
marker_current_operation,
)
});
let status_projection_reason = if terminal_finalize_projection.is_some() {
None
} else {
operator_run_status_projection_reason(
run.status(),
&status,
app_server_state,
protocol_summary,
timing,
marker_current_operation,
)
};
let (retry_kind, retry_ready_at_unix_epoch) = visible_operator_run_retry_schedule(
&status,
marker.and_then(RunActivityMarker::retry_kind),
Expand Down Expand Up @@ -5610,6 +5625,7 @@ fn operator_run_lifecycle_projection(

OperatorRunLifecycleProjection {
status,
status_projection_reason,
phase,
wait_reason,
current_operation,
Expand Down Expand Up @@ -6366,6 +6382,63 @@ fn operator_run_visible_status(
attempt_status.to_owned()
}

fn operator_run_status_projection_reason(
attempt_status: &str,
visible_status: &str,
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
timing: &OperatorRunTiming,
marker_current_operation: Option<&str>,
) -> Option<String> {
if attempt_status == visible_status || visible_status != "running" {
return None;
}

let projection_kind = if attempt_status == "starting" {
"starting_attempt"
} else if matches!(attempt_status, "failed" | "interrupted" | "stalled" | "succeeded")
&& operator_marker_operation_allows_terminal_status_promotion(marker_current_operation)
{
"terminal_attempt"
} else {
return None;
};

operator_run_live_evidence_source(app_server_state, protocol_summary, timing)
.map(|source| format!("{projection_kind}_promoted_by_{source}"))
}

fn operator_run_live_evidence_source(
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
timing: &OperatorRunTiming,
) -> Option<&'static str> {
if timing.process_alive == Some(true) {
return Some("process_alive");
}
if matches!(app_server_state.thread_status.as_deref(), Some("active")) {
return Some("thread_active");
}
if !app_server_state.thread_active_flags.is_empty() {
return Some("thread_active_flags");
}
if operator_run_has_recent_protocol_execution_evidence(protocol_summary, timing) {
return Some("recent_protocol_activity");
}
if app_server_state.effective_model.is_some()
|| app_server_state.effective_model_provider.is_some()
|| protocol_summary.event_count > 0
|| protocol_summary.last_event_type.is_some()
{
return Some("app_server_metadata");
}
if timing.protocol_idle_for_seconds.is_some() {
return Some("protocol_timing");
}

None
}

fn operator_run_has_live_process_or_thread_evidence(
app_server_state: &OperatorRunAppServerState,
timing: &OperatorRunTiming,
Expand Down Expand Up @@ -8205,7 +8278,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) {
let control_capability = render_control_capability_summary(run.control_capability.as_ref());

output.push_str(&format!(
"- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n progress_diagnostic: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n",
"- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n status_projection_reason: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n progress_diagnostic: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n",
run.run_id,
run.project_id,
run.issue_id,
Expand All @@ -8214,6 +8287,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) {
run.attempt_number,
run.status,
run.attempt_status,
run.status_projection_reason.as_deref().unwrap_or("none"),
run.phase,
run.wait_reason.as_deref().unwrap_or("none"),
run.current_operation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1502,6 +1502,10 @@ fn operator_status_snapshot_keeps_terminal_status_live_process_in_running_lanes(
assert_eq!(run.run_id, "run-1");
assert_eq!(run.status, "running");
assert_eq!(run.attempt_status, "failed");
assert_eq!(
run.status_projection_reason.as_deref(),
Some("terminal_attempt_promoted_by_process_alive")
);
assert_eq!(run.phase, "executing");
assert!(!run.active_lease);
assert_eq!(run.queue_lease_state, "not_held");
Expand Down Expand Up @@ -1542,6 +1546,60 @@ fn operator_status_snapshot_excludes_terminal_thread_archive_from_running_lanes(
);
}

#[test]
fn operator_status_snapshot_explains_terminal_run_promoted_by_active_thread() {
let (_temp_dir, config, _workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");
let issue = sample_issue("Todo", &[]);
let worktree_path = config.worktree_root().join("PUB-101");

state_store
.record_run_attempt("run-1", &issue.id, 1, "stalled")
.expect("run attempt should record");
state_store
.upsert_worktree(
"pubfi",
&issue.id,
"x/pubfi-pub-101",
&worktree_path.display().to_string(),
)
.expect("worktree should record");

state::write_run_thread_status_marker(
&worktree_path,
"run-1",
1,
Some("thread-1"),
Some("turn-1"),
"active",
&[],
)
.expect("thread status should write");

rewrite_run_activity_marker_host_boot_id(&worktree_path, "previous-boot");

let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10)
.expect("snapshot should build");
let run = snapshot.active_runs.first().expect("active run should remain visible");
let rendered = orchestrator::render_operator_status(&snapshot);

assert_eq!(run.status, "running");
assert_eq!(run.attempt_status, "stalled");
assert_eq!(
run.status_projection_reason.as_deref(),
Some("terminal_attempt_promoted_by_thread_active")
);
assert!(!run.active_lease);
assert_eq!(run.queue_lease_state, "not_held");
assert_eq!(run.execution_liveness, "process_identity_mismatch");
assert_eq!(run.process_alive, Some(false));
assert_eq!(run.process_liveness_reason.as_deref(), Some("host_boot_id_mismatch"));
assert_eq!(run.thread_status.as_deref(), Some("active"));
assert!(rendered.contains(
"status_projection_reason: terminal_attempt_promoted_by_thread_active"
));
}

#[test]
fn operator_status_snapshot_keeps_succeeded_status_live_process_in_running_lanes() {
let (_temp_dir, config, _workflow) = temp_project_layout();
Expand Down Expand Up @@ -1574,6 +1632,10 @@ fn operator_status_snapshot_keeps_succeeded_status_live_process_in_running_lanes
assert_eq!(run.run_id, "run-1");
assert_eq!(run.status, "running");
assert_eq!(run.attempt_status, "succeeded");
assert_eq!(
run.status_projection_reason.as_deref(),
Some("terminal_attempt_promoted_by_process_alive")
);
assert_eq!(run.phase, "executing");
assert!(!run.active_lease);
assert_eq!(run.queue_lease_state, "not_held");
Expand Down
8 changes: 8 additions & 0 deletions apps/decodex/src/orchestrator/tests/operator/status/text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1067,6 +1067,9 @@ fn operator_status_text_explains_unleased_live_running_lane() {

active_run.active_lease = false;
active_run.queue_lease_state = String::from("not_held");
active_run.attempt_status = String::from("stalled");
active_run.status_projection_reason =
Some(String::from("terminal_attempt_promoted_by_process_alive"));

let snapshot = OperatorStatusSnapshot {
project_id: String::from("pubfi"),
Expand Down Expand Up @@ -1095,5 +1098,10 @@ fn operator_status_text_explains_unleased_live_running_lane() {
assert!(rendered.contains("active_lease: no"));
assert!(rendered.contains("queue_lease_state: not_held"));
assert!(rendered.contains("queue_lease: not_held (process_alive keeps lane visible)"));
assert!(
rendered.contains(
"status_projection_reason: terminal_attempt_promoted_by_process_alive"
)
);
assert!(rendered.contains("execution_liveness: process_alive"));
}
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ fn operator_status_text_active_run() -> OperatorRunStatus {
attempt_number: 1,
status: String::from("running"),
attempt_status: String::from("running"),
status_projection_reason: None,
phase: String::from("executing"),
wait_reason: None,
current_operation: String::from(RUN_OPERATION_AGENT_RUN),
Expand Down
2 changes: 2 additions & 0 deletions apps/decodex/src/orchestrator/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1520,6 +1520,8 @@ struct OperatorRunStatus {
attempt_number: i64,
status: String,
attempt_status: String,
#[serde(skip_serializing_if = "Option::is_none")]
status_projection_reason: Option<String>,
phase: String,
wait_reason: Option<String>,
current_operation: String,
Expand Down