Harden Decodex stall progress accounting

apps/decodex/src/orchestrator/status.rs

@@ -5225,6 +5225,13 @@ fn operator_run_status(
 		lifecycle.wait_reason.clone(),
 		protocol_activity.as_ref(),
 	);
+	let progress_diagnostic = operator_run_progress_diagnostic(
+		&lifecycle.phase,
+		&timing,
+		protocol_activity.as_ref(),
+		now_unix_epoch,
+		run_activity_idle_timeout(marker.as_ref()),
+	);
 	let (account, accounts) = operator_run_accounts(marker.as_ref());
 	let branch_name = run.branch_name().map(str::to_owned);
 	let worktree_path = operator_run_relative_worktree_path(project, &run);
@@ -5277,6 +5284,7 @@ fn operator_run_status(
 		idle_for_seconds: timing.idle_for_seconds,
 		protocol_idle_for_seconds: timing.protocol_idle_for_seconds,
 		suspected_stall: lifecycle.suspected_stall,
+		progress_diagnostic,
 		last_event_type: protocol_summary.last_event_type,
 		last_event_at: protocol_summary.last_event_at,
 		event_count: protocol_summary.event_count,
@@ -5955,9 +5963,13 @@ fn operator_run_timing(
 		run.last_event_at_unix(),
 		marker.and_then(RunActivityMarker::last_protocol_activity_unix_epoch),
 	);
+	let run_event_progress_unix_epoch = run
+		.last_event_type()
+		.filter(|event_type| state::protocol_event_counts_as_work_progress(event_type))
+		.and_then(|_| run.last_event_at_unix());
 	let last_progress_unix_epoch = max_optional_i64(
 		marker.and_then(RunActivityMarker::last_progress_unix_epoch),
-		last_protocol_activity_unix_epoch,
+		run_event_progress_unix_epoch,
 	);
 	let process_liveness = marker.and_then(marker_process_liveness_for_marker);
 
@@ -6328,6 +6340,51 @@ fn suspected_operator_run_stall_threshold(idle_timeout: Duration) -> Duration {
 	Duration::from_secs((idle_timeout.as_secs() / 2).max(1))
 }
 
+fn operator_run_progress_diagnostic(
+	phase: &str,
+	timing: &OperatorRunTiming,
+	protocol_activity: Option<&ProtocolActivitySummary>,
+	now_unix_epoch: i64,
+	idle_timeout: Duration,
+) -> Option<String> {
+	if phase != "executing" {
+		return None;
+	}
+
+	let protocol_activity = protocol_activity?;
+
+	if protocol_activity.waiting_reason.as_deref() != Some("model_execution")
+		|| !protocol_activity_is_non_work_only(protocol_activity)
+	{
+		return None;
+	}
+
+	let protocol_idle = timing
+		.last_protocol_activity_unix_epoch
+		.and_then(|last_protocol| observed_idle_duration(last_protocol, now_unix_epoch))?;
+
+	if protocol_idle >= idle_timeout {
+		return None;
+	}
+
+	let progress_is_stale = timing
+		.last_progress_unix_epoch
+		.and_then(|last_progress| observed_idle_duration(last_progress, now_unix_epoch))
+		.is_none_or(|idle_for| {
+			idle_for >= suspected_operator_run_stall_threshold(idle_timeout)
+		});
+
+	progress_is_stale.then(|| String::from("protocol_only_activity"))
+}
+
+fn protocol_activity_is_non_work_only(protocol_activity: &ProtocolActivitySummary) -> bool {
+	!protocol_activity.recent_events.is_empty()
+		&& protocol_activity
+			.recent_events
+			.iter()
+			.all(|event| !state::protocol_event_counts_as_work_progress(&event.event_type))
+}
+
 fn visible_operator_run_retry_schedule(
 	status: &str,
 	retry_kind: Option<&str>,
@@ -7536,7 +7593,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) {
 	let control_capability = render_control_capability_summary(run.control_capability.as_ref());
 
 	output.push_str(&format!(
-		"- run_id: {}\n  project_id: {}\n  issue_id: {}\n  issue_identifier: {}\n  title: {}\n  attempt: {}\n  status: {}\n  attempt_status: {}\n  phase: {}\n  wait_reason: {}\n  current_operation: {}\n  active_lease: {}\n  queue_lease_state: {}\n  queue_lease: {}\n  execution_liveness: {}\n  freshness_at: {}\n  freshness_source: {}\n  timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n  account: {}\n  accounts: {}\n  child_agent_activity: {}\n  protocol_activity: {}\n  context_pressure: {}\n  private_evidence: {}\n  loop_status: {}\n  loop_review: {}\n  loop_architecture_recovery: {}\n  loop_boundary: {}\n  control_capability: {}\n  thread_id: {}\n  turn_id: {}\n  thread_status: {}\n  thread_active_flags: {}\n  interactive_requested: {}\n  continuation_pending: {}\n  branch: {}\n  worktree_path: {}\n  updated_at: {}\n  last_run_activity_at: {}\n  last_protocol_activity_at: {}\n  last_progress_at: {}\n  idle_for_seconds: {}\n  protocol_idle_for_seconds: {}\n  suspected_stall: {}\n  process_id: {}\n  process_alive: {}\n  process_liveness_reason: {}\n  retry_kind: {}\n  next_retry_at: {}\n  effective_model: {}\n  effective_model_provider: {}\n  effective_cwd: {}\n  effective_approval_policy: {}\n  effective_approvals_reviewer: {}\n  effective_sandbox_mode: {}\n  protocol_event: {}\n  event_count: {}\n",
+		"- run_id: {}\n  project_id: {}\n  issue_id: {}\n  issue_identifier: {}\n  title: {}\n  attempt: {}\n  status: {}\n  attempt_status: {}\n  phase: {}\n  wait_reason: {}\n  current_operation: {}\n  active_lease: {}\n  queue_lease_state: {}\n  queue_lease: {}\n  execution_liveness: {}\n  freshness_at: {}\n  freshness_source: {}\n  timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n  account: {}\n  accounts: {}\n  child_agent_activity: {}\n  protocol_activity: {}\n  context_pressure: {}\n  private_evidence: {}\n  loop_status: {}\n  loop_review: {}\n  loop_architecture_recovery: {}\n  loop_boundary: {}\n  control_capability: {}\n  thread_id: {}\n  turn_id: {}\n  thread_status: {}\n  thread_active_flags: {}\n  interactive_requested: {}\n  continuation_pending: {}\n  branch: {}\n  worktree_path: {}\n  updated_at: {}\n  last_run_activity_at: {}\n  last_protocol_activity_at: {}\n  last_progress_at: {}\n  idle_for_seconds: {}\n  protocol_idle_for_seconds: {}\n  suspected_stall: {}\n  progress_diagnostic: {}\n  process_id: {}\n  process_alive: {}\n  process_liveness_reason: {}\n  retry_kind: {}\n  next_retry_at: {}\n  effective_model: {}\n  effective_model_provider: {}\n  effective_cwd: {}\n  effective_approval_policy: {}\n  effective_approvals_reviewer: {}\n  effective_sandbox_mode: {}\n  protocol_event: {}\n  event_count: {}\n",
 		run.run_id,
 		run.project_id,
 		run.issue_id,
@@ -7585,6 +7642,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) {
 		idle_for_seconds,
 		protocol_idle_for_seconds,
 		if run.suspected_stall { "yes" } else { "no" },
+		run.progress_diagnostic.as_deref().unwrap_or("none"),
 		run.process_id.map_or_else(|| String::from("none"), |value| value.to_string()),
 		run.process_alive.map_or_else(
 			|| String::from("none"),

apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs

@@ -1077,6 +1077,106 @@ fn operator_status_snapshot_marks_soft_stalls_before_hard_timeout() {
 	assert!(run.suspected_stall);
 }
 
+#[test]
+fn operator_status_snapshot_diagnoses_protocol_only_model_execution() {
+	let (_temp_dir, config, _workflow) = temp_project_layout();
+	let state_store = StateStore::open_in_memory().expect("state store should open");
+	let issue = sample_issue("Todo", &[]);
+	let worktree_path = config.worktree_root().join("PUB-101");
+
+	state_store
+		.record_run_attempt("run-1", &issue.id, 1, "running")
+		.expect("run attempt should record");
+	state_store
+		.upsert_lease("pubfi", &issue.id, "run-1", "In Progress")
+		.expect("lease should record");
+	state_store
+		.upsert_worktree(
+			"pubfi",
+			&issue.id,
+			"x/pubfi-pub-101",
+			&worktree_path.display().to_string(),
+		)
+		.expect("worktree should record");
+
+	state::write_run_operation_marker(&worktree_path, "run-1", 1, RUN_OPERATION_AGENT_RUN)
+		.expect("operation marker should write");
+
+	let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE);
+	let marker_body = fs::read_to_string(&marker_path).expect("marker body should load");
+	let suspected_age = (MODEL_EXECUTION_IDLE_TIMEOUT.as_secs() / 2).saturating_add(1) as i64;
+	let stale_progress = OffsetDateTime::now_utc().unix_timestamp() - suspected_age;
+	let rewritten = marker_body
+		.lines()
+		.map(|line| {
+			if line.starts_with("last_progress_unix_epoch=") {
+				format!("last_progress_unix_epoch={stale_progress}")
+			} else {
+				line.to_owned()
+			}
+		})
+		.collect::<Vec<_>>()
+		.join("\n")
+		+ "\n";
+
+	fs::write(&marker_path, rewritten).expect("marker body should rewrite");
+
+	let protocol_activity = ProtocolActivitySummary {
+		turn_status: Some(String::from("running")),
+		waiting_reason: Some(String::from("model_execution")),
+		recent_events: vec![
+			state::ProtocolActivityEventSummary {
+				event_type: String::from("thread/status/changed"),
+				category: String::from("thread"),
+				detail: Some(String::from("active")),
+			},
+			state::ProtocolActivityEventSummary {
+				event_type: String::from("thread/goal/updated"),
+				category: String::from("protocol"),
+				detail: Some(String::from("active")),
+			},
+			state::ProtocolActivityEventSummary {
+				event_type: String::from("account/rateLimits/updated"),
+				category: String::from("rate_limit"),
+				detail: Some(String::from("pro")),
+			},
+			state::ProtocolActivityEventSummary {
+				event_type: String::from("account/rateLimits/updated"),
+				category: String::from("rate_limit"),
+				detail: Some(String::from("pro")),
+			},
+		],
+		..ProtocolActivitySummary::default()
+	};
+
+	state::write_run_protocol_activity_marker(
+		&worktree_path,
+		&ProtocolActivityMarker {
+			run_id: "run-1",
+			attempt_number: 1,
+			thread_id: Some("thread-1"),
+			turn_id: Some("turn-1"),
+			event_count: 2,
+			last_event_type: "account/rateLimits/updated",
+			child_agent_activity: None,
+			protocol_activity: Some(&protocol_activity),
+		},
+	)
+	.expect("protocol-only marker should write");
+
+	let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10)
+		.expect("snapshot should build");
+	let run = snapshot.active_runs.first().expect("active run should exist");
+	let rendered = orchestrator::render_operator_status(&snapshot);
+
+	assert_eq!(run.wait_reason.as_deref(), Some("model_execution"));
+	assert_eq!(run.progress_diagnostic.as_deref(), Some("protocol_only_activity"));
+	assert_eq!(run.execution_liveness, "process_alive");
+	assert!(run.suspected_stall);
+	assert_ne!(run.last_progress_at, run.last_protocol_activity_at);
+	assert!(rendered.contains("progress_diagnostic: protocol_only_activity"));
+}
+
 #[test]
 fn operator_status_snapshot_counts_stopped_active_process_as_attention_not_running() {
 	let (_temp_dir, config, _workflow) = temp_project_layout();

apps/decodex/src/orchestrator/tests/operator/status_support.rs

@@ -298,6 +298,7 @@ fn operator_status_text_active_run() -> orchestrator::OperatorRunStatus {
 		idle_for_seconds: Some(1),
 		protocol_idle_for_seconds: Some(1),
 		suspected_stall: false,
+		progress_diagnostic: None,
 		last_event_type: Some(String::from("turn/completed")),
 		last_event_at: Some(String::from("2026-03-14 10:00:01")),
 		event_count: 4,

apps/decodex/src/orchestrator/types.rs

@@ -1438,6 +1438,8 @@ struct OperatorRunStatus {
 	idle_for_seconds: Option<i64>,
 	protocol_idle_for_seconds: Option<i64>,
 	suspected_stall: bool,
+	#[serde(skip_serializing_if = "Option::is_none")]
+	progress_diagnostic: Option<String>,
 	last_event_type: Option<String>,
 	last_event_at: Option<String>,
 	event_count: i64,

apps/decodex/src/state/internal.rs

@@ -2706,6 +2706,27 @@ enum GuardRetention {
 	AdoptingChild,
 }
 
+pub(crate) fn protocol_event_counts_as_work_progress(event_type: &str) -> bool {
+	let normalized = event_type.to_ascii_lowercase();
+
+	if protocol_event_is_non_work_activity(&normalized) {
+		return false;
+	}
+
+	normalized.starts_with("turn/")
+		|| normalized.starts_with("item/")
+		|| normalized == "thread/archive"
+		|| normalized.contains("plan")
+		|| normalized.contains("diff")
+		|| normalized.contains("filechange")
+		|| normalized.contains("patch")
+		|| normalized.contains("command")
+		|| normalized.contains("validation")
+		|| normalized.contains("review")
+		|| normalized.contains("pull_request")
+		|| normalized == "model/response"
+}
+
 #[cfg_attr(not(test), allow(dead_code))]
 pub(crate) fn write_run_activity_marker(
 	worktree_path: &Path,
@@ -2805,7 +2826,11 @@ pub(crate) fn write_run_protocol_activity_marker(
 
 	marker.last_activity_unix_epoch = Some(now);
 	marker.last_protocol_activity_unix_epoch = Some(now);
-	marker.last_progress_unix_epoch = Some(now);
+
+	if protocol_event_counts_as_work_progress(activity.last_event_type) {
+		marker.last_progress_unix_epoch = Some(now);
+	}
+
 	marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned());
 	marker.thread_id = activity.thread_id.map(str::to_owned).or(marker.thread_id);
 	marker.turn_id = activity.turn_id.map(str::to_owned).or(marker.turn_id);
@@ -3137,6 +3162,25 @@ pub(crate) fn process_start_identity(process_id: u32) -> Option<String> {
 		.and_then(|identity| normalized_process_start_identity(&identity))
 }
 
+fn protocol_event_is_non_work_activity(normalized_event_type: &str) -> bool {
+	normalized_event_type.starts_with("account/")
+		|| normalized_event_type.starts_with("skills/")
+		|| normalized_event_type.starts_with("thread/goal/")
+		|| normalized_event_type.contains("ratelimit")
+		|| normalized_event_type.contains("rate_limit")
+		|| normalized_event_type == "thread/status/changed"
+		|| normalized_event_type.contains("tokenusage")
+		|| matches!(
+			normalized_event_type,
+			"deprecationnotice"
+				| "warning"
+				| "configwarning"
+				| "guardianwarning"
+				| "model/rerouted"
+				| "model/verification"
+		)
+}
+
 fn normalize_accounts(
 	selected: &CodexAccountActivitySummary,
 	accounts: &[CodexAccountActivitySummary],