diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index 6c9bc04bb..41f35f69c 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -5225,6 +5225,13 @@ fn operator_run_status( lifecycle.wait_reason.clone(), protocol_activity.as_ref(), ); + let progress_diagnostic = operator_run_progress_diagnostic( + &lifecycle.phase, + &timing, + protocol_activity.as_ref(), + now_unix_epoch, + run_activity_idle_timeout(marker.as_ref()), + ); let (account, accounts) = operator_run_accounts(marker.as_ref()); let branch_name = run.branch_name().map(str::to_owned); let worktree_path = operator_run_relative_worktree_path(project, &run); @@ -5277,6 +5284,7 @@ fn operator_run_status( idle_for_seconds: timing.idle_for_seconds, protocol_idle_for_seconds: timing.protocol_idle_for_seconds, suspected_stall: lifecycle.suspected_stall, + progress_diagnostic, last_event_type: protocol_summary.last_event_type, last_event_at: protocol_summary.last_event_at, event_count: protocol_summary.event_count, @@ -5955,9 +5963,13 @@ fn operator_run_timing( run.last_event_at_unix(), marker.and_then(RunActivityMarker::last_protocol_activity_unix_epoch), ); + let run_event_progress_unix_epoch = run + .last_event_type() + .filter(|event_type| state::protocol_event_counts_as_work_progress(event_type)) + .and_then(|_| run.last_event_at_unix()); let last_progress_unix_epoch = max_optional_i64( marker.and_then(RunActivityMarker::last_progress_unix_epoch), - last_protocol_activity_unix_epoch, + run_event_progress_unix_epoch, ); let process_liveness = marker.and_then(marker_process_liveness_for_marker); @@ -6328,6 +6340,51 @@ fn suspected_operator_run_stall_threshold(idle_timeout: Duration) -> Duration { Duration::from_secs((idle_timeout.as_secs() / 2).max(1)) } +fn operator_run_progress_diagnostic( + phase: &str, + timing: &OperatorRunTiming, + protocol_activity: Option<&ProtocolActivitySummary>, + now_unix_epoch: i64, + idle_timeout: Duration, +) -> Option { + if phase != "executing" { + return None; + } + + let protocol_activity = protocol_activity?; + + if protocol_activity.waiting_reason.as_deref() != Some("model_execution") + || !protocol_activity_is_non_work_only(protocol_activity) + { + return None; + } + + let protocol_idle = timing + .last_protocol_activity_unix_epoch + .and_then(|last_protocol| observed_idle_duration(last_protocol, now_unix_epoch))?; + + if protocol_idle >= idle_timeout { + return None; + } + + let progress_is_stale = timing + .last_progress_unix_epoch + .and_then(|last_progress| observed_idle_duration(last_progress, now_unix_epoch)) + .is_none_or(|idle_for| { + idle_for >= suspected_operator_run_stall_threshold(idle_timeout) + }); + + progress_is_stale.then(|| String::from("protocol_only_activity")) +} + +fn protocol_activity_is_non_work_only(protocol_activity: &ProtocolActivitySummary) -> bool { + !protocol_activity.recent_events.is_empty() + && protocol_activity + .recent_events + .iter() + .all(|event| !state::protocol_event_counts_as_work_progress(&event.event_type)) +} + fn visible_operator_run_retry_schedule( status: &str, retry_kind: Option<&str>, @@ -7536,7 +7593,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { let control_capability = render_control_capability_summary(run.control_capability.as_ref()); output.push_str(&format!( - "- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n", + "- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n progress_diagnostic: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n", run.run_id, run.project_id, run.issue_id, @@ -7585,6 +7642,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { idle_for_seconds, protocol_idle_for_seconds, if run.suspected_stall { "yes" } else { "no" }, + run.progress_diagnostic.as_deref().unwrap_or("none"), run.process_id.map_or_else(|| String::from("none"), |value| value.to_string()), run.process_alive.map_or_else( || String::from("none"), diff --git a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs index 0e05e643c..4729ffbc3 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs @@ -1077,6 +1077,106 @@ fn operator_status_snapshot_marks_soft_stalls_before_hard_timeout() { assert!(run.suspected_stall); } +#[test] +fn operator_status_snapshot_diagnoses_protocol_only_model_execution() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, "run-1", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + state::write_run_operation_marker(&worktree_path, "run-1", 1, RUN_OPERATION_AGENT_RUN) + .expect("operation marker should write"); + + let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); + let marker_body = fs::read_to_string(&marker_path).expect("marker body should load"); + let suspected_age = (MODEL_EXECUTION_IDLE_TIMEOUT.as_secs() / 2).saturating_add(1) as i64; + let stale_progress = OffsetDateTime::now_utc().unix_timestamp() - suspected_age; + let rewritten = marker_body + .lines() + .map(|line| { + if line.starts_with("last_progress_unix_epoch=") { + format!("last_progress_unix_epoch={stale_progress}") + } else { + line.to_owned() + } + }) + .collect::>() + .join("\n") + + "\n"; + + fs::write(&marker_path, rewritten).expect("marker body should rewrite"); + + let protocol_activity = ProtocolActivitySummary { + turn_status: Some(String::from("running")), + waiting_reason: Some(String::from("model_execution")), + recent_events: vec![ + state::ProtocolActivityEventSummary { + event_type: String::from("thread/status/changed"), + category: String::from("thread"), + detail: Some(String::from("active")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("thread/goal/updated"), + category: String::from("protocol"), + detail: Some(String::from("active")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("account/rateLimits/updated"), + category: String::from("rate_limit"), + detail: Some(String::from("pro")), + }, + state::ProtocolActivityEventSummary { + event_type: String::from("account/rateLimits/updated"), + category: String::from("rate_limit"), + detail: Some(String::from("pro")), + }, + ], + ..ProtocolActivitySummary::default() + }; + + state::write_run_protocol_activity_marker( + &worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count: 2, + last_event_type: "account/rateLimits/updated", + child_agent_activity: None, + protocol_activity: Some(&protocol_activity), + }, + ) + .expect("protocol-only marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("active run should exist"); + let rendered = orchestrator::render_operator_status(&snapshot); + + assert_eq!(run.wait_reason.as_deref(), Some("model_execution")); + assert_eq!(run.progress_diagnostic.as_deref(), Some("protocol_only_activity")); + assert_eq!(run.execution_liveness, "process_alive"); + assert!(run.suspected_stall); + assert_ne!(run.last_progress_at, run.last_protocol_activity_at); + assert!(rendered.contains("progress_diagnostic: protocol_only_activity")); +} + #[test] fn operator_status_snapshot_counts_stopped_active_process_as_attention_not_running() { let (_temp_dir, config, _workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/operator/status_support.rs b/apps/decodex/src/orchestrator/tests/operator/status_support.rs index 6ffcb131e..ece2e6921 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status_support.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status_support.rs @@ -298,6 +298,7 @@ fn operator_status_text_active_run() -> orchestrator::OperatorRunStatus { idle_for_seconds: Some(1), protocol_idle_for_seconds: Some(1), suspected_stall: false, + progress_diagnostic: None, last_event_type: Some(String::from("turn/completed")), last_event_at: Some(String::from("2026-03-14 10:00:01")), event_count: 4, diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index c3a806c94..6c5729db9 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -1438,6 +1438,8 @@ struct OperatorRunStatus { idle_for_seconds: Option, protocol_idle_for_seconds: Option, suspected_stall: bool, + #[serde(skip_serializing_if = "Option::is_none")] + progress_diagnostic: Option, last_event_type: Option, last_event_at: Option, event_count: i64, diff --git a/apps/decodex/src/state/internal.rs b/apps/decodex/src/state/internal.rs index 95bc52c20..a0e744eb0 100644 --- a/apps/decodex/src/state/internal.rs +++ b/apps/decodex/src/state/internal.rs @@ -2706,6 +2706,27 @@ enum GuardRetention { AdoptingChild, } +pub(crate) fn protocol_event_counts_as_work_progress(event_type: &str) -> bool { + let normalized = event_type.to_ascii_lowercase(); + + if protocol_event_is_non_work_activity(&normalized) { + return false; + } + + normalized.starts_with("turn/") + || normalized.starts_with("item/") + || normalized == "thread/archive" + || normalized.contains("plan") + || normalized.contains("diff") + || normalized.contains("filechange") + || normalized.contains("patch") + || normalized.contains("command") + || normalized.contains("validation") + || normalized.contains("review") + || normalized.contains("pull_request") + || normalized == "model/response" +} + #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn write_run_activity_marker( worktree_path: &Path, @@ -2805,7 +2826,11 @@ pub(crate) fn write_run_protocol_activity_marker( marker.last_activity_unix_epoch = Some(now); marker.last_protocol_activity_unix_epoch = Some(now); - marker.last_progress_unix_epoch = Some(now); + + if protocol_event_counts_as_work_progress(activity.last_event_type) { + marker.last_progress_unix_epoch = Some(now); + } + marker.current_operation = Some(RUN_OPERATION_AGENT_RUN.to_owned()); marker.thread_id = activity.thread_id.map(str::to_owned).or(marker.thread_id); marker.turn_id = activity.turn_id.map(str::to_owned).or(marker.turn_id); @@ -3137,6 +3162,25 @@ pub(crate) fn process_start_identity(process_id: u32) -> Option { .and_then(|identity| normalized_process_start_identity(&identity)) } +fn protocol_event_is_non_work_activity(normalized_event_type: &str) -> bool { + normalized_event_type.starts_with("account/") + || normalized_event_type.starts_with("skills/") + || normalized_event_type.starts_with("thread/goal/") + || normalized_event_type.contains("ratelimit") + || normalized_event_type.contains("rate_limit") + || normalized_event_type == "thread/status/changed" + || normalized_event_type.contains("tokenusage") + || matches!( + normalized_event_type, + "deprecationnotice" + | "warning" + | "configwarning" + | "guardianwarning" + | "model/rerouted" + | "model/verification" + ) +} + fn normalize_accounts( selected: &CodexAccountActivitySummary, accounts: &[CodexAccountActivitySummary], diff --git a/apps/decodex/src/state/tests.rs b/apps/decodex/src/state/tests.rs index 679b89567..bd4dc4146 100644 --- a/apps/decodex/src/state/tests.rs +++ b/apps/decodex/src/state/tests.rs @@ -11,6 +11,7 @@ use std::{ use rusqlite::{self, Connection}; use serde_json::Value; use tempfile::TempDir; +use time::OffsetDateTime; use crate::{ execution_program::{ @@ -26,7 +27,7 @@ use crate::{ LoopGuardrailCheckpointInput, PreacquiredLeaseGuards, ProjectRegistration, ProtocolActivityMarker, ProtocolActivitySummary, RUN_ACTIVITY_MARKER_FILE, RUN_CONTROL_ACTION_COMPLETED, RUN_CONTROL_ACTION_FAILED, RUN_CONTROL_ACTION_FALLBACK, - RUN_CONTROL_ACTION_TIMED_OUT, RUN_OPERATION_REPO_GATE, ReviewHandoffMarker, + RUN_CONTROL_ACTION_TIMED_OUT, RUN_OPERATION_REPO_GATE, Result, ReviewHandoffMarker, ReviewOrchestrationMarker, ReviewPolicyCheckpointInput, RunControlActionRequest, StateStore, }, @@ -1913,6 +1914,132 @@ fn assert_run_activity_marker_round_trips_child_agent_activity_summary() { assert_eq!(marker.child_agent_activity(), Some(&summary)); } +#[test] +fn run_protocol_non_work_events_do_not_refresh_progress_marker() { + let temp_dir = TempDir::new().expect("tempdir should create"); + let stale_progress = OffsetDateTime::now_utc().unix_timestamp() - 3_600; + + fs::write( + temp_dir.path().join(RUN_ACTIVITY_MARKER_FILE), + format!( + "run_id=run-1\nattempt_number=1\nlast_activity_unix_epoch={stale_progress}\nlast_protocol_activity_unix_epoch={stale_progress}\nlast_progress_unix_epoch={stale_progress}\n" + ), + ) + .expect("initial marker should write"); + + let account_activity = ProtocolActivitySummary { + turn_status: Some(String::from("running")), + waiting_reason: Some(String::from("model_execution")), + recent_events: vec![state::ProtocolActivityEventSummary { + event_type: String::from("account/rateLimits/updated"), + category: String::from("rate_limit"), + detail: Some(String::from("pro")), + }], + ..ProtocolActivitySummary::default() + }; + + write_test_protocol_activity_marker( + temp_dir.path(), + 1, + "account/rateLimits/updated", + Some(&account_activity), + ) + .expect("account protocol activity should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.last_progress_unix_epoch(), Some(stale_progress)); + assert!( + marker + .last_protocol_activity_unix_epoch() + .is_some_and(|last_protocol| last_protocol > stale_progress) + ); + + let first_protocol_activity = marker + .last_protocol_activity_unix_epoch() + .expect("account protocol activity should update protocol time"); + + write_test_protocol_activity_marker( + temp_dir.path(), + 2, + "account/rateLimits/updated", + Some(&account_activity), + ) + .expect("second account protocol activity should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.last_progress_unix_epoch(), Some(stale_progress)); + assert!( + marker + .last_protocol_activity_unix_epoch() + .is_some_and(|last_protocol| last_protocol >= first_protocol_activity) + ); + + let goal_activity = ProtocolActivitySummary { + turn_status: Some(String::from("running")), + waiting_reason: Some(String::from("model_execution")), + recent_events: vec![state::ProtocolActivityEventSummary { + event_type: String::from("thread/goal/updated"), + category: String::from("protocol"), + detail: Some(String::from("active")), + }], + ..ProtocolActivitySummary::default() + }; + + write_test_protocol_activity_marker( + temp_dir.path(), + 3, + "thread/goal/updated", + Some(&goal_activity), + ) + .expect("goal status protocol activity should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert_eq!(marker.last_progress_unix_epoch(), Some(stale_progress)); + + write_test_protocol_activity_marker(temp_dir.path(), 4, "item/fileChange/patchUpdated", None) + .expect("work protocol activity should write"); + + let marker = state::read_run_activity_marker_snapshot(temp_dir.path()) + .expect("marker snapshot should load") + .expect("marker snapshot should exist"); + + assert!( + marker + .last_progress_unix_epoch() + .is_some_and(|last_progress| last_progress > stale_progress) + ); +} + +fn write_test_protocol_activity_marker( + worktree_path: &Path, + event_count: i64, + last_event_type: &str, + protocol_activity: Option<&ProtocolActivitySummary>, +) -> Result<()> { + state::write_run_protocol_activity_marker( + worktree_path, + &ProtocolActivityMarker { + run_id: "run-1", + attempt_number: 1, + thread_id: Some("thread-1"), + turn_id: Some("turn-1"), + event_count, + last_event_type, + child_agent_activity: None, + protocol_activity, + }, + ) +} + fn assert_run_activity_marker_round_trips_account_summary() { let temp_dir = TempDir::new().expect("tempdir should create"); let summary = sample_codex_account_activity_summary(); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 50c27d0db..6cce49c48 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -363,7 +363,12 @@ Worktree visibility follows the owning dashboard section: stay in the `Accounts` table; connector rate-limit backoff is surfaced as project and snapshot health, not repeated in each lane debug row. These high-frequency details remain local/operator-only and are not written to Linear except through - existing lifecycle summaries. + existing lifecycle summaries. `last_protocol_activity_at` may move for any incoming + protocol event, but `last_progress_at` moves only for meaningful work events. If a + running lane remains in model execution with fresh account, rate-limit, phase-goal, + or passive status traffic but stale work progress, status JSON exposes + `progress_diagnostic = "protocol_only_activity"` so operators can separate + process-alive and protocol-active from work-progressing. - Status JSON and the dashboard share a `loop_status` object when a row can be tied to a runtime run/attempt. It carries `review_level`, `autonomy`, concise `summary` and `next_action`, plus optional `review`, `architecture_recovery`, `boundary`, and diff --git a/docs/runbook/self-dogfood-pilot.md b/docs/runbook/self-dogfood-pilot.md index a21530d27..05eee4884 100644 --- a/docs/runbook/self-dogfood-pilot.md +++ b/docs/runbook/self-dogfood-pilot.md @@ -760,11 +760,13 @@ The operator snapshot also exposes coarse liveness semantics so you do not have The snapshot also adds fields that make running lanes easier to interpret: - `current_operation`: one of `idle`, `agent_run`, `repo_gate`, `review_writeback`, `waiting_external`, or `reconciliation` -- `last_progress_at`: the latest time Decodex recorded meaningful forward progress for the current lane +- `last_protocol_activity_at`: the latest incoming app-server protocol event, including account, rate-limit, passive status, and other non-work traffic +- `last_progress_at`: the latest time Decodex recorded meaningful forward progress for the current lane; account, rate-limit, phase-goal, passive status, warning, model-routing, token-usage, and heartbeat-like events do not refresh it - `suspected_stall = true`: a soft warning that progress has been quiet for a large fraction of the idle budget, before the lane crosses the hard `stalled` threshold +- `progress_diagnostic = protocol_only_activity`: the lane is still process/protocol-active in model execution, but recent protocol events are only non-work traffic and meaningful progress is stale or missing - `child_agent_activity`: when present, a shared dashboard and `status` breakdown of dynamic child-thread buckets, context pressure, largest tool output, and repeated large-output warnings -When present, compare `current_operation`, `last_progress_at`, `last_run_activity_at`, `last_protocol_activity_at`, `idle_for_seconds`, and `child_agent_activity.current_bucket` before assuming a lane is stuck. Quiet work with fresh child activity is different from a lane that is still alive but already drifting toward a stall. +When present, compare `current_operation`, `last_progress_at`, `last_run_activity_at`, `last_protocol_activity_at`, `progress_diagnostic`, `idle_for_seconds`, and `child_agent_activity.current_bucket` before assuming a lane is stuck. Quiet work with fresh child activity is different from a lane that is still alive but already drifting toward a stall; fresh account, phase-goal, or rate-limit events without fresh `last_progress_at` should be treated as protocol liveness, not proof of forward work. For the running-lane fields: diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index e52de79e7..92174b824 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -701,7 +701,7 @@ After a process restart, recent-run history, active lease ownership, retained po - If stalled reconciliation finds tracked changes in the retained worktree, it must classify the lane as retained partial progress directly. This path must write a human-required `needs_attention` ledger record with `error_class = "partial_progress_retained"` and `terminal_path = "retained_partial_progress"` instead of first routing the lane through `stalled_run_detected` or `terminal_failure`. - If stalled reconciliation finds no tracked changes in the retained worktree, it must converge the issue through the existing human-required failure path with `error_class = "stalled_run_detected"` instead of silently retrying in this phase. - If the supervised child already exited before the next control-plane tick, stalled reconciliation must still inspect the just-finished lane using recorded protocol activity and retained worktree state rather than skipping directly to generic failure handling. -- Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. +- Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, optional progress diagnostics, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. `last_progress_at` is meaningful-work progress only: tool calls, file or diff changes, plan/model output, repo validation, PR/review/terminal lifecycle, or other explicit work events may refresh it, but account, rate-limit, phase-goal, passive status, warning, token-usage, heartbeat, or similar non-work protocol traffic must only refresh protocol liveness. When a lane remains in `model_execution` with fresh protocol activity but stale or missing work progress and the recent protocol events are only non-work traffic, status should expose `progress_diagnostic = "protocol_only_activity"` while preserving process and protocol liveness separately. - Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it. - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. - If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. @@ -723,7 +723,7 @@ After a process restart, recent-run history, active lease ownership, retained po this is a last-resort operator audit path, not an automatic rebind or cleanup signal. - During an active run, operator snapshots must expose `thread_id` as soon as the Codex thread exists, plus monotonically advancing `event_count`, `last_event_type`, and `last_event_at` once protocol events are recorded. These fields may be hydrated either from the current process journal or from the active lane's `.decodex-run-activity` marker when `status` is running in a separate process. - `thread_id = null` is expected only before the worker creates the Codex thread for the current run. `event_count = 0`, `last_event_type = null`, and `last_event_at = null` are expected only before the first protocol event for that same run. After the thread exists and protocol activity has started, those empty values indicate missing hydration rather than normal progress. -- Operator snapshots may expose an additive `protocol_activity` object derived from app-server structured messages for the current run. The object stays local/operator-only and should summarize turn status, waiting reason, rate-limit status, and a compact recent event list for high-value app-server activity such as `turn/started`, `turn/completed`, plan updates, diff updates, item start/completion, command output deltas, server request responses, account updates, and rate-limit updates. Missing `protocol_activity` means no structured summary was captured yet; consumers must continue to rely on the older `event_count`, `last_event_type`, `last_event_at`, thread fields, and `child_agent_activity` fields when it is absent. +- Operator snapshots may expose an additive `protocol_activity` object derived from app-server structured messages for the current run. The object stays local/operator-only and should summarize turn status, waiting reason, rate-limit status, and a compact recent event list for high-value app-server activity such as `turn/started`, `turn/completed`, plan updates, diff updates, item start/completion, command output deltas, server request responses, account updates, and rate-limit updates. Missing `protocol_activity` means no structured summary was captured yet; consumers must continue to rely on the older `event_count`, `last_event_type`, `last_event_at`, thread fields, and `child_agent_activity` fields when it is absent. Presence in `protocol_activity` is not by itself meaningful progress; non-work account, rate-limit, phase-goal, passive status, warning, model-routing, and token-usage events must remain distinguishable from work-progress events through `last_progress_at` and `progress_diagnostic`. - The operator snapshot transport must stay local/operator-only. `decodex serve` exposes the human-facing operator console from the canonical HTTP `GET /` and `GET /dashboard` routes, serves only the necessary dashboard assets, `GET /livez` liveness probe, and local account-control API over HTTP, and delivers published snapshots, active-run activity, and dashboard control acknowledgements through the local `GET /dashboard/control` WebSocket upgrade. - `GET /livez` is only a process- and listener-level liveness probe. It must not claim control-plane tick freshness or forward progress by itself. - The dashboard must not depend on a separate HTTP snapshot or readiness endpoint; snapshot freshness belongs to the WebSocket-delivered snapshot payload and the browser connection state.