From d571292e461e645725a864cc4e931ec86f308b36 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Fri, 12 Jun 2026 09:23:43 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Keep terminal live status accurate","authority":"manual"} --- apps/decodex/src/orchestrator/status.rs | 44 ++++++++++-- .../tests/operator/status/http.rs | 6 +- .../tests/operator/status/running_lanes.rs | 72 +++++++++++++++++++ docs/spec/runtime.md | 10 +-- 4 files changed, 119 insertions(+), 13 deletions(-) diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index f1c1d526b..e0cc3ee72 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -6197,9 +6197,15 @@ fn operator_run_visible_status( { return String::from("running"); } + if attempt_status == "succeeded" + && operator_marker_operation_allows_terminal_status_promotion(marker_current_operation) + && operator_run_has_live_process_or_thread_evidence(app_server_state, timing) + { + return String::from("running"); + } if matches!(attempt_status, "failed" | "interrupted" | "stalled") && operator_marker_operation_allows_terminal_status_promotion(marker_current_operation) - && operator_run_has_live_execution_evidence(app_server_state, timing) + && operator_run_has_live_execution_evidence(app_server_state, protocol_summary, timing) { return String::from("running"); } @@ -6207,6 +6213,15 @@ fn operator_run_visible_status( attempt_status.to_owned() } +fn operator_run_has_live_process_or_thread_evidence( + app_server_state: &OperatorRunAppServerState, + timing: &OperatorRunTiming, +) -> bool { + timing.process_alive == Some(true) + || matches!(app_server_state.thread_status.as_deref(), Some("active")) + || !app_server_state.thread_active_flags.is_empty() +} + fn operator_marker_operation_allows_terminal_status_promotion( marker_current_operation: Option<&str>, ) -> bool { @@ -6215,17 +6230,36 @@ fn operator_marker_operation_allows_terminal_status_promotion( fn operator_run_has_live_execution_evidence( app_server_state: &OperatorRunAppServerState, + protocol_summary: &OperatorRunProtocolSummary, timing: &OperatorRunTiming, ) -> bool { - timing.process_alive == Some(true) - || matches!(app_server_state.thread_status.as_deref(), Some("active")) - || !app_server_state.thread_active_flags.is_empty() - || timing.protocol_idle_for_seconds.is_some_and(|idle_for| { + operator_run_has_live_process_or_thread_evidence(app_server_state, timing) + || operator_run_has_recent_protocol_execution_evidence(protocol_summary, timing) +} + +fn operator_run_has_recent_protocol_execution_evidence( + protocol_summary: &OperatorRunProtocolSummary, + timing: &OperatorRunTiming, +) -> bool { + operator_protocol_event_counts_as_live_execution(protocol_summary.last_event_type.as_deref()) + && timing.protocol_idle_for_seconds.is_some_and(|idle_for| { u64::try_from(idle_for) .is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) }) } +fn operator_protocol_event_counts_as_live_execution(event_type: Option<&str>) -> bool { + let Some(event_type) = event_type else { + return false; + }; + + state::protocol_event_counts_as_work_progress(event_type) + && !matches!( + event_type.to_ascii_lowercase().as_str(), + "thread/archive" | "turn/completed" + ) +} + fn operator_run_has_app_server_execution_evidence( app_server_state: &OperatorRunAppServerState, protocol_summary: &OperatorRunProtocolSummary, diff --git a/apps/decodex/src/orchestrator/tests/operator/status/http.rs b/apps/decodex/src/orchestrator/tests/operator/status/http.rs index 34e3836a9..2b7e8a108 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/http.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/http.rs @@ -2256,7 +2256,7 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_active_lease_missing() #[cfg(unix)] #[test] -fn operator_lane_interrupt_api_force_hard_fallbacks_after_lane_not_active_with_live_process() { +fn operator_lane_interrupt_api_force_hard_fallbacks_after_succeeded_status_with_live_process() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let registration = ProjectRegistration::from_config( @@ -2323,8 +2323,8 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_lane_not_active_with_l assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "{response}"); assert_eq!(data["classification"], "hard_interrupt_fallback"); - assert_eq!(data["softInterrupt"]["status"], "unavailable"); - assert_eq!(data["softInterrupt"]["errorClass"], "lane_not_active"); + assert_eq!(data["softInterrupt"]["status"], "rejected"); + assert_eq!(data["softInterrupt"]["errorClass"], "active_lease_missing"); assert_eq!(data["hardInterrupt"]["classification"], "hard_interrupt_fallback"); assert_eq!(data["hardInterrupt"]["status"], "sent"); assert_eq!( diff --git a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs index 4729ffbc3..3cfaa18eb 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs @@ -1513,6 +1513,78 @@ fn operator_status_snapshot_keeps_terminal_status_live_process_in_running_lanes( assert_eq!(project.retained_worktree_count, 0); } +#[test] +fn operator_status_snapshot_excludes_terminal_thread_archive_from_running_lanes() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "failed") + .expect("run attempt should record"); + state_store + .append_event("run-1", 1, "thread/archive", "{}") + .expect("thread archive event should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let project = snapshot.projects.first().expect("project summary should exist"); + + assert!( + snapshot.active_runs.is_empty(), + "terminal archive-only protocol events must not present as active execution" + ); + assert_eq!(project.active_run_count, 0); + assert_eq!(project.running_lane_count, 0); + assert!( + snapshot.recent_runs.iter().all(|run| run.run_id != "run-1"), + "archive-only terminal attempts do not need to remain operator-visible" + ); +} + +#[test] +fn operator_status_snapshot_keeps_succeeded_status_live_process_in_running_lanes() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("Todo", &[]); + let worktree_path = config.worktree_root().join("PUB-101"); + + state_store + .record_run_attempt("run-1", &issue.id, 1, "succeeded") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + fs::create_dir_all(&worktree_path).expect("worktree path should exist"); + state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, process::id()) + .expect("live process marker should write"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.active_runs.first().expect("live succeeded run should remain visible"); + let project = snapshot.projects.first().expect("project summary should exist"); + + assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(run.run_id, "run-1"); + assert_eq!(run.status, "running"); + assert_eq!(run.attempt_status, "succeeded"); + assert_eq!(run.phase, "executing"); + assert!(!run.active_lease); + assert_eq!(run.queue_lease_state, "not_held"); + assert_eq!(run.execution_liveness, "process_alive"); + assert_eq!(run.process_alive, Some(true)); + assert_eq!(run.process_liveness_reason.as_deref(), Some("process_alive")); + assert_eq!(project.active_run_count, 1); + assert_eq!(project.running_lane_count, 1); + assert_eq!(project.retained_worktree_count, 0); +} + #[test] fn operator_status_projects_terminal_finalized_run_as_pending_not_active() { let (_temp_dir, config, _workflow) = temp_project_layout(); diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 0ec191b2b..1bbce2b3b 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -615,12 +615,12 @@ The runtime database stores at least: - tracker and PR cache rows needed to survive connector outages - typed connector health and external API backoff -For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees//.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `active_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md). +For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees//.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `active_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. If a raw attempt is already terminal but the matching marker still proves live process, active thread, or active work-protocol execution, operator-facing status must also keep the lane visible as `running` while preserving the raw terminal value in `attempt_status`; terminal maintenance events such as `thread/archive` and completed-turn bookkeeping are not active execution evidence. Only terminal-finalize writeback projections may hide a live marker from active execution. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md). If a persisted attempt has a terminal-looking status such as `failed`, `interrupted`, -or `stalled` while current marker or protocol evidence still identifies the same -`run_id` and `attempt_number` as live, operator status must keep the lane visible with -the process/protocol liveness details instead of hiding it as only terminal history or -cleanup work. +or `stalled` while current marker, active thread, or active work-protocol evidence +still identifies the same `run_id` and `attempt_number` as live, operator status must +keep the lane visible with the process/protocol liveness details instead of hiding it +as only terminal history or cleanup work. Post-review ownership is stored in the runtime database. Retained handoff rows record the authoritative PR URL, branch lineage, validated PR head OID, run id, and attempt number that completed the `In Review` handoff. Retained orchestration rows record the current post-review phase for that exact handoff identity. If the matching database row is missing, post-review ownership must block as unresolved instead of rebinding from branch-name, current-head, Linear comments, or other heuristics. An explicit operator manual takeover command may adopt a human-owned PR into this same retained database shape only after validating the active service label, managed clean worktree, PR repository, default-branch target, exact branch/head match, and green landable PR gates. If a retained review marker exists but a stored handoff or orchestration head no longer matches a clean retained worktree and matching PR head, operator status must keep the marker PR URL visible when known and recovery diagnosis must report the concrete mismatched field before any explicit rebind refresh. When retained PR readback degrades but the handoff identity is still safe to preserve, operator-local status may expose a typed `readback_root_cause` diagnostic such as missing GitHub CLI, missing GitHub token, GitHub auth failure, API/read failure, parse/shape failure, or lineage validation failure while keeping public-safe warning reasons such as `pull_request_state_read_failed` stable. The only source-tree runtime artifacts that clean-source checks may ignore are the untracked top-level `.decodex-run-activity` heartbeat marker and `.decodex-run-control/` local control-channel directory. Durable review handoff, orchestration, review-policy checkpoints, retry, phase timing, and retained PR state belong in the Decodex runtime database, not in root-level or worktree-local review marker files. If the heartbeat marker carries similarly named fields for compatibility or operator diagnostics, those breadcrumb values cannot override runtime-store rows.