Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions apps/decodex/src/orchestrator/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6197,16 +6197,31 @@ fn operator_run_visible_status(
{
return String::from("running");
}
if attempt_status == "succeeded"
&& operator_marker_operation_allows_terminal_status_promotion(marker_current_operation)
&& operator_run_has_live_process_or_thread_evidence(app_server_state, timing)
{
return String::from("running");
}
if matches!(attempt_status, "failed" | "interrupted" | "stalled")
&& operator_marker_operation_allows_terminal_status_promotion(marker_current_operation)
&& operator_run_has_live_execution_evidence(app_server_state, timing)
&& operator_run_has_live_execution_evidence(app_server_state, protocol_summary, timing)
{
return String::from("running");
}

attempt_status.to_owned()
}

fn operator_run_has_live_process_or_thread_evidence(
app_server_state: &OperatorRunAppServerState,
timing: &OperatorRunTiming,
) -> bool {
timing.process_alive == Some(true)
|| matches!(app_server_state.thread_status.as_deref(), Some("active"))
|| !app_server_state.thread_active_flags.is_empty()
}

fn operator_marker_operation_allows_terminal_status_promotion(
marker_current_operation: Option<&str>,
) -> bool {
Expand All @@ -6215,17 +6230,36 @@ fn operator_marker_operation_allows_terminal_status_promotion(

fn operator_run_has_live_execution_evidence(
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
timing: &OperatorRunTiming,
) -> bool {
timing.process_alive == Some(true)
|| matches!(app_server_state.thread_status.as_deref(), Some("active"))
|| !app_server_state.thread_active_flags.is_empty()
|| timing.protocol_idle_for_seconds.is_some_and(|idle_for| {
operator_run_has_live_process_or_thread_evidence(app_server_state, timing)
|| operator_run_has_recent_protocol_execution_evidence(protocol_summary, timing)
}

fn operator_run_has_recent_protocol_execution_evidence(
protocol_summary: &OperatorRunProtocolSummary,
timing: &OperatorRunTiming,
) -> bool {
operator_protocol_event_counts_as_live_execution(protocol_summary.last_event_type.as_deref())
&& timing.protocol_idle_for_seconds.is_some_and(|idle_for| {
u64::try_from(idle_for)
.is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs())
})
}

fn operator_protocol_event_counts_as_live_execution(event_type: Option<&str>) -> bool {
let Some(event_type) = event_type else {
return false;
};

state::protocol_event_counts_as_work_progress(event_type)
&& !matches!(
event_type.to_ascii_lowercase().as_str(),
"thread/archive" | "turn/completed"
)
}

fn operator_run_has_app_server_execution_evidence(
app_server_state: &OperatorRunAppServerState,
protocol_summary: &OperatorRunProtocolSummary,
Expand Down
6 changes: 3 additions & 3 deletions apps/decodex/src/orchestrator/tests/operator/status/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2256,7 +2256,7 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_active_lease_missing()

#[cfg(unix)]
#[test]
fn operator_lane_interrupt_api_force_hard_fallbacks_after_lane_not_active_with_live_process() {
fn operator_lane_interrupt_api_force_hard_fallbacks_after_succeeded_status_with_live_process() {
let (_temp_dir, config, _workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");
let registration = ProjectRegistration::from_config(
Expand Down Expand Up @@ -2323,8 +2323,8 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_lane_not_active_with_l

assert!(response.starts_with("HTTP/1.1 200 OK\r\n"), "{response}");
assert_eq!(data["classification"], "hard_interrupt_fallback");
assert_eq!(data["softInterrupt"]["status"], "unavailable");
assert_eq!(data["softInterrupt"]["errorClass"], "lane_not_active");
assert_eq!(data["softInterrupt"]["status"], "rejected");
assert_eq!(data["softInterrupt"]["errorClass"], "active_lease_missing");
assert_eq!(data["hardInterrupt"]["classification"], "hard_interrupt_fallback");
assert_eq!(data["hardInterrupt"]["status"], "sent");
assert_eq!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1513,6 +1513,78 @@ fn operator_status_snapshot_keeps_terminal_status_live_process_in_running_lanes(
assert_eq!(project.retained_worktree_count, 0);
}

#[test]
fn operator_status_snapshot_excludes_terminal_thread_archive_from_running_lanes() {
let (_temp_dir, config, _workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");
let issue = sample_issue("Todo", &[]);

state_store
.record_run_attempt("run-1", &issue.id, 1, "failed")
.expect("run attempt should record");
state_store
.append_event("run-1", 1, "thread/archive", "{}")
.expect("thread archive event should record");

let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10)
.expect("snapshot should build");
let project = snapshot.projects.first().expect("project summary should exist");

assert!(
snapshot.active_runs.is_empty(),
"terminal archive-only protocol events must not present as active execution"
);
assert_eq!(project.active_run_count, 0);
assert_eq!(project.running_lane_count, 0);
assert!(
snapshot.recent_runs.iter().all(|run| run.run_id != "run-1"),
"archive-only terminal attempts do not need to remain operator-visible"
);
}

#[test]
fn operator_status_snapshot_keeps_succeeded_status_live_process_in_running_lanes() {
let (_temp_dir, config, _workflow) = temp_project_layout();
let state_store = StateStore::open_in_memory().expect("state store should open");
let issue = sample_issue("Todo", &[]);
let worktree_path = config.worktree_root().join("PUB-101");

state_store
.record_run_attempt("run-1", &issue.id, 1, "succeeded")
.expect("run attempt should record");
state_store
.upsert_worktree(
"pubfi",
&issue.id,
"x/pubfi-pub-101",
&worktree_path.display().to_string(),
)
.expect("worktree should record");

fs::create_dir_all(&worktree_path).expect("worktree path should exist");
state::write_run_activity_marker_for_process(&worktree_path, "run-1", 1, process::id())
.expect("live process marker should write");

let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10)
.expect("snapshot should build");
let run = snapshot.active_runs.first().expect("live succeeded run should remain visible");
let project = snapshot.projects.first().expect("project summary should exist");

assert_eq!(snapshot.active_runs.len(), 1);
assert_eq!(run.run_id, "run-1");
assert_eq!(run.status, "running");
assert_eq!(run.attempt_status, "succeeded");
assert_eq!(run.phase, "executing");
assert!(!run.active_lease);
assert_eq!(run.queue_lease_state, "not_held");
assert_eq!(run.execution_liveness, "process_alive");
assert_eq!(run.process_alive, Some(true));
assert_eq!(run.process_liveness_reason.as_deref(), Some("process_alive"));
assert_eq!(project.active_run_count, 1);
assert_eq!(project.running_lane_count, 1);
assert_eq!(project.retained_worktree_count, 0);
}

#[test]
fn operator_status_projects_terminal_finalized_run_as_pending_not_active() {
let (_temp_dir, config, _workflow) = temp_project_layout();
Expand Down
10 changes: 5 additions & 5 deletions docs/spec/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -615,12 +615,12 @@ The runtime database stores at least:
- tracker and PR cache rows needed to survive connector outages
- typed connector health and external API backoff

For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees/<ISSUE>/.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `active_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md).
For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees/<ISSUE>/.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `active_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. If a raw attempt is already terminal but the matching marker still proves live process, active thread, or active work-protocol execution, operator-facing status must also keep the lane visible as `running` while preserving the raw terminal value in `attempt_status`; terminal maintenance events such as `thread/archive` and completed-turn bookkeeping are not active execution evidence. Only terminal-finalize writeback projections may hide a live marker from active execution. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md).
If a persisted attempt has a terminal-looking status such as `failed`, `interrupted`,
or `stalled` while current marker or protocol evidence still identifies the same
`run_id` and `attempt_number` as live, operator status must keep the lane visible with
the process/protocol liveness details instead of hiding it as only terminal history or
cleanup work.
or `stalled` while current marker, active thread, or active work-protocol evidence
still identifies the same `run_id` and `attempt_number` as live, operator status must
keep the lane visible with the process/protocol liveness details instead of hiding it
as only terminal history or cleanup work.
Post-review ownership is stored in the runtime database. Retained handoff rows record the authoritative PR URL, branch lineage, validated PR head OID, run id, and attempt number that completed the `In Review` handoff. Retained orchestration rows record the current post-review phase for that exact handoff identity. If the matching database row is missing, post-review ownership must block as unresolved instead of rebinding from branch-name, current-head, Linear comments, or other heuristics. An explicit operator manual takeover command may adopt a human-owned PR into this same retained database shape only after validating the active service label, managed clean worktree, PR repository, default-branch target, exact branch/head match, and green landable PR gates. If a retained review marker exists but a stored handoff or orchestration head no longer matches a clean retained worktree and matching PR head, operator status must keep the marker PR URL visible when known and recovery diagnosis must report the concrete mismatched field before any explicit rebind refresh. When retained PR readback degrades but the handoff identity is still safe to preserve, operator-local status may expose a typed `readback_root_cause` diagnostic such as missing GitHub CLI, missing GitHub token, GitHub auth failure, API/read failure, parse/shape failure, or lineage validation failure while keeping public-safe warning reasons such as `pull_request_state_read_failed` stable.
The only source-tree runtime artifacts that clean-source checks may ignore are the untracked top-level `.decodex-run-activity` heartbeat marker and `.decodex-run-control/` local control-channel directory. Durable review handoff, orchestration, review-policy checkpoints, retry, phase timing, and retained PR state belong in the Decodex runtime database, not in root-level or worktree-local review marker files. If the heartbeat marker carries similarly named fields for compatibility or operator diagnostics, those breadcrumb values cannot override runtime-store rows.

Expand Down