diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index ef1e69578..501fbd704 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -1330,6 +1330,9 @@ fn latest_active_phase_goal_recovery_candidate( | "phase_goal_transition" | "review_completion_intent" | "terminal_finalize" => return Ok(None), + AUTHORITY_DECISION_REQUEST_EVENT_TYPE => return Ok(None), + "progress_checkpoint" if progress_checkpoint_has_blockers(event.payload()) => + return Ok(None), "phase_goal_set" | "phase_goal_status" => { let Some(phase) = phase_goal_event_phase(event.payload()) else { return Ok(None); @@ -1347,6 +1350,14 @@ fn latest_active_phase_goal_recovery_candidate( Ok(None) } +fn progress_checkpoint_has_blockers(payload: &Value) -> bool { + payload.get("blockers").is_some_and(|blockers| match blockers { + Value::Array(items) => !items.is_empty(), + Value::Null => false, + _ => true, + }) +} + fn phase_goal_event_phase(payload: &Value) -> Option { payload .get("phase") diff --git a/apps/decodex/src/orchestrator/git_ops.rs b/apps/decodex/src/orchestrator/git_ops.rs index b8777db2b..c5e669691 100644 --- a/apps/decodex/src/orchestrator/git_ops.rs +++ b/apps/decodex/src/orchestrator/git_ops.rs @@ -615,14 +615,81 @@ fn ensure_repo_gate_left_no_tracked_changes(cwd: &Path, phase: &str) -> Result<( Ok(()) } +fn read_repo_gate_tracked_diff(cwd: &Path, phase: &str) -> Result { + let output = Command::new("git") + .arg("-C") + .arg(cwd) + .args(["diff", "--no-ext-diff", "--binary", "HEAD", "--"]) + .output() + .map_err(|error| { + Report::new(RepoGateFailure::new( + RepoGateFailureKind::CommandSpawnFailed, + format!( + "Failed to spawn tracked-file diff check after repo gate {phase} in `{}`: {error}", + cwd.display() + ), + )) + })?; + + if !output.status.success() { + let output_text = repo_gate_output_text(&output); + + return Err(Report::new(RepoGateFailure::new( + repo_gate_failure_kind_for_output( + RepoGateFailureKind::CleanlinessCheckFailed, + &output_text, + ), + format!( + "Failed to inspect tracked-file diff after repo gate {phase} in `{}`: {}", + cwd.display(), + output_text + ), + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn ensure_repo_gate_preserved_tracked_diff( + cwd: &Path, + phase: &str, + before_diff: &str, +) -> Result<()> { + let after_diff = read_repo_gate_tracked_diff(cwd, phase)?; + + if after_diff == before_diff { + return Ok(()); + } + + let output = run_repo_gate_cleanliness_check_with_git(std::ffi::OsStr::new("git"), cwd)?; + let output_text = repo_gate_output_text(&output); + let dirty_entries = output_text.trim(); + + Err(Report::new(RepoGateFailure::new( + RepoGateFailureKind::TrackedRewritesLeft, + format!( + "Repo gate {phase} rewrote tracked files in `{}`; commit or revert these changes before continuing:\n{}", + cwd.display(), + dirty_entries + ), + ))) +} + fn run_repo_gate_commands( canonicalize_commands: &[String], verify_commands: &[String], cwd: &Path, ) -> Result<()> { + let baseline_tracked_diff = read_repo_gate_tracked_diff(cwd, "baseline")?; + run_canonicalize_commands(canonicalize_commands, cwd)?; run_verify_commands(verify_commands, cwd)?; - ensure_repo_gate_left_no_tracked_changes(cwd, "verification")?; + + if baseline_tracked_diff.is_empty() { + ensure_repo_gate_left_no_tracked_changes(cwd, "verification")?; + } else { + ensure_repo_gate_preserved_tracked_diff(cwd, "verification", &baseline_tracked_diff)?; + } Ok(()) } diff --git a/apps/decodex/src/orchestrator/reconciliation.rs b/apps/decodex/src/orchestrator/reconciliation.rs index a5f44f5af..72985c951 100644 --- a/apps/decodex/src/orchestrator/reconciliation.rs +++ b/apps/decodex/src/orchestrator/reconciliation.rs @@ -423,10 +423,40 @@ where "Reconciling stalled run with retained partial progress." ); + let issue_run = stalled_reconciliation_issue_run(state_store, worktree_manager, action)?; + let recovered = match try_recover_stalled_retained_phase_goal( + project, + &action.workflow, + state_store, + &action.issue, + &issue_run, + ) { + Ok(recovered) => recovered, + Err(error) if run_failure_requires_terminal_attention(&error) => { + state_store.update_run_status(action.run_attempt.run_id(), "stalled")?; + state_store.clear_lease(&action.issue.id)?; + + handle_failure( + tracker, + project, + &action.workflow, + state_store, + &issue_run, + &error, + )?; + + return Ok(()); + }, + Err(error) => return Err(error), + }; + + if recovered { + return Ok(()); + } + state_store.update_run_status(action.run_attempt.run_id(), "stalled")?; state_store.clear_lease(&action.issue.id)?; - let issue_run = stalled_reconciliation_issue_run(state_store, worktree_manager, action)?; let worktree_path = relative_worktree_path(project, &issue_run.worktree); write_reconciliation_operation_marker_best_effort( @@ -452,6 +482,71 @@ where Ok(()) } +fn try_recover_stalled_retained_phase_goal( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue: &TrackerIssue, + issue_run: &IssueRunPlan, +) -> Result { + write_reconciliation_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_RECONCILIATION, + ); + + let recovery = recover_active_phase_goal_continuation( + project, + workflow, + state_store, + issue_run, + "stalled_run_detected", + )?; + let Some(recovery) = recovery else { + return Ok(false); + }; + + state_store.update_run_status(&issue_run.run_id, CONTINUATION_PENDING_RUN_STATUS)?; + state_store.clear_lease(&issue.id)?; + + write_stalled_phase_goal_continuation_retry_marker(state_store, workflow, issue_run)?; + + tracing::warn!( + project_id = project.service_id(), + issue_id = issue.id, + issue = issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + source_phase = recovery.source_phase.as_str(), + next_phase = recovery.next_phase.as_str(), + "Recovered stalled retained phase goal; scheduling continuation instead of manual attention." + ); + + Ok(true) +} + +fn write_stalled_phase_goal_continuation_retry_marker( + state_store: &StateStore, + workflow: &WorkflowDocument, + issue_run: &IssueRunPlan, +) -> Result<()> { + let attempt = u32::try_from(issue_run.attempt_number).unwrap_or(u32::MAX).max(1); + let delay = retry_delay(RetryKind::Continuation, attempt, workflow); + let retry_ready_at_unix_epoch = OffsetDateTime::now_utc().unix_timestamp().saturating_add( + i64::try_from((delay.as_millis().saturating_add(999)) / 1_000).unwrap_or(i64::MAX), + ); + + write_retry_schedule_for_run( + state_store, + &issue_run.issue.id, + &issue_run.run_id, + issue_run.attempt_number, + RetryKind::Continuation, + retry_ready_at_unix_epoch, + ) +} + fn stalled_reconciliation_issue_run( state_store: &StateStore, worktree_manager: &WorktreeManager, @@ -590,6 +685,9 @@ fn stalled_idle_duration( if !matches!(run_attempt.status(), "starting" | "running") { return Ok(None); } + if stalled_reconciliation_deferred_by_marker(run_attempt, worktree_mapping)? { + return Ok(None); + } let Some(last_activity) = last_observed_run_activity_unix_epoch(state_store, run_attempt, worktree_mapping)? @@ -660,6 +758,10 @@ fn stalled_protocol_idle_duration( worktree_mapping: Option<&WorktreeMapping>, now_unix_epoch: i64, ) -> Result> { + if stalled_reconciliation_deferred_by_marker(run_attempt, worktree_mapping)? { + return Ok(None); + } + let Some(last_activity) = last_observed_protocol_activity_unix_epoch(state_store, run_attempt, worktree_mapping)? else { @@ -705,3 +807,40 @@ fn observed_idle_duration(last_activity_unix_epoch: i64, now_unix_epoch: i64) -> .and_then(|idle_seconds| u64::try_from(idle_seconds).ok()) .map(Duration::from_secs) } + +fn stalled_reconciliation_deferred_by_marker( + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, +) -> Result { + let Some(marker) = current_run_activity_marker(run_attempt, worktree_mapping)? else { + return Ok(false); + }; + + if marker.retry_kind().is_some() { + return Ok(true); + } + + Ok(marker.current_operation() == Some(RUN_OPERATION_REPO_GATE) + && marker_process_is_alive(&marker)) +} + +fn current_run_activity_marker( + run_attempt: &RunAttempt, + worktree_mapping: Option<&WorktreeMapping>, +) -> Result> { + let Some(worktree_mapping) = worktree_mapping else { + return Ok(None); + }; + let Some(marker) = state::read_run_activity_marker_snapshot(worktree_mapping.worktree_path())? + else { + return Ok(None); + }; + + if marker.run_id() == run_attempt.run_id() + && marker.attempt_number() == run_attempt.attempt_number() + { + return Ok(Some(marker)); + } + + Ok(None) +} diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs index 8c6578f0b..5b7e1df02 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -1,3 +1,15 @@ +use serde::Serialize; + +struct StalledPhaseGoalOutcome { + run_status: String, + retry_kind: Option, + retry_ready_at: Option, + comments: Vec, + label_additions_empty: bool, + event_types: Vec, + handoff_next_recorded: bool, +} + fn reconciliation_sample_service_owned_issue(state_name: &str) -> TrackerIssue { let active_label = tracker::automation_active_label(TEST_SERVICE_ID); @@ -804,6 +816,343 @@ fn active_run_reconciliation_allows_running_model_execution_until_model_timeout( ActiveRunDisposition::Stalled{ idle_for } if idle_for >= MODEL_EXECUTION_IDLE_TIMEOUT ) + })); +} + +#[test] +fn active_run_reconciliation_defers_live_repo_gate_even_with_dirty_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-live-repo-gate"; + let worktree_path = config.worktree_root().join("PUB-101-repo-gate"); + + git_status_success( + config.repo_root(), + &[ + "worktree", + "add", + "-b", + "x/pubfi-pub-101-repo-gate", + ".worktrees/PUB-101-repo-gate", + "main", + ], + ); + + fs::write(worktree_path.join("README.md"), "repo gate is still running\n") + .expect("tracked worktree file should change"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101-repo-gate", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + state::write_run_operation_marker_for_process( + &worktree_path, + run_id, + 1, + process::id(), + RUN_OPERATION_REPO_GATE, + ) + .expect("live repo-gate marker should write"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("marker should load") + .expect("marker should exist"); + let last_activity = marker.last_activity_unix_epoch().expect("marker should record activity"); + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + ) + .expect("active run inspection should succeed"); + + assert!( + actions.is_empty(), + "live repo gate operation should retain scheduler ownership instead of attention" + ); + + state::write_run_operation_marker_for_process( + &worktree_path, + run_id, + 1, + u32::MAX, + RUN_OPERATION_REPO_GATE, + ) + .expect("stopped repo-gate marker should write"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("marker should reload") + .expect("marker should exist"); + let last_activity = marker.last_activity_unix_epoch().expect("marker should record activity"); + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + ) + .expect("active run inspection should succeed"); + + assert!(actions.iter().any(|action| { + action.issue.id == issue.id + && matches!( + action.disposition, + ActiveRunDisposition::StalledRetainedPartialProgress{ idle_for } + if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + ) + })); +} + +#[test] +fn exited_child_reconciliation_defers_dirty_worktree_with_retry_marker() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let active_label = tracker::automation_active_label(TEST_SERVICE_ID); + let issue = sample_issue("In Progress", &[active_label.as_str()]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let run_id = "run-failed-with-retry-marker"; + let worktree_path = config.worktree_root().join("PUB-101-retry-marker"); + + git_status_success( + config.repo_root(), + &[ + "worktree", + "add", + "-b", + "x/pubfi-pub-101-retry-marker", + ".worktrees/PUB-101-retry-marker", + "main", + ], + ); + + fs::write(worktree_path.join("README.md"), "retry still owns this patch\n") + .expect("tracked worktree file should change"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "failed") + .expect("run attempt should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101-retry-marker", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + state::write_run_retry_schedule(&worktree_path, run_id, 1, "failure", 1) + .expect("retry schedule marker should write"); + + state_store + .append_event(run_id, 1, "turn/completed", "{\"status\":\"completed\"}") + .expect("protocol event should record"); + + let actions = orchestrator::inspect_exited_daemon_child_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + &issue.id, + run_id, + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + ) + .expect("exited child reconciliation should evaluate"); + + assert!( + actions.is_empty(), + "retry marker should keep failed dirty worktree under retry scheduler ownership" + ); +} + +fn record_active_validation_ready_phase_goal_progress( + state_store: &StateStore, + issue_id: &str, + run_id: &str, + progress_phase: &str, + blockers: impl Serialize, +) { + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + issue_id, + run_id, + 1, + "phase_goal_set", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "payload": { + "phase": "implement_to_validation_ready", + "status": "active", + }, + }), + ) + .expect("phase goal should record"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + issue_id, + run_id, + 1, + "progress_checkpoint", + serde_json::json!({ + "phase": progress_phase, + "blockers": blockers, + "verification": ["cargo make check"], + }), + ) + .expect("progress checkpoint should record"); +} + +fn apply_stalled_phase_goal_reconciliation( + progress_phase: &str, + blockers: impl Serialize, +) -> StalledPhaseGoalOutcome { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_manager = + WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); + let run_id = "run-stalled-phase-goal"; + let worktree_path = config.worktree_root().join("PUB-101-phase-goal"); + + git_status_success( + config.repo_root(), + &[ + "worktree", + "add", + "-b", + "x/pubfi-pub-101-phase-goal", + ".worktrees/PUB-101-phase-goal", + "main", + ], + ); + + fs::write(worktree_path.join("README.md"), "validation-ready retained patch\n") + .expect("tracked worktree file should change"); + + state_store + .record_run_attempt(run_id, &issue.id, 1, "running") + .expect("run attempt should record"); + state_store + .upsert_lease("pubfi", &issue.id, run_id, "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + "pubfi", + &issue.id, + "x/pubfi-pub-101-phase-goal", + &worktree_path.display().to_string(), + ) + .expect("worktree mapping should record"); + + record_active_validation_ready_phase_goal_progress( + &state_store, + &issue.id, + run_id, + progress_phase, + blockers, + ); + + state_store + .append_event(run_id, 1, "turn/diff/updated", "{\"changes\":1}") + .expect("stalled dirty issue protocol event should record"); + + let now = + OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_active_run_reconciliation_at( + &tracker, + &config, + &workflow, + &state_store, + None, + now, + ) + .expect("stalled phase-goal inspection should succeed"); + + assert_eq!(actions.len(), 1); + assert!(matches!( + actions[0].disposition, + ActiveRunDisposition::StalledRetainedPartialProgress { .. } + )); + + orchestrator::apply_active_run_reconciliation( + &tracker, + &config, + &state_store, + &worktree_manager, + actions, + ) + .expect("phase-goal reconciliation should apply"); + + let run_status = state_store + .run_attempt(run_id) + .expect("run attempt lookup should succeed") + .expect("run attempt should exist") + .status() + .to_owned(); + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("activity marker should load") + .expect("activity marker should exist"); + let events = state_store + .list_private_execution_events(TEST_SERVICE_ID, &issue.id, run_id, 1) + .expect("private events should load"); + + StalledPhaseGoalOutcome { + run_status, + retry_kind: marker.retry_kind().map(str::to_owned), + retry_ready_at: marker.retry_ready_at_unix_epoch(), + comments: tracker.comments.borrow().clone(), + label_additions_empty: tracker.label_additions.borrow().is_empty(), + event_types: events.iter().map(|event| event.event_type().to_owned()).collect(), + handoff_next_recorded: events.iter().any(|event| { + event.event_type() == "phase_goal_next" + && event.payload()["phase"] == "handoff_evidence" + }), + } +} + +#[test] +fn stalled_retained_phase_goal_reconciliation_schedules_continuation_without_attention() { + let outcome = apply_stalled_phase_goal_reconciliation("verifying", serde_json::json!([])); + + assert_eq!(outcome.run_status, CONTINUATION_PENDING_RUN_STATUS); + assert_eq!(outcome.retry_kind.as_deref(), Some("continuation")); + assert!(outcome.retry_ready_at.is_some()); + assert!(outcome.comments.is_empty()); + assert!(outcome.label_additions_empty); + assert!(outcome.event_types.iter().any(|event| event == "phase_goal_recovery")); + assert!(outcome.handoff_next_recorded); +} + +#[test] +fn stalled_retained_phase_goal_reconciliation_preserves_attention_when_blocked() { + let outcome = apply_stalled_phase_goal_reconciliation( + "blocked", + serde_json::json!(["external evidence is missing"]), + ); + + assert_ne!(outcome.retry_kind.as_deref(), Some("continuation")); + assert!(outcome.comments.iter().any(|comment| { + comment.contains("decodex retained partial progress and needs attention") + && comment.contains("partial_progress_retained") })); } diff --git a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs index 531d2b494..f164b7738 100644 --- a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs +++ b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs @@ -821,7 +821,18 @@ fn schedule_retry_after_child_exit_records_continuation_retry_for_clean_exit() { #[test] fn schedule_retry_after_child_exit_terminalizes_active_phase_goal_tracked_rewrites() { - let (_temp_dir, config, workflow) = temp_project_layout(); + let (_temp_dir, config, workflow) = temp_project_layout_with_workflow_markdown( + &sample_workflow_markdown( + "pubfi", + &[], + "Phase goal validation policy.\n", + 1, + ) + .replace( + "canonicalize_commands = []", + "canonicalize_commands = [\"printf 'rewritten\\\\n' > ready.txt\"]", + ), + ); let issue = sample_service_owned_issue("In Progress"); let tracker = FakeTracker::with_refresh_snapshots(vec![issue.clone()], vec![vec![issue.clone()]]); diff --git a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs index 8473f1efb..fa8f1cd58 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs @@ -28,6 +28,23 @@ fn repo_gate_rejects_dirty_tracked_files_left_by_canonicalize_commands() { assert!(tracked_status.contains("tracked.txt")); } +#[test] +fn repo_gate_allows_existing_tracked_diff_when_commands_preserve_it() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + + commit_worktree_change(repo_root, "tracked.txt", "before\n", "add tracked file"); + + fs::write(repo_root.join("tracked.txt"), "after\n") + .expect("tracked implementation diff should write"); + orchestrator::run_repo_gate_commands( + &[], + &[String::from("grep -qx 'after' tracked.txt")], + repo_root, + ) + .expect("repo gate should allow an existing implementation diff"); +} + #[test] fn repo_gate_cleanliness_check_spawn_failures_require_human_attention() { let (_temp_dir, config, _workflow) = temp_project_layout(); @@ -188,7 +205,18 @@ fn phase_goal_completion_runs_repo_gate_and_persists_handoff_phase() { #[test] fn active_phase_goal_tracked_rewrites_stop_instead_of_repair_continuation() { - let (_temp_dir, config, workflow) = temp_project_layout(); + let (_temp_dir, config, workflow) = temp_project_layout_with_workflow_markdown( + &sample_workflow_markdown( + "pubfi", + &[], + "Phase goal validation policy.\n", + 1, + ) + .replace( + "canonicalize_commands = []", + "canonicalize_commands = [\"printf 'rewritten\\\\n' > ready.txt\"]", + ), + ); let repo_root = config.repo_root(); let issue = sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); let state_store = StateStore::open_in_memory().expect("state store should open"); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index e262c4172..02d45b8da 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -510,9 +510,11 @@ Worktree visibility follows the owning dashboard section: stale active-label or worktree echoes from an older terminal ledger record stay in Run Ledger history instead of reappearing as current attention. - If private evidence shows `phase_goal_recovery` followed by a queued continuation, - the lane is not a retained-attention worktree even when the preceding child failed. - Treat it as Decodex-owned re-entry into the next phase unless a later terminal Run - Ledger row or current attention signal supersedes it. + the lane is not a retained-attention worktree even when the preceding child failed + or stalled reconciliation first found retained dirty progress. Treat it as + Decodex-owned re-entry into the next phase unless a later terminal Run Ledger row, + current attention signal, authority decision request, or blocker checkpoint + supersedes it. Every operator snapshot worktree row includes `ownership`, `ownership_reason`, `provenance`, and optional `recovery_next_action` fields that distinguish active-lane diff --git a/docs/runbook/lane-control-recovery.md b/docs/runbook/lane-control-recovery.md index 4c6a83700..7ecbc6e72 100644 --- a/docs/runbook/lane-control-recovery.md +++ b/docs/runbook/lane-control-recovery.md @@ -127,6 +127,14 @@ typed reason such as `contract_boundary_required`, `external_dependency_required | `uncovered_direction` | The missing requirement, decision, or research gap named in public/private evidence. | A research or Decision Contract captures the missing direction and the issue is updated or requeued from that authority. | | `ambiguous_retained_progress` | Retained worktree diff, ownership markers, PR lineage if present, private evidence, and boundary disposition. | A human chooses one path: resume same lane, finish manual repair, or reset/discard the retained patch explicitly. | +Before treating retained progress as human-owned, check the current run activity +marker. A `retry_kind` marker means retry scheduling still owns the run. A live +`current_operation=repo_gate` marker means the repository gate still owns the run. +For active phase-goal lanes, inspect the latest private progress and decision +evidence: no blockers or decision request means Decodex should recover the phase goal +and schedule continuation; concrete blockers or decision requests keep the normal +manual-attention path. + For every human-required guardrail stop, keep `decodex:needs-attention` until the blocker above is resolved. If the issue returns to automation, request a Linear scan or let the next scheduled scan observe the corrected tracker state; do not bypass the diff --git a/docs/spec/lane-control.md b/docs/spec/lane-control.md index 138a04e13..2d328d046 100644 --- a/docs/spec/lane-control.md +++ b/docs/spec/lane-control.md @@ -213,6 +213,15 @@ the next lane action from authoritative signals. It is also the correct route wh requested control would overwrite useful partial work, hide a blocker, or require guessing human intent. +Retained partial progress is not a manual-attention stop while another runtime owner +is still authoritative for the same run. If the current run activity marker records a +retry schedule, the retry scheduler owns the next action. If the marker records a live +`repo_gate` operation, that gate remains the active owner until the process exits or a +later marker changes ownership. If stalled retained work still has an active phase +goal and the latest progress evidence has no blockers or decision request, Decodex +must try phase-goal recovery and schedule the next continuation instead of writing +`partial_progress_retained`. + Loop guardrail outcomes are not active-lane controls. They first stop the current ineffective strategy. Engineering convergence reasons such as `validation_repeat`, `no_effective_diff`, `remaining_delta_unchanged`, or `review_churn` may then enter diff --git a/docs/spec/linear-execution-ledger.md b/docs/spec/linear-execution-ledger.md index 1864adf9a..aca3eabca 100644 --- a/docs/spec/linear-execution-ledger.md +++ b/docs/spec/linear-execution-ledger.md @@ -245,6 +245,13 @@ terminal disposition absorbs a later runtime failure, the producer should preser source failure class in `evidence` instead of changing the event type or terminal path. +Producers must not emit `partial_progress_retained` while the current run is still +owned by a retry schedule, a live repo-gate operation, or recoverable phase-goal +continuation. Active phase-goal retained work is recoverable when the latest private +progress evidence has no blockers and no authority decision request has been recorded; +in that case the runtime records phase-goal recovery evidence and schedules +continuation instead of a public needs-attention ledger event. + Loop guardrail observations are private runtime evidence while autonomous architecture recovery is still available. A public Linear record is written only when the guardrail has terminalized into a human-required or terminal-failure outcome, with the diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 21c9bac40..0baaf5603 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -290,17 +290,18 @@ the repo gate and select the next phase. An `issue_progress_checkpoint`, final c text, or "await next phase" statement is evidence only; it is not a phase exit and must not be treated as a substitute for goal completion. -If an app-server run fails or a supervised child exits unsuccessfully while the latest -private phase-goal signal for that same run is still an `active` implementation or -repair phase, Decodex must run the registered repo gate before converting retained -tracked changes into human attention. When that runtime-owned gate returns a -continuation transition, Decodex records `phase_goal_recovery`, persists the next -phase goal (`handoff_evidence` after validation pass or `repair_validation_failures` -after continued-repair validation failure), marks the attempt as -`continuation_pending`, and schedules normal continuation re-entry. This recovery path -does not apply to `handoff_evidence`, explicit manual-attention terminal intent, -unsupported phase-goal app-server methods, repo-gate human-attention failures, or -runs with no current active phase-goal signal. +If an app-server run fails, a supervised child exits unsuccessfully, or active-run +reconciliation finds a stalled retained lane while the latest private phase-goal signal +for that same run is still an `active` implementation or repair phase, Decodex must +run the registered repo gate before converting retained tracked changes into human +attention. When that runtime-owned gate returns a continuation transition, Decodex +records `phase_goal_recovery`, persists the next phase goal (`handoff_evidence` after +validation pass or `repair_validation_failures` after continued-repair validation +failure), marks the attempt as `continuation_pending`, and schedules normal +continuation re-entry. This recovery path does not apply to `handoff_evidence`, +explicit manual-attention terminal intent, unsupported phase-goal app-server methods, +repo-gate human-attention failures, runs with no current active phase-goal signal, +authority decision requests, or newer progress checkpoints that record blockers. App-server JSON-RPC transport failures keep the phase where the disconnect occurred. Disconnects before a durable thread session is attached (`initialize`, @@ -417,10 +418,12 @@ When `decodex` runs the repo-native gate during `validating`, it must preserve t - `canonicalize_commands` non-zero exit: continued repair in the retained lane - `verify_commands` non-zero exit: continued repair in the retained lane -- repo gate leaves tracked-file rewrites behind after its commands complete: retained partial progress that requires operator attention +- repo gate changes the tracked-file diff from the pre-gate implementation baseline + after its commands complete: retained partial progress that requires operator + attention - repo-gate command spawn failures or tracked-file cleanliness inspection failures: human-attention failure path immediately -The continued-repair classes above are ordinary bounded churn: the coding agent should keep repairing code and rerun the repo gate rather than requesting `manual_attention` just because the gate has not passed yet. A tracked-rewrite residue after the gate commands have completed is different: Decodex must not infer repository file meaning, generated-artifact ownership, or fixture policy. It must preserve the retained worktree, write `error_class = "partial_progress_retained"` with the repo-gate source class in evidence, and stop until an operator decides whether to finish validation and handoff or reset the patch. Human-attention exits also remain reserved for environment, toolchain, or operator-owned blockers that the coding agent cannot clear from the retained worktree alone. +The continued-repair classes above are ordinary bounded churn: the coding agent should keep repairing code and rerun the repo gate rather than requesting `manual_attention` just because the gate has not passed yet. A tracked-rewrite residue after the gate commands have changed the pre-gate implementation diff is different: Decodex must not infer repository file meaning, generated-artifact ownership, or fixture policy. It must preserve the retained worktree, write `error_class = "partial_progress_retained"` with the repo-gate source class in evidence, and stop until an operator decides whether to finish validation and handoff or reset the patch. Human-attention exits also remain reserved for environment, toolchain, or operator-owned blockers that the coding agent cannot clear from the retained worktree alone. Continued repair is still bounded by loop guardrails. For each retryable failure, Decodex records local `loop_guardrail_checkpoint` evidence and updates the @@ -796,7 +799,17 @@ After a process restart, recent-run history, active lease ownership, retained po budget must not force a synthetic closeout attempt number. - A leased issue that is still in a configured startable state during early control-plane ticks must be treated as a lane that has not finished claiming tracker ownership yet, not as an immediate non-active interruption. - If a running attempt exceeds the app-server idle timeout, `decodex` must treat it as stalled, stop the active run, and mark the attempt `stalled`. -- If stalled reconciliation finds tracked changes in the retained worktree, it must classify the lane as retained partial progress directly. This path must write a human-required `needs_attention` ledger record with `error_class = "partial_progress_retained"` and `terminal_path = "retained_partial_progress"` instead of first routing the lane through `stalled_run_detected` or `terminal_failure`. +- If stalled reconciliation finds tracked changes in the retained worktree, it must + first preserve current runtime ownership. A current retry marker leaves the retry + scheduler in charge, and a live `repo_gate` operation leaves the repo gate in + charge. If the lane still has an active implementation or repair phase goal and the + latest private progress evidence has no blockers or decision request, reconciliation + must run phase-goal recovery, mark the attempt `continuation_pending`, and schedule + continuation instead of writing human attention. Only retained tracked changes with + no current retry owner, no live repo gate, and no applicable phase-goal recovery + path are classified as retained partial progress with a human-required + `needs_attention` ledger record using `error_class = "partial_progress_retained"` + and `terminal_path = "retained_partial_progress"`. - If stalled reconciliation finds no tracked changes in the retained worktree, it must classify the lane as structured retryable recovery with `error_class = "stalled_run_detected"` while retry budget remains. The retry must keep active ownership, write a failure retry schedule for the same worktree, and must not add `decodex:needs-attention` until retry budget exhaustion or another terminal boundary applies. - If the supervised child already exited before the next control-plane tick, stalled reconciliation must still inspect the just-finished lane using recorded protocol activity and retained worktree state rather than skipping directly to generic failure handling. - Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, optional progress diagnostics, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. `last_progress_at` is meaningful-work progress only: tool calls, file or diff changes, plan/model output, repo validation, PR/review/terminal lifecycle, or other explicit work events may refresh it, but account, rate-limit, phase-goal, passive status, warning, token-usage, heartbeat, or similar non-work protocol traffic must only refresh protocol liveness. When a lane remains in `model_execution` with fresh protocol activity but stale or missing work progress and the recent protocol events are only non-work traffic, status should expose `progress_diagnostic = "protocol_only_activity"` while preserving process and protocol liveness separately.