From de0321e116cef0d64809d533a31cd55ab6104574 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Thu, 11 Jun 2026 10:59:50 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Add legacy closeout provenance recovery","authority":"manual"} --- apps/decodex/src/cli.rs | 81 ++++- .../src/orchestrator/operator_dashboard.html | 33 ++ apps/decodex/src/orchestrator/status.rs | 292 ++++++++++++++---- .../tests/operator/status/running_lanes.rs | 168 ++++++++++ .../tests/operator/status/text.rs | 7 + .../tests/operator/status_support.rs | 15 + apps/decodex/src/orchestrator/types.rs | 10 + apps/decodex/src/recovery.rs | 271 +++++++++++++++- apps/decodex/src/state/internal.rs | 71 ++++- apps/decodex/src/state/models.rs | 49 +++ apps/decodex/src/state/store.rs | 52 ++++ apps/decodex/src/state/tests.rs | 36 +++ docs/reference/operator-control-plane.md | 34 +- docs/runbook/recover-review-handoff.md | 26 ++ docs/spec/post-review-lifecycle.md | 13 + docs/spec/runtime.md | 15 +- 16 files changed, 1081 insertions(+), 92 deletions(-) diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs index 3519b6e01..f46f2fe49 100644 --- a/apps/decodex/src/cli.rs +++ b/apps/decodex/src/cli.rs @@ -33,7 +33,10 @@ use crate::{ RadarRefreshQueueRequest, RadarRefreshReleaseDeltaRequest, RadarRenderSignalRequest, RadarValidateRequest, }, - recovery::{self, ReviewHandoffDiagnoseRequest, ReviewHandoffRebindRequest}, + recovery::{ + self, LegacyCloseoutRecoveryRequest, ReviewHandoffDiagnoseRequest, + ReviewHandoffRebindRequest, + }, research_design::{ self, ResearchDesignCompileRequest, ResearchDesignOutcome, ResearchDesignPromoteRequest, ResearchDesignRunInput, @@ -749,6 +752,15 @@ impl RecoverCommand { fn run(&self) -> Result<()> { match &self.command { RecoverSubcommand::ReviewHandoff(args) => args.run(self.project_config.as_path()), + RecoverSubcommand::LegacyCloseout(args) => recovery::run_legacy_closeout( + self.project_config.as_path(), + &LegacyCloseoutRecoveryRequest { + issue: args.issue.clone(), + pr_url: args.pr.clone(), + dry_run: args.dry_run, + manual_authority: args.manual_authority, + }, + ), } } } @@ -801,6 +813,22 @@ struct ReviewHandoffRebindCommand { dry_run: bool, } +#[derive(Debug, Args)] +struct LegacyCloseoutRecoveryCommand { + /// Issue identifier for the legacy cleanup-only worktree. + #[arg(value_name = "ISSUE")] + issue: String, + /// Merged pull request URL that proves the lane's terminal code lineage. + #[arg(long, value_name = "PR_URL")] + pr: String, + /// Validate without writing a Linear execution audit event. + #[arg(long)] + dry_run: bool, + /// Required for non-dry-run audited legacy closeout. + #[arg(long)] + manual_authority: bool, +} + #[derive(Debug, Args)] struct ArchiveLinearCommand { #[command(flatten)] @@ -1539,6 +1567,8 @@ impl From for ResearchDesignOutcome { enum RecoverSubcommand { /// Recover retained review lanes whose handoff marker is missing. ReviewHandoff(ReviewHandoffRecoveryCommand), + /// Record an audited fallback closeout for a legacy cleanup-only worktree. + LegacyCloseout(LegacyCloseoutRecoveryCommand), } #[derive(Debug, Subcommand)] @@ -1679,17 +1709,17 @@ mod tests { use crate::cli::{ AccountCommand, AccountSubcommand, AccountUseCommand, AttemptCommand, Cli, Command, CommitCommand, DiagnoseCommand, EvidenceCommand, LandCommand, LaneCommand, - LaneInspectCommand, LaneInterruptCommand, LaneSteerCommand, LaneSubcommand, ProbeCommand, - ProjectCommand, ProjectConfigArgs, ProjectSubcommand, RadarBackfillReleaseRangeCommand, - RadarBundleBuildCommand, RadarBundleCommand, RadarBundleSubcommand, - RadarBundleValidateCommand, RadarCommand, RadarLedgerCommand, - RadarLedgerIngestExistingCommand, RadarLedgerSubcommand, RadarLedgerSummaryCommand, - RadarRefreshReleaseDeltaCommand, RadarRefreshUpstreamQueueCommand, - RadarRenderSignalCommand, RadarSubcommand, RadarValidateCommand, RecoverCommand, - RecoverSubcommand, ResearchCommand, ResearchCompileCommand, ResearchOutcomeArg, - ResearchPromoteCommand, ResearchSubcommand, ReviewHandoffDiagnoseCommand, - ReviewHandoffRebindCommand, ReviewHandoffRecoveryCommand, ReviewHandoffRecoverySubcommand, - RunCommand, ServeCommand, StatusCommand, + LaneInspectCommand, LaneInterruptCommand, LaneSteerCommand, LaneSubcommand, + LegacyCloseoutRecoveryCommand, ProbeCommand, ProjectCommand, ProjectConfigArgs, + ProjectSubcommand, RadarBackfillReleaseRangeCommand, RadarBundleBuildCommand, + RadarBundleCommand, RadarBundleSubcommand, RadarBundleValidateCommand, RadarCommand, + RadarLedgerCommand, RadarLedgerIngestExistingCommand, RadarLedgerSubcommand, + RadarLedgerSummaryCommand, RadarRefreshReleaseDeltaCommand, + RadarRefreshUpstreamQueueCommand, RadarRenderSignalCommand, RadarSubcommand, + RadarValidateCommand, RecoverCommand, RecoverSubcommand, ResearchCommand, + ResearchCompileCommand, ResearchOutcomeArg, ResearchPromoteCommand, ResearchSubcommand, + ReviewHandoffDiagnoseCommand, ReviewHandoffRebindCommand, ReviewHandoffRecoveryCommand, + ReviewHandoffRecoverySubcommand, RunCommand, ServeCommand, StatusCommand, }; #[test] @@ -2585,4 +2615,31 @@ mod tests { && pr == "https://github.com/hack-ink/pubfi-mono-v2/pull/14" )); } + + #[test] + fn parses_legacy_closeout_manual_authority() { + let cli = Cli::parse_from([ + "decodex", + "recover", + "legacy-closeout", + "PUB-718", + "--pr", + "https://github.com/hack-ink/pubfi-mono-v2/pull/14", + "--manual-authority", + ]); + + assert!(matches!( + cli.command, + Command::Recover(RecoverCommand { + command: RecoverSubcommand::LegacyCloseout(LegacyCloseoutRecoveryCommand { + issue, + pr, + dry_run: false, + manual_authority: true, + }), + .. + }) if issue == "PUB-718" + && pr == "https://github.com/hack-ink/pubfi-mono-v2/pull/14" + )); + } } diff --git a/apps/decodex/src/orchestrator/operator_dashboard.html b/apps/decodex/src/orchestrator/operator_dashboard.html index a8908f956..fa017627c 100644 --- a/apps/decodex/src/orchestrator/operator_dashboard.html +++ b/apps/decodex/src/orchestrator/operator_dashboard.html @@ -9988,6 +9988,16 @@

${escapeHtml(title)}

"Post-land cleanup pending.", }; } + if (worktree.provenance?.audit_required) { + return { + sortRank: 2, + tone: "tone-blocked", + label: "legacy cleanup audit", + summary: + worktree.ownership_reason || + "Legacy worktree provenance is missing; verify terminal state before cleanup.", + }; + } return { sortRank: 2, tone: "tone-recovery", @@ -10011,6 +10021,28 @@

${escapeHtml(title)}

`; } + function renderWorktreeProvenanceFields(worktree) { + const provenance = worktree.provenance; + if (!provenance) { + return ""; + } + + const createdAt = unixEpochSecondsToIso(provenance.created_at_unix) || "unknown"; + const updatedAt = unixEpochSecondsToIso(provenance.updated_at_unix) || "unknown"; + const audit = provenance.audit_required ? field("Audit", "required") : ""; + const nextAction = worktree.recovery_next_action + ? field("Next action", worktree.recovery_next_action) + : ""; + + return ` + ${field("Provenance", displayToken(provenance.source || "unknown"))} + ${field("Recorded", createdAt)} + ${field("Refreshed", updatedAt)} + ${audit} + ${nextAction} + `; + } + function recoveryWorktreeShouldDefaultOpen(renderedWorktree) { const role = renderedWorktree.role; @@ -10075,6 +10107,7 @@

${escapeHtml(worktree.branch_name)}

${field("Ownership", displayToken(worktree.ownership || role.label))} ${field("Branch", worktree.branch_name)} ${field("Worktree path", worktree.worktree_path)} + ${renderWorktreeProvenanceFields(worktree)} ${renderWorktreeHygieneFields(worktree)} diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index 670ddba0c..ab97e0bc7 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -13,6 +13,9 @@ use libc::c_void; #[cfg(target_os = "macos")] use libc::proc_bsdinfo; use github::GhCommandResolution; +use state::WORKTREE_PROVENANCE_FILESYSTEM_SCAN; +use state::WORKTREE_PROVENANCE_GIT_HYGIENE_SCAN; +use state::WORKTREE_PROVENANCE_LEGACY_UNKNOWN; use crate::pull_request::{self, PullRequestLandingGateView}; use crate::worktree; @@ -242,6 +245,8 @@ struct OperatorIssueDisplayMetadata { struct WorktreeOwnership { kind: &'static str, reason: String, + next_action: Option, + audit_required: bool, } pub(crate) fn ensure_project_has_no_merged_worktree_cleanup_debt( @@ -949,6 +954,8 @@ fn refresh_worktree_ownership( for (worktree, ownership) in snapshot.worktrees.iter_mut().zip(ownership) { worktree.ownership = ownership.kind.to_owned(); worktree.ownership_reason = ownership.reason; + worktree.recovery_next_action = ownership.next_action; + worktree.provenance.audit_required = ownership.audit_required; } } @@ -961,6 +968,8 @@ fn worktree_ownership( return WorktreeOwnership { kind: "active_lane", reason: format!("Active lane `{}` owns this worktree.", run.run_id), + next_action: None, + audit_required: false, }; } if let Some(lane) = worktree_post_review_owner(worktree, snapshot) { @@ -970,6 +979,8 @@ fn worktree_ownership( "Review & Landing owns this worktree as `{}`.", lane.classification ), + next_action: None, + audit_required: false, }; } @@ -979,6 +990,8 @@ fn worktree_ownership( reason: String::from( "Intake Queue owns this worktree because the issue needs operator attention.", ), + next_action: None, + audit_required: false, }; } @@ -986,12 +999,20 @@ fn worktree_ownership( return WorktreeOwnership { kind: "post_land_cleanup", reason: hygiene.reason.clone(), + next_action: Some(String::from( + "inspect the merged worktree, preserve or discard local changes intentionally, then remove the linked worktree", + )), + audit_required: false, }; } + let audit_required = worktree.provenance.source == WORKTREE_PROVENANCE_LEGACY_UNKNOWN; + WorktreeOwnership { kind: "cleanup_only", reason: worktree_cleanup_only_reason(worktree, completed_state), + next_action: audit_required.then(|| legacy_cleanup_next_action(worktree)), + audit_required, } } @@ -1043,6 +1064,12 @@ fn worktree_cleanup_only_reason( worktree: &OperatorWorktreeStatus, completed_state: Option<&str>, ) -> String { + if worktree.provenance.source == WORKTREE_PROVENANCE_LEGACY_UNKNOWN { + return String::from( + "Legacy worktree mapping has no durable runtime provenance; no active, queued, or post-review lane owns it, so Decodex cannot automatically prove PR or closeout lineage.", + ); + } + if let (Some(issue_state), Some(completed_state)) = (worktree.issue_state.as_deref(), completed_state) && issue_state == completed_state { @@ -1056,6 +1083,15 @@ fn worktree_cleanup_only_reason( ) } +fn legacy_cleanup_next_action(worktree: &OperatorWorktreeStatus) -> String { + let issue = worktree.issue_identifier.as_deref().unwrap_or(&worktree.issue_id); + + format!( + "verify tracker/PR terminal state and clean git status for `{}`, then run `decodex recover legacy-closeout {issue} --pr --dry-run`; rerun with `--manual-authority` before removing this worktree", + worktree.worktree_path + ) +} + fn refresh_operator_project_summary(snapshot: &mut OperatorStatusSnapshot) { let active_run_count = snapshot.active_runs.len(); let running_lane_count = @@ -1315,6 +1351,8 @@ fn operator_status_worktrees( ownership_reason: String::from( "No active lane, queued recovery, or post-review lane currently owns this worktree.", ), + provenance: operator_worktree_provenance_from_mapping(&mapping), + recovery_next_action: None, hygiene: None, }) .collect::>(); @@ -1346,6 +1384,12 @@ fn operator_status_worktrees( ownership_reason: String::from( "No active lane, queued recovery, or post-review lane currently owns this worktree.", ), + provenance: operator_worktree_provenance( + WORKTREE_PROVENANCE_FILESYSTEM_SCAN, + None, + None, + ), + recovery_next_action: None, hygiene: None, }); } @@ -1471,6 +1515,14 @@ fn operator_worktree_status_from_cleanup_debt( worktree_path: relative_path, ownership: String::from("post_land_cleanup"), ownership_reason: reason.clone(), + provenance: operator_worktree_provenance( + WORKTREE_PROVENANCE_GIT_HYGIENE_SCAN, + None, + None, + ), + recovery_next_action: Some(String::from( + "inspect the merged worktree, preserve or discard local changes intentionally, then remove the linked worktree", + )), hygiene: Some(OperatorWorktreeHygieneStatus { classification: String::from(classification), default_branch, @@ -1480,6 +1532,29 @@ fn operator_worktree_status_from_cleanup_debt( } } +fn operator_worktree_provenance_from_mapping( + mapping: &WorktreeMapping, +) -> OperatorWorktreeProvenanceStatus { + operator_worktree_provenance( + mapping.provenance().source(), + mapping.provenance().created_at_unix(), + mapping.provenance().updated_at_unix(), + ) +} + +fn operator_worktree_provenance( + source: &str, + created_at_unix: Option, + updated_at_unix: Option, +) -> OperatorWorktreeProvenanceStatus { + OperatorWorktreeProvenanceStatus { + source: source.to_owned(), + created_at_unix, + updated_at_unix, + audit_required: false, + } +} + fn add_operator_snapshot_warning(snapshot: &mut OperatorStatusSnapshot, warning: &str) { if !snapshot.warnings.iter().any(|existing| existing == warning) { snapshot.warnings.push(warning.to_owned()); @@ -4355,81 +4430,165 @@ where let mut active_issues = Vec::new(); for issue in issues { - let worktree = worktree_manager.plan_for_issue(&issue.identifier); - - if !worktree.path.exists() { - continue; + if let Some(active_issue) = recover_issue_runtime_state( + tracker, + project, + workflow, + state_store, + &worktree_manager, + issue, + now_unix_epoch, + )? { + active_issues.push(active_issue); } + } - state_store.canonicalize_issue_identity(&issue.identifier, &issue.id)?; - state_store.upsert_worktree( - project.service_id(), - &issue.id, - &worktree.branch_name, - &worktree.path.display().to_string(), - )?; + active_issues.sort_by(compare_issue_candidates); - let activity_marker = state::read_run_activity_marker_snapshot(&worktree.path)?; + Ok(RecoveredRuntimeState { active_issues }) +} - if issue.state.name == workflow.frontmatter().tracker().success_state() - && issue_has_service_ownership(tracker, &issue, project.service_id())? - && let Some(marker) = activity_marker.as_ref() - && worktree_activity_marker_is_fresh(marker, now_unix_epoch) - { - record_recovered_activity_lease(project, state_store, &issue, marker)?; +fn recover_issue_runtime_state( + tracker: &T, + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + worktree_manager: &WorktreeManager, + issue: TrackerIssue, + now_unix_epoch: i64, +) -> crate::prelude::Result> +where + T: IssueTracker, +{ + let worktree = worktree_manager.plan_for_issue(&issue.identifier); - continue; - } - if issue_passes_closeout_dispatch_policy(tracker, &issue, project, workflow, state_store)? - { - match activity_marker.as_ref() { - Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { - record_recovered_activity_lease(project, state_store, &issue, marker)?; + if !worktree.path.exists() { + return Ok(None); + } - continue; - }, - _ => {}, - } - } - if issue_passes_retry_dispatch_policy( - tracker, + state_store.canonicalize_issue_identity(&issue.identifier, &issue.id)?; + + let activity_marker = state::read_run_activity_marker_snapshot(&worktree.path)?; + let existing_worktree_mapping = state_store.worktree_for_issue(&issue.id)?; + let recovered_service_ownership = + issue_has_recovered_service_ownership(tracker, &issue, project.service_id())?; + + if existing_worktree_mapping.is_none() && recovered_service_ownership { + upsert_recovered_worktree_mapping( + project, + state_store, &issue, + &worktree, + activity_marker.as_ref(), + )?; + } + if issue.state.name == workflow.frontmatter().tracker().success_state() + && recovered_service_ownership + && let Some(marker) = activity_marker.as_ref() + && worktree_activity_marker_is_fresh(marker, now_unix_epoch) + { + upsert_recovered_worktree_mapping( project, - workflow, state_store, - RetryIssueStateHint::default(), - )? { - match activity_marker.as_ref() { - Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { - record_recovered_activity_lease(project, state_store, &issue, marker)?; + &issue, + &worktree, + activity_marker.as_ref(), + )?; + record_recovered_activity_lease(project, state_store, &issue, marker)?; - continue; - }, - Some(marker) => { - clear_recovered_issue_lease( - project.service_id(), - &issue.id, - Some(marker.run_id()), - state_store, - )?; - }, - None => { - clear_recovered_issue_lease( - project.service_id(), - &issue.id, - None, - state_store, - )?; - }, - } + return Ok(None); + } + if issue_passes_closeout_dispatch_policy(tracker, &issue, project, workflow, state_store)? { + upsert_recovered_worktree_mapping( + project, + state_store, + &issue, + &worktree, + activity_marker.as_ref(), + )?; + + match activity_marker.as_ref() { + Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { + record_recovered_activity_lease(project, state_store, &issue, marker)?; + + return Ok(None); + }, + _ => {}, + } + } + if issue_passes_retry_dispatch_policy( + tracker, + &issue, + project, + workflow, + state_store, + RetryIssueStateHint::default(), + )? { + upsert_recovered_worktree_mapping( + project, + state_store, + &issue, + &worktree, + activity_marker.as_ref(), + )?; - active_issues.push(issue); + match activity_marker.as_ref() { + Some(marker) if worktree_activity_marker_is_fresh(marker, now_unix_epoch) => { + record_recovered_activity_lease(project, state_store, &issue, marker)?; + + return Ok(None); + }, + Some(marker) => { + clear_recovered_issue_lease( + project.service_id(), + &issue.id, + Some(marker.run_id()), + state_store, + )?; + }, + None => { + clear_recovered_issue_lease( + project.service_id(), + &issue.id, + None, + state_store, + )?; + }, } + + return Ok(Some(issue)); } - active_issues.sort_by(compare_issue_candidates); + Ok(None) +} - Ok(RecoveredRuntimeState { active_issues }) +fn upsert_recovered_worktree_mapping( + project: &ServiceConfig, + state_store: &StateStore, + issue: &TrackerIssue, + worktree: &WorktreeSpec, + activity_marker: Option<&RunActivityMarker>, +) -> crate::prelude::Result<()> { + state_store.upsert_recovered_worktree( + project.service_id(), + &issue.id, + &worktree.branch_name, + &worktree.path.display().to_string(), + recovered_worktree_observed_at_unix(activity_marker), + ) +} + +fn recovered_worktree_observed_at_unix(activity_marker: Option<&RunActivityMarker>) -> Option { + activity_marker.and_then(|marker| { + [ + marker.last_activity_unix_epoch(), + marker.last_protocol_activity_unix_epoch(), + marker.last_progress_unix_epoch(), + ] + .into_iter() + .flatten() + .max() + }) } fn record_recovered_activity_lease( @@ -5706,6 +5865,10 @@ fn format_optional_unix_timestamp(unix_epoch: Option) -> Option { }) } +fn format_optional_i64(value: Option) -> String { + value.map_or_else(|| String::from("none"), |value| value.to_string()) +} + fn classify_operator_run_operation(phase: &str, marker_current_operation: Option<&str>) -> String { match phase { "retry_backoff" | "waiting_continuation" => { @@ -6156,14 +6319,19 @@ fn append_rendered_recovery_worktrees( for (role, worktree) in rendered_worktrees { output.push_str(&format!( - "- issue_id: {}\n issue: {}\n state: {}\n role: {}\n reason: {}\n branch: {}\n worktree_path: {}\n", + "- issue_id: {}\n issue: {}\n state: {}\n role: {}\n reason: {}\n branch: {}\n worktree_path: {}\n provenance_source: {}\n provenance_created_at_unix: {}\n provenance_updated_at_unix: {}\n audit_required: {}\n recovery_next_action: {}\n", worktree.issue_id, worktree.issue_identifier.as_deref().unwrap_or("none"), worktree.issue_state.as_deref().unwrap_or("unknown"), role, worktree.ownership_reason, worktree.branch_name, - worktree.worktree_path + worktree.worktree_path, + worktree.provenance.source, + format_optional_i64(worktree.provenance.created_at_unix), + format_optional_i64(worktree.provenance.updated_at_unix), + worktree.provenance.audit_required, + worktree.recovery_next_action.as_deref().unwrap_or("none") )); } } diff --git a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs index 325bced22..c24f7f09a 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs @@ -1,3 +1,5 @@ +use rusqlite::Connection; + #[test] fn failure_comments_use_repo_relative_worktree_paths() { let (_temp_dir, config, _workflow) = temp_project_layout(); @@ -639,6 +641,172 @@ fn completed_retained_worktree_without_post_review_owner_is_cleanup_only() { assert!(!rendered.contains("review_handoff_missing")); } +#[test] +fn legacy_cleanup_only_worktree_requires_audited_manual_closeout() { + let (temp_dir, config, workflow) = temp_project_layout(); + let db_path = temp_dir.path().join("legacy-runtime.sqlite3"); + let issue = sample_issue_with_sort_fields( + "issue-cleanup", + "PUB-199", + "Done", + &[], + Some(4), + "2026-03-13T07:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + + { + let connection = Connection::open(&db_path).expect("legacy db should open"); + + connection + .execute_batch(&format!( + "CREATE TABLE worktrees ( + issue_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + worktree_path TEXT NOT NULL + ); + INSERT INTO worktrees (issue_id, project_id, branch_name, worktree_path) + VALUES ('{}', 'pubfi', 'x/pubfi-pub-199', '{}');", + issue.id, + worktree_path.display() + )) + .expect("legacy worktree row should write"); + } + + let tracker = FakeTracker::new(vec![issue]); + let state_store = StateStore::open(&db_path).expect("state store should migrate"); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(snapshot.worktrees[0].ownership, "cleanup_only"); + assert_eq!(snapshot.worktrees[0].provenance.source, "legacy_unknown"); + assert!(snapshot.worktrees[0].provenance.audit_required); + assert!( + snapshot.worktrees[0] + .recovery_next_action + .as_deref() + .is_some_and(|action| action.contains("decodex recover legacy-closeout PUB-199")) + ); + assert_eq!(snapshot_json["worktrees"][0]["provenance"]["source"], "legacy_unknown"); + assert_eq!(snapshot_json["worktrees"][0]["provenance"]["audit_required"], true); + assert!(rendered.contains("provenance_source: legacy_unknown")); + assert!(rendered.contains("audit_required: true")); + assert!(rendered.contains("recovery_next_action: verify tracker/PR terminal state")); + assert!(rendered.contains("decodex recover legacy-closeout PUB-199")); +} + +#[test] +fn runtime_recovery_preserves_legacy_cleanup_only_provenance_without_recoverable_owner() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let db_path = temp_dir.path().join("runtime.sqlite3"); + let (_layout_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue_with_sort_fields( + "issue-legacy", + "PUB-199", + "Done", + &[], + Some(4), + "2026-03-13T07:16:17.133Z", + ); + let worktree_path = config.worktree_root().join(&issue.identifier); + + fs::create_dir_all(&worktree_path).expect("legacy worktree path should exist"); + + { + let connection = Connection::open(&db_path).expect("legacy db should open"); + + connection + .execute_batch(&format!( + "CREATE TABLE worktrees ( + issue_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + worktree_path TEXT NOT NULL + ); + INSERT INTO worktrees (issue_id, project_id, branch_name, worktree_path) + VALUES ('{}', 'pubfi', 'x/pubfi-pub-199', '{}');", + issue.id, + worktree_path.display() + )) + .expect("legacy worktree row should write"); + } + + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open(&db_path).expect("state store should migrate"); + let recovered_state = orchestrator::recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("runtime recovery should succeed"); + let mapping = state_store + .worktree_for_issue(&issue.id) + .expect("mapping lookup should succeed") + .expect("legacy mapping should remain"); + + assert!( + recovered_state.active_issues.is_empty(), + "terminal cleanup-only worktree should not become a retry lane" + ); + assert_eq!(mapping.provenance().source(), "legacy_unknown"); + assert_eq!(mapping.provenance().created_at_unix(), None); + assert_eq!(mapping.provenance().updated_at_unix(), None); +} + +#[test] +fn runtime_recovery_records_recovered_provenance_for_fresh_active_worktree() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_active_issue("In Progress"); + let tracker = FakeTracker::new(vec![issue.clone()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let worktree_path = config.worktree_root().join(&issue.identifier); + + fs::create_dir_all(&worktree_path).expect("active worktree path should exist"); + state::write_run_activity_marker(&worktree_path, "run-1", 1) + .expect("activity marker should write"); + + let marker = state::read_run_activity_marker_snapshot(&worktree_path) + .expect("activity marker should load") + .expect("activity marker should exist"); + let observed_at_unix = marker + .last_activity_unix_epoch() + .expect("activity marker should have a stable timestamp"); + let recovered_state = orchestrator::recover_runtime_state_from_tracker_and_worktrees( + &tracker, + &config, + &workflow, + &state_store, + ) + .expect("runtime recovery should succeed"); + let mapping = state_store + .worktree_for_issue(&issue.id) + .expect("mapping lookup should succeed") + .expect("recovered mapping should exist"); + let lease = state_store + .lease_for_issue(&issue.id) + .expect("lease lookup should succeed") + .expect("fresh active marker should recover the lease"); + + assert!( + recovered_state.active_issues.is_empty(), + "fresh marker should recover as the active lease instead of a retry queue item" + ); + assert_eq!(mapping.provenance().source(), "runtime_recovered"); + assert_eq!(mapping.provenance().created_at_unix(), Some(observed_at_unix)); + assert_eq!(mapping.provenance().updated_at_unix(), Some(observed_at_unix)); + assert_eq!(lease.run_id(), "run-1"); +} + #[test] fn operator_status_snapshot_reports_retry_backoff_from_worktree_marker() { for (retry_kind, expected_wait_reason) in diff --git a/apps/decodex/src/orchestrator/tests/operator/status/text.rs b/apps/decodex/src/orchestrator/tests/operator/status/text.rs index 08a9e3841..41b42fbfd 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/text.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/text.rs @@ -500,6 +500,13 @@ fn operator_status_text_surfaces_cleanup_blocker_pr_url() { ownership_reason: String::from( "Review & Landing owns this worktree as `cleanup_blocked`.", ), + provenance: orchestrator::OperatorWorktreeProvenanceStatus { + source: String::from("runtime_recorded"), + created_at_unix: Some(1), + updated_at_unix: Some(2), + audit_required: false, + }, + recovery_next_action: None, hygiene: None, }], post_review_lanes: vec![orchestrator::OperatorPostReviewLaneStatus { diff --git a/apps/decodex/src/orchestrator/tests/operator/status_support.rs b/apps/decodex/src/orchestrator/tests/operator/status_support.rs index fd59f44ce..7d3982b6f 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status_support.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status_support.rs @@ -395,6 +395,8 @@ fn operator_status_text_worktrees() -> Vec { ownership_reason: String::from( "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", ), + provenance: test_worktree_provenance("runtime_recorded"), + recovery_next_action: None, hygiene: None, }, orchestrator::OperatorWorktreeStatus { @@ -405,6 +407,8 @@ fn operator_status_text_worktrees() -> Vec { worktree_path: String::from(".worktrees/PUB-101"), ownership: String::from("active_lane"), ownership_reason: String::from("Active lane `run-1` owns this worktree."), + provenance: test_worktree_provenance("runtime_recorded"), + recovery_next_action: None, hygiene: None, }, orchestrator::OperatorWorktreeStatus { @@ -417,11 +421,22 @@ fn operator_status_text_worktrees() -> Vec { ownership_reason: String::from( "Review & Landing owns this worktree as `ready_to_land`.", ), + provenance: test_worktree_provenance("runtime_recorded"), + recovery_next_action: None, hygiene: None, }, ] } +fn test_worktree_provenance(source: &str) -> orchestrator::OperatorWorktreeProvenanceStatus { + orchestrator::OperatorWorktreeProvenanceStatus { + source: source.to_owned(), + created_at_unix: Some(1), + updated_at_unix: Some(2), + audit_required: false, + } +} + fn operator_status_text_post_review_lanes() -> Vec { vec![orchestrator::OperatorPostReviewLaneStatus { issue_id: String::from("issue-3"), diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index d153bac2b..f24a34b07 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -1575,9 +1575,19 @@ struct OperatorWorktreeStatus { worktree_path: String, ownership: String, ownership_reason: String, + provenance: OperatorWorktreeProvenanceStatus, + recovery_next_action: Option, hygiene: Option, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct OperatorWorktreeProvenanceStatus { + source: String, + created_at_unix: Option, + updated_at_unix: Option, + audit_required: bool, +} + #[derive(Clone, Debug, Eq, PartialEq, Serialize)] struct OperatorWorktreeHygieneStatus { classification: String, diff --git a/apps/decodex/src/recovery.rs b/apps/decodex/src/recovery.rs index 61171445c..51d52302f 100644 --- a/apps/decodex/src/recovery.rs +++ b/apps/decodex/src/recovery.rs @@ -38,6 +38,8 @@ const REVIEW_HANDOFF_REBIND_REQUIRED_CLASSIFICATION: &str = "review_handoff_rebi const REVIEW_HANDOFF_UNVERIFIED_CLASSIFICATION: &str = "review_handoff_unverified"; const REVIEW_HANDOFF_MISMATCH_CLASSIFICATION: &str = "review_handoff_mismatch"; const REVIEW_HANDOFF_REBIND_EVENT: &str = "review_handoff_rebind"; +const LEGACY_MANUAL_CLOSEOUT_EVENT: &str = "closeout"; +const LEGACY_MANUAL_CLOSEOUT_ANCHOR: &str = "legacy_manual_closeout"; const REBOUND_ORCHESTRATION_PHASE: &str = "request_pending"; const LINEAR_CONNECTOR_BACKOFF_WARNING: &str = "tracker_rate_limited"; const LINEAR_CONNECTOR_BACKOFF_SECS: i64 = 15 * 60; @@ -62,6 +64,19 @@ pub(crate) struct ReviewHandoffRebindRequest { pub(crate) dry_run: bool, } +/// Explicit legacy closeout audit request. +#[derive(Debug)] +pub(crate) struct LegacyCloseoutRecoveryRequest { + /// Issue identifier to audit. + pub(crate) issue: String, + /// Merged pull request URL that proves terminal code lineage. + pub(crate) pr_url: String, + /// Validate without writing a tracker audit comment. + pub(crate) dry_run: bool, + /// Required for non-dry-run mutation. + pub(crate) manual_authority: bool, +} + #[derive(Serialize)] struct ReviewHandoffRecoveryReport { project_id: String, @@ -146,6 +161,15 @@ struct RebindValidation { success_state_transition: Option, } +struct LegacyCloseoutValidation { + issue: TrackerIssue, + worktree: WorktreeMapping, + landing_state: PullRequestLandingState, + local_head_oid: String, + merge_commit: String, + worktree_path_for_event: Option, +} + #[derive(Debug)] struct RebindSuccessStateTransition { state_name: String, @@ -273,6 +297,50 @@ pub(crate) fn run_review_handoff_rebind( Ok(()) } +/// Run an explicit audited legacy closeout fallback. +pub(crate) fn run_legacy_closeout( + config_path: Option<&Path>, + request: &LegacyCloseoutRecoveryRequest, +) -> Result<()> { + let context = load_recovery_context(config_path)?; + let validation = validate_legacy_closeout_request(&context, request)?; + + if request.dry_run { + println!( + "dry run: legacy closeout validated for project={} issue={} branch={} pr={} head={} merge_commit={} provenance={}", + context.config.service_id(), + validation.issue.identifier, + validation.worktree.branch_name(), + landing_url(&validation.landing_state), + validation.local_head_oid, + validation.merge_commit, + validation.worktree.provenance().source() + ); + + return Ok(()); + } + if !request.manual_authority { + eyre::bail!( + "`recover legacy-closeout` writes a closeout audit and requires --manual-authority outside dry-run mode." + ); + } + + let event = legacy_closeout_event(&context, &validation); + let audit_recorded = write_legacy_closeout_audit(&context, &validation, &event)?; + + println!( + "legacy closeout audit ok: project={} issue={} branch={} pr={} head={} merge_commit={} audit_recorded={audit_recorded}", + context.config.service_id(), + validation.issue.identifier, + validation.worktree.branch_name(), + landing_url(&validation.landing_state), + validation.local_head_oid, + validation.merge_commit, + ); + + Ok(()) +} + fn load_recovery_context(config_path: Option<&Path>) -> Result { let state_store = runtime::open_runtime_store()?; let config_path = resolve_recovery_config_path(config_path, &state_store)?; @@ -915,6 +983,87 @@ fn validate_rebind_request( }) } +fn validate_legacy_closeout_request( + context: &RecoveryContext, + request: &LegacyCloseoutRecoveryRequest, +) -> Result { + let issue = load_issue_by_identifier(&context.tracker, &request.issue)?; + + validate_legacy_closeout_issue_state(context.workflow.frontmatter().tracker(), &issue)?; + + let worktree = legacy_closeout_worktree(context, &issue)?; + + if !worktree.provenance().is_legacy_unknown() { + eyre::bail!( + "Issue `{}` worktree provenance is `{}`; legacy closeout requires `legacy_unknown` cleanup-only provenance.", + issue.identifier, + worktree.provenance().source() + ); + } + + let (landing_state, default_branch) = inspect_project_pull_request(context, &request.pr_url)?; + + if landing_state.base_ref_name != default_branch { + eyre::bail!( + "Pull request `{}` targets `{}`, but configured default branch is `{}`.", + request.pr_url, + landing_state.base_ref_name, + default_branch + ); + } + if landing_state.state != "MERGED" { + eyre::bail!( + "Pull request `{}` is `{}`; legacy closeout requires `MERGED`.", + request.pr_url, + landing_state.state + ); + } + + let local_head_oid = validate_legacy_closeout_worktree(&worktree, &landing_state)?; + let merge_commit = inspect_project_pull_request_merge_commit(context, &request.pr_url)?; + let worktree_path_for_event = + repository_relative_path(context.config.repo_root(), worktree.worktree_path()); + + Ok(LegacyCloseoutValidation { + issue, + worktree, + landing_state, + local_head_oid, + merge_commit, + worktree_path_for_event, + }) +} + +fn validate_legacy_closeout_issue_state( + tracker_policy: &WorkflowTracker, + issue: &TrackerIssue, +) -> Result<()> { + if tracker_policy.terminal_states().iter().any(|state| state == &issue.state.name) { + return Ok(()); + } + + eyre::bail!( + "Issue `{}` is in `{}`, but legacy closeout requires a terminal state: {}.", + issue.identifier, + issue.state.name, + tracker_policy.terminal_states().join(", ") + ) +} + +fn legacy_closeout_worktree( + context: &RecoveryContext, + issue: &TrackerIssue, +) -> Result { + if let Some(worktree) = context.state_store.worktree_for_issue(&issue.id)? { + return Ok(worktree); + } + if let Some(worktree) = context.state_store.worktree_for_issue(&issue.identifier)? { + return Ok(worktree); + } + + eyre::bail!("Issue `{}` has no retained worktree mapping.", issue.identifier) +} + fn load_issue_by_identifier(tracker: &T, issue_identifier: &str) -> Result where T: IssueTracker + ?Sized, @@ -1138,9 +1287,38 @@ fn inspect_project_pull_request( Ok((landing_state, repository.default_branch)) } +fn inspect_project_pull_request_merge_commit( + context: &RecoveryContext, + pr_url: &str, +) -> Result { + let github_token = context.config.github().resolve_token()?; + + github::inspect_pull_request_merge_commit( + context.config.repo_root(), + pr_url, + &github_token, + context.config.github().command_path(), + ) +} + fn validate_rebind_worktree( worktree: &WorktreeMapping, landing_state: &PullRequestLandingState, +) -> Result { + validate_retained_pr_worktree(worktree, landing_state, "rebind") +} + +fn validate_legacy_closeout_worktree( + worktree: &WorktreeMapping, + landing_state: &PullRequestLandingState, +) -> Result { + validate_retained_pr_worktree(worktree, landing_state, "legacy closeout") +} + +fn validate_retained_pr_worktree( + worktree: &WorktreeMapping, + landing_state: &PullRequestLandingState, + action_label: &str, ) -> Result { let local_branch = worktree_checkout_branch_name(worktree.worktree_path())? .ok_or_else(|| eyre::eyre!("Retained worktree is detached."))?; @@ -1161,8 +1339,8 @@ fn validate_rebind_worktree( } if !worktree_is_clean(worktree.worktree_path())? { eyre::bail!( - "Retained worktree `{}` has local changes; rebind requires a clean lane checkout.", - worktree.worktree_path().display() + "Retained worktree `{}` has local changes; {action_label} requires a clean lane checkout.", + worktree.worktree_path().display(), ); } @@ -1327,6 +1505,95 @@ fn write_rebind_audit( Ok(()) } +fn legacy_closeout_event( + context: &RecoveryContext, + validation: &LegacyCloseoutValidation, +) -> LinearExecutionEventRecord { + let pr_url = landing_url(&validation.landing_state); + let stable_anchor = records::stable_event_anchor(&[ + pr_url, + &validation.local_head_oid, + &validation.merge_commit, + LEGACY_MANUAL_CLOSEOUT_ANCHOR, + ]); + let run_id = format!("legacy-closeout-{}", validation.issue.identifier.to_ascii_lowercase()); + let mut event = LinearExecutionEventRecord::new( + LinearExecutionEventIdentity { + service_id: context.config.service_id(), + issue_id: &validation.issue.id, + issue_identifier: &validation.issue.identifier, + run_id: &run_id, + attempt_number: 1, + }, + LEGACY_MANUAL_CLOSEOUT_EVENT, + current_timestamp(), + &stable_anchor, + ); + + event.branch = Some(validation.worktree.branch_name().to_owned()); + event.worktree_path = validation.worktree_path_for_event.clone(); + event.pr_url = Some(pr_url.to_owned()); + event.pr_head_sha = Some(validation.local_head_oid.clone()); + event.pr_base_ref = Some(validation.landing_state.base_ref_name.clone()); + event.commit_sha = Some(validation.merge_commit.clone()); + event.validation_result = Some(String::from("passed")); + event.target_state = Some(validation.issue.state.name.clone()); + event.cleanup_status = Some(String::from("manual_audit_recorded")); + event.summary = Some(format!( + "Legacy manual closeout audit recorded for {} after merged PR {}.", + validation.issue.identifier, pr_url + )); + event.evidence = Some(vec![ + format!("issue_state={}", validation.issue.state.name), + format!("branch={}", validation.worktree.branch_name()), + format!("pr_url={pr_url}"), + format!("pr_head_sha={}", validation.local_head_oid), + format!("merge_commit={}", validation.merge_commit), + format!("worktree_provenance={}", validation.worktree.provenance().source()), + String::from("worktree_clean=true"), + ]); + event.next_action = Some(String::from( + "remove the local worktree only after preserving or discarding local-only changes intentionally", + )); + + event +} + +fn write_legacy_closeout_audit( + context: &RecoveryContext, + validation: &LegacyCloseoutValidation, + event: &LinearExecutionEventRecord, +) -> Result { + let body = format!( + "Decodex legacy manual closeout audit: verified merged PR `{}` for `{}`. Runtime provenance was `{}`, so this records the manual fallback before local cleanup.", + landing_url(&validation.landing_state), + validation.issue.identifier, + validation.worktree.provenance().source() + ); + let privacy_classifier = ConfiguredPublicProjectionPrivacyClassifier::from_config( + context.config.privacy_classifier(), + )?; + let projection = + tracker::prepare_linear_execution_event_comment(&body, event, &privacy_classifier)?; + let recorded = context.state_store.record_linear_execution_event(&projection.record)?; + + if !recorded { + return Ok(false); + } + + if let Err(error) = tracker::create_prepared_linear_execution_event_comment_without_remote_scan( + &context.tracker, + &validation.issue.id, + &projection, + ) { + context.state_store.forget_linear_execution_event(&projection.record.idempotency_key)?; + + return Err(error); + } + + Ok(true) +} + fn landing_url(landing_state: &PullRequestLandingState) -> &str { &landing_state.url } diff --git a/apps/decodex/src/state/internal.rs b/apps/decodex/src/state/internal.rs index 0d6c61a3a..0030e6b15 100644 --- a/apps/decodex/src/state/internal.rs +++ b/apps/decodex/src/state/internal.rs @@ -336,7 +336,10 @@ CREATE TABLE IF NOT EXISTS worktrees ( issue_id TEXT PRIMARY KEY NOT NULL, project_id TEXT NOT NULL, branch_name TEXT NOT NULL, - worktree_path TEXT NOT NULL + worktree_path TEXT NOT NULL, + provenance_source TEXT NOT NULL DEFAULT 'runtime_recorded', + created_at_unix INTEGER, + updated_at_unix INTEGER ); CREATE TABLE IF NOT EXISTS linear_execution_events ( idempotency_key TEXT PRIMARY KEY NOT NULL, @@ -353,6 +356,7 @@ CREATE INDEX IF NOT EXISTS linear_execution_events_issue_idx ON linear_execution_events (service_id, issue_id, event_unix, recorded_at_unix); "#, )?; + self.bootstrap_worktree_schema()?; self.bootstrap_review_schema()?; self.bootstrap_run_control_channels_schema()?; self.bootstrap_connector_backoffs_schema()?; @@ -365,6 +369,26 @@ ON linear_execution_events (service_id, issue_id, event_unix, recorded_at_unix); Ok(()) } + fn bootstrap_worktree_schema(&self) -> Result<()> { + self.ensure_column( + "worktrees", + "provenance_source", + "ALTER TABLE worktrees ADD COLUMN provenance_source TEXT NOT NULL DEFAULT 'legacy_unknown'", + )?; + self.ensure_column( + "worktrees", + "created_at_unix", + "ALTER TABLE worktrees ADD COLUMN created_at_unix INTEGER", + )?; + self.ensure_column( + "worktrees", + "updated_at_unix", + "ALTER TABLE worktrees ADD COLUMN updated_at_unix INTEGER", + )?; + + Ok(()) + } + fn bootstrap_review_schema(&self) -> Result<()> { self.connection.execute_batch( r#" @@ -768,13 +792,19 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; let transaction = self.connection.transaction()?; transaction.execute( - "INSERT OR REPLACE INTO worktrees (issue_id, project_id, branch_name, worktree_path) - VALUES (?1, ?2, ?3, ?4)", + "INSERT OR REPLACE INTO worktrees ( + issue_id, project_id, branch_name, worktree_path, + provenance_source, created_at_unix, updated_at_unix + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", params![ &mapping.issue_id, &mapping.project_id, &mapping.branch_name, mapping.worktree_path.to_string_lossy().as_ref(), + &mapping.provenance_source, + mapping.created_at_unix, + mapping.updated_at_unix, ], )?; @@ -944,8 +974,13 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; )?; transaction.execute("DELETE FROM leases WHERE issue_id = ?1", params![previous_issue_id])?; transaction.execute( - "INSERT OR IGNORE INTO worktrees (issue_id, project_id, branch_name, worktree_path) - SELECT ?2, project_id, branch_name, worktree_path FROM worktrees WHERE issue_id = ?1", + "INSERT OR IGNORE INTO worktrees ( + issue_id, project_id, branch_name, worktree_path, + provenance_source, created_at_unix, updated_at_unix + ) + SELECT ?2, project_id, branch_name, worktree_path, + provenance_source, created_at_unix, updated_at_unix + FROM worktrees WHERE issue_id = ?1", params![previous_issue_id, canonical_issue_id], )?; transaction.execute("DELETE FROM worktrees WHERE issue_id = ?1", params![previous_issue_id])?; @@ -1471,7 +1506,11 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; fn load_worktrees(&self, state: &mut StateData) -> Result<()> { let mut statement = self .connection - .prepare("SELECT issue_id, project_id, branch_name, worktree_path FROM worktrees")?; + .prepare( + "SELECT issue_id, project_id, branch_name, worktree_path, + provenance_source, created_at_unix, updated_at_unix + FROM worktrees", + )?; let rows = statement.query_map([], |row| { let issue_id: String = row.get(0)?; @@ -1482,6 +1521,9 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; project_id: row.get(1)?, branch_name: row.get(2)?, worktree_path: PathBuf::from(row.get::<_, String>(3)?), + provenance_source: row.get(4)?, + created_at_unix: row.get(5)?, + updated_at_unix: row.get(6)?, }, )) })?; @@ -2183,6 +2225,9 @@ struct WorktreeMappingRecord { issue_id: String, branch_name: String, worktree_path: PathBuf, + provenance_source: String, + created_at_unix: Option, + updated_at_unix: Option, } impl WorktreeMappingRecord { fn as_public(&self) -> WorktreeMapping { @@ -2191,6 +2236,11 @@ impl WorktreeMappingRecord { issue_id: self.issue_id.clone(), branch_name: self.branch_name.clone(), worktree_path: self.worktree_path.clone(), + provenance: worktree_provenance( + self.provenance_source.clone(), + self.created_at_unix, + self.updated_at_unix, + ), } } } @@ -3131,13 +3181,18 @@ fn persist_protocol_events( fn persist_worktrees(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { for mapping in state.worktrees.values() { transaction.execute( - "INSERT OR REPLACE INTO worktrees (issue_id, project_id, branch_name, worktree_path) \ - VALUES (?1, ?2, ?3, ?4)", + "INSERT OR REPLACE INTO worktrees ( + issue_id, project_id, branch_name, worktree_path, + provenance_source, created_at_unix, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", params![ &mapping.issue_id, &mapping.project_id, &mapping.branch_name, mapping.worktree_path.to_string_lossy().as_ref(), + &mapping.provenance_source, + mapping.created_at_unix, + mapping.updated_at_unix, ], )?; } diff --git a/apps/decodex/src/state/models.rs b/apps/decodex/src/state/models.rs index 3a61e7b10..9c0476c0e 100644 --- a/apps/decodex/src/state/models.rs +++ b/apps/decodex/src/state/models.rs @@ -1,3 +1,9 @@ +pub(crate) const WORKTREE_PROVENANCE_FILESYSTEM_SCAN: &str = "filesystem_scan"; +pub(crate) const WORKTREE_PROVENANCE_GIT_HYGIENE_SCAN: &str = "git_hygiene_scan"; +pub(crate) const WORKTREE_PROVENANCE_LEGACY_UNKNOWN: &str = "legacy_unknown"; +pub(crate) const WORKTREE_PROVENANCE_RUNTIME_RECOVERED: &str = "runtime_recovered"; +pub(crate) const WORKTREE_PROVENANCE_RUNTIME_RECORDED: &str = "runtime_recorded"; + /// Active lease for one issue. #[derive(Clone, Debug, Eq, PartialEq)] pub struct IssueLease { @@ -405,6 +411,7 @@ pub struct WorktreeMapping { issue_id: String, branch_name: String, worktree_path: PathBuf, + provenance: WorktreeProvenance, } impl WorktreeMapping { /// Local project identifier owning this lane. @@ -426,6 +433,40 @@ impl WorktreeMapping { pub fn worktree_path(&self) -> &Path { &self.worktree_path } + + /// Durable provenance captured when Decodex recorded or migrated this mapping. + pub fn provenance(&self) -> &WorktreeProvenance { + &self.provenance + } +} + +/// Durable provenance for a retained worktree mapping. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WorktreeProvenance { + source: String, + created_at_unix: Option, + updated_at_unix: Option, +} +impl WorktreeProvenance { + /// Source that created or last classified this mapping. + pub fn source(&self) -> &str { + &self.source + } + + /// Unix timestamp for when this mapping was first recorded, when available. + pub fn created_at_unix(&self) -> Option { + self.created_at_unix + } + + /// Unix timestamp for when this mapping was last refreshed, when available. + pub fn updated_at_unix(&self) -> Option { + self.updated_at_unix + } + + /// Whether this mapping was migrated from a legacy row without durable provenance. + pub fn is_legacy_unknown(&self) -> bool { + self.source == WORKTREE_PROVENANCE_LEGACY_UNKNOWN + } } /// Project-scoped external connector backoff retained in the runtime store. @@ -1299,3 +1340,11 @@ impl ReviewOrchestrationMarker { self.auto_merge_enabled_at_unix_epoch } } + +pub(crate) fn worktree_provenance( + source: impl Into, + created_at_unix: Option, + updated_at_unix: Option, +) -> WorktreeProvenance { + WorktreeProvenance { source: source.into(), created_at_unix, updated_at_unix } +} diff --git a/apps/decodex/src/state/store.rs b/apps/decodex/src/state/store.rs index de2906044..00ecb2e0b 100644 --- a/apps/decodex/src/state/store.rs +++ b/apps/decodex/src/state/store.rs @@ -1848,13 +1848,65 @@ impl StateStore { branch_name: &str, worktree_path: &str, ) -> Result<()> { + let mut state = self.lock_without_refresh()?; + let now_unix = OffsetDateTime::now_utc().unix_timestamp(); + let created_at_unix = + state.worktrees.get(issue_id).and_then(|mapping| mapping.created_at_unix).or(Some(now_unix)); let mapping = WorktreeMappingRecord { project_id: project_id.to_owned(), issue_id: issue_id.to_owned(), branch_name: branch_name.to_owned(), worktree_path: PathBuf::from(worktree_path), + provenance_source: WORKTREE_PROVENANCE_RUNTIME_RECORDED.to_owned(), + created_at_unix, + updated_at_unix: Some(now_unix), }; + + state.worktrees.insert(issue_id.to_owned(), mapping.clone()); + state.remember_run_project(project_id, issue_id, None); + + self.upsert_worktree_and_remember_run_project_locked(&mapping) + } + + /// Create or refresh a worktree mapping reconstructed from retained local state. + pub(crate) fn upsert_recovered_worktree( + &self, + project_id: &str, + issue_id: &str, + branch_name: &str, + worktree_path: &str, + observed_at_unix: Option, + ) -> Result<()> { let mut state = self.lock_without_refresh()?; + let existing = state.worktrees.get(issue_id); + let existing_provenance_source = + existing.map(|mapping| mapping.provenance_source.as_str()); + let provenance_source = + match existing_provenance_source { + Some(WORKTREE_PROVENANCE_RUNTIME_RECORDED) => + WORKTREE_PROVENANCE_RUNTIME_RECORDED, + Some(WORKTREE_PROVENANCE_RUNTIME_RECOVERED) => + WORKTREE_PROVENANCE_RUNTIME_RECOVERED, + _ => WORKTREE_PROVENANCE_RUNTIME_RECOVERED, + } + .to_owned(); + let existing_created_at_unix = existing.and_then(|mapping| mapping.created_at_unix); + let existing_updated_at_unix = existing.and_then(|mapping| mapping.updated_at_unix); + let created_at_unix = existing_created_at_unix.or(observed_at_unix); + let updated_at_unix = match (existing_updated_at_unix, observed_at_unix) { + (Some(existing), Some(observed)) => Some(existing.max(observed)), + (Some(existing), None) => Some(existing), + (None, observed) => observed, + }; + let mapping = WorktreeMappingRecord { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + branch_name: branch_name.to_owned(), + worktree_path: PathBuf::from(worktree_path), + provenance_source, + created_at_unix, + updated_at_unix, + }; state.worktrees.insert(issue_id.to_owned(), mapping.clone()); state.remember_run_project(project_id, issue_id, None); diff --git a/apps/decodex/src/state/tests.rs b/apps/decodex/src/state/tests.rs index e18ef5e6c..eb55cfd81 100644 --- a/apps/decodex/src/state/tests.rs +++ b/apps/decodex/src/state/tests.rs @@ -2204,6 +2204,9 @@ fn manages_worktree_mappings() { assert_eq!(mapping.branch_name(), "x/pub-101"); assert_eq!(mapping.worktree_path(), Path::new("/tmp/worktrees/pub-101")); assert_eq!(mapping.project_id(), "pubfi"); + assert_eq!(mapping.provenance().source(), "runtime_recorded"); + assert!(mapping.provenance().created_at_unix().is_some()); + assert!(mapping.provenance().updated_at_unix().is_some()); assert_eq!(store.list_worktrees("pubfi").expect("list should succeed").len(), 1); store.clear_worktree("PUB-101").expect("mapping should be deleted"); @@ -2211,6 +2214,39 @@ fn manages_worktree_mappings() { assert!(store.worktree_for_issue("PUB-101").expect("lookup should succeed").is_none()); } +#[test] +fn opens_legacy_worktree_rows_with_unknown_provenance() { + let temp_dir = TempDir::new().expect("temp dir should create"); + let db_path = temp_dir.path().join("runtime.sqlite3"); + + { + let connection = Connection::open(&db_path).expect("legacy db should open"); + + connection + .execute_batch( + "CREATE TABLE worktrees ( + issue_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL, + branch_name TEXT NOT NULL, + worktree_path TEXT NOT NULL + ); + INSERT INTO worktrees (issue_id, project_id, branch_name, worktree_path) + VALUES ('issue-legacy', 'pubfi', 'x/pubfi-pub-101', '/tmp/worktrees/pub-101');", + ) + .expect("legacy worktree row should write"); + } + + let store = StateStore::open(&db_path).expect("state store should migrate"); + let mapping = store + .worktree_for_issue("issue-legacy") + .expect("mapping lookup should succeed") + .expect("legacy mapping should exist"); + + assert_eq!(mapping.provenance().source(), "legacy_unknown"); + assert_eq!(mapping.provenance().created_at_unix(), None); + assert_eq!(mapping.provenance().updated_at_unix(), None); +} + #[test] fn persistent_clear_worktree_deletes_review_markers() { let temp_dir = TempDir::new().expect("tempdir should create"); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 35325d5bc..a7631d68d 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -415,13 +415,23 @@ Worktree visibility follows the owning dashboard section: runtime owner is gone or cannot explain it as active, review/landing, or queued work. -Every operator snapshot worktree row includes an `ownership` and `ownership_reason` -that distinguishes active-lane ownership, post-review ownership, queued attention, and -cleanup-only local retention. A `Recovery Worktrees` row tells the operator to inspect -the local path and either clean it up or recover local-only changes; it is not, by -itself, evidence that the SQLite runtime store lost an active lane. When the tracker -issue is already `Done` and no retained lane owns the worktree, the row is neutral -cleanup-only state, not a blocking recovery error. +Every operator snapshot worktree row includes `ownership`, `ownership_reason`, +`provenance`, and optional `recovery_next_action` fields that distinguish active-lane +ownership, post-review ownership, queued attention, post-land cleanup, and cleanup-only +local retention. Runtime-recorded mappings report `provenance.source = +"runtime_recorded"` with created and refreshed Unix timestamps. Deterministically +rebuilt mappings report `provenance.source = "runtime_recovered"` when tracker, +retained marker, or closeout evidence proves a current owner after local state was +missing. Filesystem-only scans use scan-specific provenance such as `filesystem_scan` +or `git_hygiene_scan`. Rows migrated from older runtime stores that had no provenance +report `provenance.source = "legacy_unknown"` and may set +`provenance.audit_required = true`. + +A `Recovery Worktrees` row tells the operator to inspect the local path and either +clean it up or recover local-only changes; it is not, by itself, evidence that the +SQLite runtime store lost an active lane. When the tracker issue is already `Done`, +the row has runtime provenance, and no retained lane owns the worktree, the row is +neutral cleanup-only state, not a blocking recovery error. When a retained worktree reports `role: cleanup_only`, treat it as local cleanup hygiene rather than an active lane. It does not imply that an agent, child @@ -432,6 +442,16 @@ operator verifies the issue or PR is terminal, `main` contains the intended work and the checkout has no local-only changes that need recovery, the safe action is to remove that local worktree. +If that same cleanup-only row reports `provenance.source = "legacy_unknown"` and +`audit_required = true`, treat it as a legacy orphan cleanup decision rather than +ordinary hygiene. Decodex is explicitly saying it cannot prove PR or closeout lineage +from durable runtime records. Do not use review-handoff rebind for this state unless a +separate diagnosis finds an open PR lane with exact lineage. Verify the tracker issue +and PR terminal state, inspect the checkout, run +`decodex recover legacy-closeout --pr --dry-run`, rerun with +`--manual-authority` only after validation passes, and then remove the local worktree +only after that evidence is understood. + The expected operator path for a cleanup-only row is short: 1. Verify the tracker issue and any associated PR are merged, done, or otherwise diff --git a/docs/runbook/recover-review-handoff.md b/docs/runbook/recover-review-handoff.md index 3532677f4..559e08e39 100644 --- a/docs/runbook/recover-review-handoff.md +++ b/docs/runbook/recover-review-handoff.md @@ -84,6 +84,32 @@ The lane should leave `missing_review_handoff_record` and return to the existing post-review lifecycle classification such as waiting for review, ready to land, review repair required, or blocked for a different concrete reason. +## Legacy Cleanup-Only Rows + +Do not use `recover review-handoff rebind` for a row that appears only under +`Recovery Worktrees` as `role: cleanup_only`. If that row reports +`provenance_source: legacy_unknown`, `audit_required: true`, or a dashboard +`legacy cleanup audit` state, Decodex has found an old local worktree mapping without +enough runtime DB provenance to reconstruct the post-review lane automatically. + +Use the fallback path only after the normal paths are unavailable: + +1. Confirm the same issue is absent from `Running Lanes`, `Intake Queue`, and + `Review & Landing`. +2. Verify the tracker issue and any PR you are closing against are terminal. +3. Inspect the retained checkout with `git -C status --short` and preserve + or discard local-only changes intentionally. +4. Run `decodex recover legacy-closeout --pr --dry-run`, then rerun + with `--manual-authority` only if validation passes. This records an explicit + manual closeout audit that names the issue, PR, branch, head, merge commit, and why + runtime reconstruction was not available. +5. Remove the local worktree only after the audit and local-change decision are done. + +This fallback is intentionally more manual than diagnosis or rebind. It exists so an +operator can close legacy residue honestly, but healthy lanes should still use normal +closeout, and recoverable orphaned review lanes should still use read-only diagnosis +plus explicit rebind before any manual cleanup. + ## Active Ownership Recovery If diagnosis reports `classification: review_handoff_ownership_drift`, diff --git a/docs/spec/post-review-lifecycle.md b/docs/spec/post-review-lifecycle.md index ce5803183..a1d11dc4e 100644 --- a/docs/spec/post-review-lifecycle.md +++ b/docs/spec/post-review-lifecycle.md @@ -123,6 +123,19 @@ success path. path. If any audit write fails after marker creation, the command must clear the new markers and report failure instead of leaving a silently rebound lane. +`cleanup_only` rows are outside this rebind surface. When operator status reports a +cleanup-only worktree with `provenance_source = "legacy_unknown"`, Decodex has only an +old local mapping and cannot prove PR or closeout lineage from the runtime store. The +operator path is: verify the tracker issue and PR terminal state, inspect the retained +checkout for local-only changes, run +`decodex recover legacy-closeout --pr --dry-run`, rerun with +`--manual-authority` only after validation passes, and only then remove the worktree. +That fallback must stay rarer than normal closeout, explicit rebind, or deterministic +legacy reconstruction from authoritative markers. Runtime recovery may classify a +retained worktree as `runtime_recovered` only after tracker, retained marker, or +closeout evidence proves a current owner; it must not silently upgrade a terminal +cleanup-only `legacy_unknown` row. + ## Phase model The post-`In Review` lifecycle is expressed in lane phases. These phases refine, but do not replace, the owned-lane action classes. diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 740f8b9a7..5a4e46c97 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -650,12 +650,21 @@ After a process restart, recent-run history, active lease ownership, retained po while polling. That subset may rotate oversized local files, prune old backups, and run a passive WAL checkpoint, but it must not compact runtime protocol events. - Worktrees: retain while the issue is non-terminal, and also retain terminal owned lanes while authoritative post-merge closeout or deterministic cleanup is still incomplete. +- Worktree mappings must carry durable local provenance. New runtime-recorded mappings + use `provenance_source = "runtime_recorded"` with created and updated Unix + timestamps. Mappings reconstructed from retained tracker/worktree/marker evidence + after local runtime state is missing use `provenance_source = "runtime_recovered"`; + they are recoverable runtime state, but not proof that the original runtime row was + still present. Rows migrated from older runtime stores that lack this information + must remain readable but must be classified as `provenance_source = + "legacy_unknown"` instead of being silently treated as a fully proven runtime-owned + lane. - Terminal issue cleanup: once the issue reaches a terminal tracker state and no authoritative post-merge tail remains pending, remove the worktree during reconciliation or startup cleanup. - If an issue becomes non-terminal but no longer eligible while `decodex` is still preparing the lane, keep the worktree and skip execution for that pass. ## Recovery rules -- On service startup, `decodex` must inspect deterministic `.worktrees/` paths together with tracker issue ids already known from local leases or worktree mappings to rebuild retained worktree mappings before starting new work. +- On service startup, `decodex` must inspect deterministic `.worktrees/` paths together with tracker issue ids already known from local leases or worktree mappings to rebuild retained worktree mappings before starting new work. This recovery must only write a recovered worktree mapping after tracker, retained marker, or closeout evidence proves that retry, active-lane, or post-review closeout state owns the worktree; a terminal cleanup-only legacy row must keep `provenance_source = "legacy_unknown"` until the operator runs the explicit legacy closeout audit path. - If Linear still shows a non-terminal `In Progress` issue and its retained worktree exists locally, `decodex` must treat that lane as a retry-style recovery candidate before selecting fresh `Todo` work. - Retry recovery must bind retained lanes to issue identity and local runtime state rather than to Linear project membership. - While the control plane is running an active lane, every poll tick must refresh cached tracker state for the leased issue before considering any new selection. @@ -685,6 +694,10 @@ After a process restart, recent-run history, active lease ownership, retained po - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. - If retry budget or needs-attention recovery finds tracked changes in the retained worktree, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. - If Linear still has `decodex:active:` on an issue that also remains queued, but the local runtime cannot prove a matching active lease, status must classify the queued row as blocked with reason `linear_active_label_present`; it must not treat the issue as ready intake. If the retained marker or private execution event rows for that run are missing, status must surface `evidence_missing` in the recovery details. If the retained worktree has tracked changes, that dirty worktree remains owned by queued recovery/attention instead of being hidden as cleanup-only state. +- Operator status snapshots must expose worktree provenance in both JSON and human text + output. A cleanup-only worktree with `provenance_source = "legacy_unknown"` must set + `audit_required = true` and provide a `decodex recover legacy-closeout` next action; + this is a last-resort operator audit path, not an automatic rebind or cleanup signal. - During an active run, operator snapshots must expose `thread_id` as soon as the Codex thread exists, plus monotonically advancing `event_count`, `last_event_type`, and `last_event_at` once protocol events are recorded. These fields may be hydrated either from the current process journal or from the active lane's `.decodex-run-activity` marker when `status` is running in a separate process. - `thread_id = null` is expected only before the worker creates the Codex thread for the current run. `event_count = 0`, `last_event_type = null`, and `last_event_at = null` are expected only before the first protocol event for that same run. After the thread exists and protocol activity has started, those empty values indicate missing hydration rather than normal progress. - Operator snapshots may expose an additive `protocol_activity` object derived from app-server structured messages for the current run. The object stays local/operator-only and should summarize turn status, waiting reason, rate-limit status, and a compact recent event list for high-value app-server activity such as `turn/started`, `turn/completed`, plan updates, diff updates, item start/completion, command output deltas, server request responses, account updates, and rate-limit updates. Missing `protocol_activity` means no structured summary was captured yet; consumers must continue to rely on the older `event_count`, `last_event_type`, `last_event_at`, thread fields, and `child_agent_activity` fields when it is absent.