diff --git a/README.md b/README.md index 4a42c68fc..cbcf4b40c 100644 --- a/README.md +++ b/README.md @@ -309,7 +309,7 @@ The governing workflow lives at `docs/runbook/local-github-signal-workflow.md`. ## Operator Dashboard `decodex serve` owns the local operator listener. It serves the operator dashboard from -`GET /` and `GET /dashboard`; published snapshots, active-run updates, and local +`GET /` and `GET /dashboard`; published snapshots, current-lane updates, and local dashboard controls flow through the `/dashboard/control` WebSocket. The HTTP surface is kept to dashboard pages/assets, `GET /livez`, and the local account-control API used by Decodex App. diff --git a/apps/decodex-app/Sources/DecodexApp/AccountPanelView.swift b/apps/decodex-app/Sources/DecodexApp/AccountPanelView.swift index d1c3fe792..7b7603c22 100644 --- a/apps/decodex-app/Sources/DecodexApp/AccountPanelView.swift +++ b/apps/decodex-app/Sources/DecodexApp/AccountPanelView.swift @@ -669,7 +669,7 @@ struct AccountPanelView: View { } private func operatorRuns(for account: CodexAccount) -> [OperatorRunStatus] { - store.operatorSnapshot?.activeRuns(for: account) ?? [] + store.operatorSnapshot?.currentLanes(for: account) ?? [] } private func accountRowHeight(for account: CodexAccount) -> CGFloat { @@ -951,7 +951,7 @@ struct AccountRunSummaryView: View { .frame(height: AccountRunChipLayout.height) .frame(maxWidth: .infinity, alignment: .leading) .contentShape(Rectangle()) - .accessibilityLabel("\(runs.count) active lane\(runs.count == 1 ? "" : "s")") + .accessibilityLabel("\(runs.count) current lane\(runs.count == 1 ? "" : "s")") .onAppear { placementStore.retainOnly(Set(runs.map(\.id))) } @@ -1870,6 +1870,16 @@ struct AccountRunChipView: View { .lineLimit(1) .truncationMode(.middle) .fixedSize(horizontal: true, vertical: false) + + if let modelProgress { + Text("\(modelProgress.percent)%") + .font(PanelFont.usageLabel) + .foregroundStyle(PanelPalette.secondaryText(colorScheme).opacity(0.86)) + .monospacedDigit() + .lineLimit(1) + .fixedSize(horizontal: true, vertical: false) + .help("\(modelProgress.title) \(modelProgress.percent)%") + } } .frame(height: AccountRunChipLayout.height) .padding(.horizontal, AccountRunChipLayout.horizontalPadding) @@ -1933,6 +1943,10 @@ struct AccountRunChipView: View { panelTrimmed(run.issueIdentifier) ?? "Run" } + private var modelProgress: OperatorModelProgressReadout? { + operatorModelProgressReadout(for: run, currentTime: currentTime) + } + private var tint: Color { if run.hasAttentionTone { return PanelPalette.warning(colorScheme) @@ -2889,6 +2903,12 @@ struct OperatorLanePopoverView: View { ) } } + + if let continuationRecoveryReadout { + measuredReadout { + OperatorLaneReadoutRow(title: "Recovery", items: continuationRecoveryReadout) + } + } } if modelProgress != nil, @@ -2980,6 +3000,30 @@ struct OperatorLanePopoverView: View { panelTrimmed(run.projectDisplayName) ?? panelTrimmed(run.projectID) } + private var continuationRecoveryReadout: [OperatorLaneReadoutItem]? { + guard let recovery = run.continuationRecovery else { + return nil + } + + let state = rawPanelToken(recovery.state ?? "continuation") + let phase = [ + recovery.sourcePhase.map(rawPanelToken), + recovery.nextPhase.map(rawPanelToken), + ] + .compactMap { $0 } + .joined(separator: " -> ") + let count = "\(recovery.recoveryCount ?? 0)/\(recovery.automaticContinuationLimit ?? 0)" + let budget = recovery.budgetExceeded == true ? "exceeded" : "within" + let error = rawPanelToken(recovery.sourceErrorClass ?? "unknown") + + return [ + OperatorLaneReadoutItem(label: "state", value: state), + OperatorLaneReadoutItem(label: "phase", value: phase.isEmpty ? "unknown" : phase), + OperatorLaneReadoutItem(label: "count", value: "\(count) \(budget)"), + OperatorLaneReadoutItem(label: "error", value: error), + ] + } + private var alignedWidth: CGFloat? { readoutWidth > 0 ? readoutWidth : nil } @@ -3047,44 +3091,8 @@ struct OperatorLanePopoverView: View { return items } - private var modelBucket: OperatorChildAgentBucket? { - return orderedBuckets.first { bucket in - bucket.name.caseInsensitiveCompare("Model") == .orderedSame - } - } - private var modelProgress: OperatorModelProgressReadout? { - if let lifecycleMetrics = run.lifecycleMetrics, - let modelSeconds = lifecycleModelSeconds(lifecycleMetrics.buckets) - { - let totalSeconds = max( - 1, - lifecycleMetrics.wallSeconds, - lifecycleMetrics.buckets.reduce(0) { $0 + max(0, $1.wallSeconds) }, - modelSeconds - ) - let share = CGFloat(modelSeconds) / CGFloat(totalSeconds) - - return OperatorModelProgressReadout( - title: "Inference", - percent: Int((Double(modelSeconds) / Double(totalSeconds) * 100).rounded()), - elapsed: formatActivityDuration(modelSeconds) ?? "0s", - total: formatActivityDuration(totalSeconds) ?? "0s", - barShare: min(1, max(0.02, share)) - ) - } - - guard let modelBucket else { - return nil - } - - return OperatorModelProgressReadout( - title: "Inference", - percent: bucketPercent(modelBucket), - elapsed: formatActivityDuration(bucketWallSeconds(modelBucket)) ?? "0s", - total: formatActivityDuration(totalWallSeconds) ?? "0s", - barShare: bucketShare(modelBucket) - ) + operatorModelProgressReadout(for: run, currentTime: currentTime) } private var detailBuckets: [OperatorChildAgentBucket] { @@ -3380,20 +3388,6 @@ struct OperatorLanePopoverView: View { return 10 } - private func bucketShare(_ bucket: OperatorChildAgentBucket) -> CGFloat { - let wallSeconds = bucketWallSeconds(bucket) - - guard wallSeconds > 0 else { - return 0 - } - - return min(1, max(0.02, CGFloat(wallSeconds) / CGFloat(max(1, totalWallSeconds)))) - } - - private func bucketPercent(_ bucket: OperatorChildAgentBucket) -> Int { - Int((Double(bucketWallSeconds(bucket)) / Double(max(1, totalWallSeconds)) * 100).rounded()) - } - private func bucketWallSeconds(_ bucket: OperatorChildAgentBucket) -> Int { activity?.wallSeconds(for: bucket, at: currentTime) ?? bucket.wallSeconds } @@ -3434,6 +3428,68 @@ struct OperatorModelProgressReadout { let barShare: CGFloat } +fileprivate func operatorModelProgressReadout( + for run: OperatorRunStatus, + currentTime: Date +) -> OperatorModelProgressReadout? { + if let lifecycleMetrics = run.lifecycleMetrics, + let modelSeconds = operatorLifecycleModelSeconds(lifecycleMetrics.buckets) + { + let totalSeconds = max( + 1, + lifecycleMetrics.wallSeconds, + lifecycleMetrics.buckets.reduce(0) { $0 + max(0, $1.wallSeconds) }, + modelSeconds + ) + let share = CGFloat(modelSeconds) / CGFloat(totalSeconds) + + return OperatorModelProgressReadout( + title: "Inference", + percent: Int((Double(modelSeconds) / Double(totalSeconds) * 100).rounded()), + elapsed: formatActivityDuration(modelSeconds) ?? "0s", + total: formatActivityDuration(totalSeconds) ?? "0s", + barShare: min(1, max(0.02, share)) + ) + } + + guard let activity = run.childAgentActivity, + let modelBucket = activity.buckets.first(where: { bucket in + bucket.name.caseInsensitiveCompare("Model") == .orderedSame + }) + else { + return nil + } + + let totalSeconds = max( + 1, + activity.wallSeconds(at: currentTime), + activity.buckets.reduce(0) { total, bucket in + total + max(0, activity.wallSeconds(for: bucket, at: currentTime)) + } + ) + let modelSeconds = activity.wallSeconds(for: modelBucket, at: currentTime) + guard modelSeconds > 0 else { + return nil + } + let share = CGFloat(modelSeconds) / CGFloat(totalSeconds) + + return OperatorModelProgressReadout( + title: "Inference", + percent: Int((Double(modelSeconds) / Double(totalSeconds) * 100).rounded()), + elapsed: formatActivityDuration(modelSeconds) ?? "0s", + total: formatActivityDuration(totalSeconds) ?? "0s", + barShare: min(1, max(0.02, share)) + ) +} + +fileprivate func operatorLifecycleModelSeconds( + _ buckets: [OperatorLifecycleMetricBucket] +) -> Int? { + buckets.first { bucket in + bucket.name.caseInsensitiveCompare("Model") == .orderedSame + }?.wallSeconds +} + struct OperatorTotalMetric: Identifiable { let title: String let items: [OperatorLaneReadoutItem] diff --git a/apps/decodex-app/Sources/DecodexApp/AccountStore.swift b/apps/decodex-app/Sources/DecodexApp/AccountStore.swift index 587f49244..03e8926eb 100644 --- a/apps/decodex-app/Sources/DecodexApp/AccountStore.swift +++ b/apps/decodex-app/Sources/DecodexApp/AccountStore.swift @@ -222,19 +222,19 @@ final class AccountStore: ObservableObject { operatorSnapshot = liveRunActivity?.merging(into: snapshot) ?? snapshot operatorSnapshotUpdatedAt = payload.snapshotPublishedAt ?? Date() case "runActivity": - guard let activeRuns = payload.activeRuns else { + guard let currentLanes = payload.currentLanes else { return } let activity = OperatorRunActivitySnapshot( - activeRuns: activeRuns, - activeRunsComplete: payload.activeRunsComplete ?? true, + currentLanes: currentLanes, + currentLanesComplete: payload.currentLanesComplete ?? true, emittedAt: payload.emittedAt ?? Date() ) if let operatorSnapshot { self.operatorSnapshot = activity.merging(into: operatorSnapshot) } else { - operatorSnapshot = OperatorSnapshotResponse.activeRunsOnly(activeRuns) + operatorSnapshot = OperatorSnapshotResponse.currentLanesOnly(currentLanes) } liveRunActivity = activity.shouldPersistAsSnapshotOverlay ? activity : nil operatorSnapshotUpdatedAt = activity.emittedAt diff --git a/apps/decodex-app/Sources/DecodexApp/OperatorSnapshotModels.swift b/apps/decodex-app/Sources/DecodexApp/OperatorSnapshotModels.swift index ec5c455c3..502c70e74 100644 --- a/apps/decodex-app/Sources/DecodexApp/OperatorSnapshotModels.swift +++ b/apps/decodex-app/Sources/DecodexApp/OperatorSnapshotModels.swift @@ -1,24 +1,24 @@ import Foundation -private let operatorActiveRunIdleTimeoutSeconds = 300 +private let operatorCurrentLaneIdleTimeoutSeconds = 300 struct OperatorSnapshotResponse: Decodable, Sendable { let warnings: [String] let projects: [OperatorProjectStatus] - let activeRuns: [OperatorRunStatus] + let currentLanes: [OperatorRunStatus] let queuedCandidates: [OperatorQueuedIssueStatus] let postReviewLanes: [OperatorPostReviewLaneStatus] - var activeRunCount: Int { + var currentLaneCount: Int { max( - activeRuns.count, - projects.reduce(0) { $0 + $1.activeRunCount } + currentLanes.count, + projects.reduce(0) { $0 + $1.currentLaneCount } ) } var runningLaneCount: Int { max( - activeRuns.filter(\.countsAsRunning).count, + currentLanes.filter(\.countsAsRunning).count, projects.reduce(0) { $0 + $1.runningLaneCount } ) } @@ -32,13 +32,13 @@ struct OperatorSnapshotResponse: Decodable, Sendable { var reviewCount: Int { max( - postReviewLanes.filter { $0.shadowedByActiveRun == false }.count, + postReviewLanes.filter { $0.shadowedByCurrentLane == false }.count, projects.reduce(0) { $0 + $1.postReviewLaneCount } ) } var landingCount: Int { - postReviewLanes.filter { $0.isReadyToLand && $0.shadowedByActiveRun == false }.count + postReviewLanes.filter { $0.isReadyToLand && $0.shadowedByCurrentLane == false }.count } var waitingCount: Int { @@ -54,7 +54,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { } var hasVisibleSignal: Bool { - activeRunCount > 0 + currentLaneCount > 0 || queuedCount > 0 || waitingCount > 0 || attentionCount > 0 @@ -63,7 +63,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { } var shouldDisplayInPanel: Bool { - hasVisibleSignal && (activeRunCount > 0 || isDevSnapshot == false) + hasVisibleSignal && (currentLaneCount > 0 || isDevSnapshot == false) } var warningSummary: String? { @@ -81,29 +81,29 @@ struct OperatorSnapshotResponse: Decodable, Sendable { return "\(first) +\(labels.count - 1)" } - func activeRuns(for account: CodexAccount) -> [OperatorRunStatus] { - activeRuns.filter { $0.isAssigned(to: account) } + func currentLanes(for account: CodexAccount) -> [OperatorRunStatus] { + currentLanes.filter { $0.isAssigned(to: account) } } func runningCount(for account: CodexAccount) -> Int { - activeRuns(for: account).filter(\.countsAsRunning).count + currentLanes(for: account).filter(\.countsAsRunning).count } func mergingRunActivity( _ activityRuns: [OperatorRunStatus], - activeRunsComplete: Bool = true + currentLanesComplete: Bool = true ) -> OperatorSnapshotResponse { let activityRunsByID = activityRuns.reduce(into: [String: OperatorRunStatus]()) { runsByID, run in runsByID[run.runID] = run } - let snapshotRunsByID = activeRuns.reduce(into: [String: OperatorRunStatus]()) { runsByID, run in + let snapshotRunsByID = currentLanes.reduce(into: [String: OperatorRunStatus]()) { runsByID, run in runsByID[run.runID] = run } - let mergedSnapshotRuns = activeRuns.compactMap { snapshotRun -> OperatorRunStatus? in + let mergedSnapshotRuns = currentLanes.compactMap { snapshotRun -> OperatorRunStatus? in if let activityRun = activityRunsByID[snapshotRun.runID] { return snapshotRun.mergingActivity(activityRun) } - if activeRunsComplete { + if currentLanesComplete { return nil } @@ -113,7 +113,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { snapshotRunsByID[activityRun.runID] == nil } let mergedRuns = mergedSnapshotRuns + newActivityRuns - let activeCountsByProject = Dictionary(grouping: mergedRuns.compactMap(\.projectID)) { $0 } + let currentLaneCountsByProject = Dictionary(grouping: mergedRuns.compactMap(\.projectID)) { $0 } .mapValues(\.count) let runningCountsByProject = Dictionary( grouping: mergedRuns.filter(\.countsAsRunning).compactMap(\.projectID) @@ -125,7 +125,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { } return project.withRunCounts( - active: activeCountsByProject[projectID] ?? 0, + currentLanes: currentLaneCountsByProject[projectID] ?? 0, running: runningCountsByProject[projectID] ?? 0 ) } @@ -133,17 +133,17 @@ struct OperatorSnapshotResponse: Decodable, Sendable { return OperatorSnapshotResponse( warnings: warnings, projects: mergedProjects, - activeRuns: mergedRuns, + currentLanes: mergedRuns, queuedCandidates: queuedCandidates, postReviewLanes: postReviewLanes ) } - static func activeRunsOnly(_ activeRuns: [OperatorRunStatus]) -> OperatorSnapshotResponse { + static func currentLanesOnly(_ currentLanes: [OperatorRunStatus]) -> OperatorSnapshotResponse { OperatorSnapshotResponse( warnings: [], projects: [], - activeRuns: activeRuns, + currentLanes: currentLanes, queuedCandidates: [], postReviewLanes: [] ) @@ -152,13 +152,13 @@ struct OperatorSnapshotResponse: Decodable, Sendable { private init( warnings: [String], projects: [OperatorProjectStatus], - activeRuns: [OperatorRunStatus], + currentLanes: [OperatorRunStatus], queuedCandidates: [OperatorQueuedIssueStatus], postReviewLanes: [OperatorPostReviewLaneStatus] ) { self.warnings = warnings self.projects = projects - self.activeRuns = activeRuns + self.currentLanes = currentLanes self.queuedCandidates = queuedCandidates self.postReviewLanes = postReviewLanes } @@ -171,7 +171,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { enum CodingKeys: String, CodingKey { case warnings case projects - case activeRuns = "active_runs" + case currentLanes = "current_lanes" case queuedCandidates = "queued_candidates" case postReviewLanes = "post_review_lanes" } @@ -181,7 +181,7 @@ struct OperatorSnapshotResponse: Decodable, Sendable { warnings = try container.decodeIfPresent([String].self, forKey: .warnings) ?? [] projects = try container.decodeIfPresent([OperatorProjectStatus].self, forKey: .projects) ?? [] - activeRuns = try container.decodeIfPresent([OperatorRunStatus].self, forKey: .activeRuns) ?? [] + currentLanes = try container.decodeIfPresent([OperatorRunStatus].self, forKey: .currentLanes) ?? [] queuedCandidates = try container.decodeIfPresent( [OperatorQueuedIssueStatus].self, forKey: .queuedCandidates @@ -198,7 +198,7 @@ struct OperatorProjectStatus: Decodable, Sendable { let enabled: Bool let connectorState: String? let warningCount: Int - let activeRunCount: Int + let currentLaneCount: Int let runningLaneCount: Int let queuedCandidateCount: Int let postReviewLaneCount: Int @@ -207,13 +207,13 @@ struct OperatorProjectStatus: Decodable, Sendable { let cleanupBlockedCount: Int let cleanupPendingCount: Int - func withRunCounts(active: Int, running: Int) -> OperatorProjectStatus { + func withRunCounts(currentLanes: Int, running: Int) -> OperatorProjectStatus { OperatorProjectStatus( projectID: projectID, enabled: enabled, connectorState: connectorState, warningCount: warningCount, - activeRunCount: active, + currentLaneCount: currentLanes, runningLaneCount: running, queuedCandidateCount: queuedCandidateCount, postReviewLaneCount: postReviewLaneCount, @@ -229,7 +229,7 @@ struct OperatorProjectStatus: Decodable, Sendable { case enabled case connectorState = "connector_state" case warningCount = "warning_count" - case activeRunCount = "active_run_count" + case currentLaneCount = "current_lane_count" case runningLaneCount = "running_lane_count" case queuedCandidateCount = "queued_candidate_count" case postReviewLaneCount = "post_review_lane_count" @@ -246,9 +246,9 @@ struct OperatorProjectStatus: Decodable, Sendable { enabled = try container.decodeIfPresent(Bool.self, forKey: .enabled) ?? true connectorState = try container.decodeIfPresent(String.self, forKey: .connectorState) warningCount = try container.decodeIfPresent(Int.self, forKey: .warningCount) ?? 0 - activeRunCount = try container.decodeIfPresent(Int.self, forKey: .activeRunCount) ?? 0 + currentLaneCount = try container.decodeIfPresent(Int.self, forKey: .currentLaneCount) ?? 0 runningLaneCount = - try container.decodeIfPresent(Int.self, forKey: .runningLaneCount) ?? activeRunCount + try container.decodeIfPresent(Int.self, forKey: .runningLaneCount) ?? currentLaneCount queuedCandidateCount = try container.decodeIfPresent(Int.self, forKey: .queuedCandidateCount) ?? 0 postReviewLaneCount = try container.decodeIfPresent(Int.self, forKey: .postReviewLaneCount) ?? 0 waitingLaneCount = try container.decodeIfPresent(Int.self, forKey: .waitingLaneCount) ?? 0 @@ -262,7 +262,7 @@ struct OperatorProjectStatus: Decodable, Sendable { enabled: Bool, connectorState: String?, warningCount: Int, - activeRunCount: Int, + currentLaneCount: Int, runningLaneCount: Int, queuedCandidateCount: Int, postReviewLaneCount: Int, @@ -275,7 +275,7 @@ struct OperatorProjectStatus: Decodable, Sendable { self.enabled = enabled self.connectorState = connectorState self.warningCount = warningCount - self.activeRunCount = activeRunCount + self.currentLaneCount = currentLaneCount self.runningLaneCount = runningLaneCount self.queuedCandidateCount = queuedCandidateCount self.postReviewLaneCount = postReviewLaneCount @@ -300,21 +300,21 @@ struct OperatorQueuedIssueStatus: Decodable, Sendable { struct OperatorPostReviewLaneStatus: Decodable, Sendable { let classification: String? - let shadowedByActiveRun: Bool + let shadowedByCurrentLane: Bool var isReadyToLand: Bool { - classification == "ready_to_land" && shadowedByActiveRun == false + classification == "ready_to_land" && shadowedByCurrentLane == false } enum CodingKeys: String, CodingKey { case classification - case shadowedByActiveRun = "shadowed_by_active_run" + case shadowedByCurrentLane = "shadowed_by_current_lane" } init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) classification = try container.decodeIfPresent(String.self, forKey: .classification) - shadowedByActiveRun = try container.decodeIfPresent(Bool.self, forKey: .shadowedByActiveRun) ?? false + shadowedByCurrentLane = try container.decode(Bool.self, forKey: .shadowedByCurrentLane) } } @@ -444,12 +444,13 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { let needsAttentionSnapshot: Bool? let processAlive: Bool? let processLivenessReason: String? - let activeLease: Bool? + let runLease: Bool? let branchName: String? let worktreePath: String? let suspectedStall: Bool let childAgentActivity: OperatorChildAgentActivity? let lifecycleMetrics: OperatorLifecycleMetrics? + let continuationRecovery: OperatorContinuationRecoveryStatus? let account: OperatorRunAccountSummary? let accounts: [OperatorRunAccountSummary] @@ -560,7 +561,7 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { } var shouldRetainDuringPartialRunActivity: Bool { - activeLease == true + runLease == true || processAlive == true || hasRecentAppServerExecution || status == "running" @@ -592,7 +593,7 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { threadStatus == "active" || threadActiveFlags.isEmpty == false || ["thread_active", "protocol_observed"].contains(executionLiveness ?? "") - || protocolIdleForSeconds.isSomeAndLessThan(operatorActiveRunIdleTimeoutSeconds) + || protocolIdleForSeconds.isSomeAndLessThan(operatorCurrentLaneIdleTimeoutSeconds) } private var hasRunningStatus: Bool { @@ -653,12 +654,13 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { needsAttentionSnapshot: activity.needsAttentionSnapshot ?? needsAttentionSnapshot, processAlive: activity.processAlive ?? processAlive, processLivenessReason: activity.processLivenessReason ?? processLivenessReason, - activeLease: activity.activeLease ?? activeLease, + runLease: activity.runLease ?? runLease, branchName: activity.branchName ?? branchName, worktreePath: activity.worktreePath ?? worktreePath, suspectedStall: activity.suspectedStall || suspectedStall, childAgentActivity: activity.childAgentActivity ?? childAgentActivity, lifecycleMetrics: activity.lifecycleMetrics ?? lifecycleMetrics, + continuationRecovery: activity.continuationRecovery ?? continuationRecovery, account: activity.account ?? account, accounts: activity.accounts.isEmpty ? accounts : activity.accounts ) @@ -708,12 +710,13 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { case needsAttentionSnapshot = "needs_attention" case processAlive = "process_alive" case processLivenessReason = "process_liveness_reason" - case activeLease = "active_lease" + case runLease = "run_lease" case branchName = "branch_name" case worktreePath = "worktree_path" case suspectedStall = "suspected_stall" case childAgentActivity = "child_agent_activity" case lifecycleMetrics = "lifecycle_metrics" + case continuationRecovery = "continuation_recovery" case account case accounts case codexAccount = "codex_account" @@ -750,7 +753,7 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { needsAttentionSnapshot = try container.decodeIfPresent(Bool.self, forKey: .needsAttentionSnapshot) processAlive = try container.decodeIfPresent(Bool.self, forKey: .processAlive) processLivenessReason = try container.decodeIfPresent(String.self, forKey: .processLivenessReason) - activeLease = try container.decodeIfPresent(Bool.self, forKey: .activeLease) + runLease = try container.decodeIfPresent(Bool.self, forKey: .runLease) branchName = try container.decodeIfPresent(String.self, forKey: .branchName) worktreePath = try container.decodeIfPresent(String.self, forKey: .worktreePath) suspectedStall = try container.decodeIfPresent(Bool.self, forKey: .suspectedStall) ?? false @@ -759,6 +762,10 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { forKey: .childAgentActivity ) lifecycleMetrics = try container.decodeIfPresent(OperatorLifecycleMetrics.self, forKey: .lifecycleMetrics) + continuationRecovery = try container.decodeIfPresent( + OperatorContinuationRecoveryStatus.self, + forKey: .continuationRecovery + ) account = try container.decodeIfPresent(OperatorRunAccountSummary.self, forKey: .account) ?? container.decodeIfPresent(OperatorRunAccountSummary.self, forKey: .codexAccount) accounts = try container.decodeIfPresent([OperatorRunAccountSummary].self, forKey: .accounts) @@ -794,12 +801,13 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { needsAttentionSnapshot: Bool?, processAlive: Bool?, processLivenessReason: String?, - activeLease: Bool?, + runLease: Bool?, branchName: String?, worktreePath: String?, suspectedStall: Bool, childAgentActivity: OperatorChildAgentActivity?, lifecycleMetrics: OperatorLifecycleMetrics?, + continuationRecovery: OperatorContinuationRecoveryStatus?, account: OperatorRunAccountSummary?, accounts: [OperatorRunAccountSummary] ) { @@ -830,17 +838,48 @@ struct OperatorRunStatus: Decodable, Identifiable, Sendable { self.needsAttentionSnapshot = needsAttentionSnapshot self.processAlive = processAlive self.processLivenessReason = processLivenessReason - self.activeLease = activeLease + self.runLease = runLease self.branchName = branchName self.worktreePath = worktreePath self.suspectedStall = suspectedStall self.childAgentActivity = childAgentActivity self.lifecycleMetrics = lifecycleMetrics + self.continuationRecovery = continuationRecovery self.account = account self.accounts = accounts } } +struct OperatorContinuationRecoveryStatus: Decodable, Sendable { + let state: String? + let sourcePhase: String? + let nextPhase: String? + let sourceErrorClass: String? + let sourceErrorMessage: String? + let recordedAt: String? + let runID: String? + let attemptNumber: Int? + let recoveryCount: Int? + let automaticContinuationLimit: Int? + let budgetExceeded: Bool? + let nextAction: String? + + enum CodingKeys: String, CodingKey { + case state + case sourcePhase = "source_phase" + case nextPhase = "next_phase" + case sourceErrorClass = "source_error_class" + case sourceErrorMessage = "source_error_message" + case recordedAt = "recorded_at" + case runID = "run_id" + case attemptNumber = "attempt_number" + case recoveryCount = "recovery_count" + case automaticContinuationLimit = "automatic_continuation_limit" + case budgetExceeded = "budget_exceeded" + case nextAction = "next_action" + } +} + private extension Optional where Wrapped == Int { func isSomeAndLessThan(_ threshold: Int) -> Bool { guard let value = self else { @@ -860,8 +899,8 @@ struct OperatorDashboardSocketPayload: Decodable, Sendable { let emittedAtUnixEpoch: Int64? let snapshotPublishedAtUnixEpoch: Int64? let snapshot: OperatorSnapshotResponse? - let activeRuns: [OperatorRunStatus]? - let activeRunsComplete: Bool? + let currentLanes: [OperatorRunStatus]? + let currentLanesComplete: Bool? var emittedAt: Date? { date(fromUnixEpoch: emittedAtUnixEpoch) @@ -875,22 +914,22 @@ struct OperatorDashboardSocketPayload: Decodable, Sendable { case emittedAtUnixEpoch case snapshotPublishedAtUnixEpoch case snapshot - case activeRuns - case activeRunsComplete + case currentLanes + case currentLanesComplete } } struct OperatorRunActivitySnapshot: Sendable { - let activeRuns: [OperatorRunStatus] - let activeRunsComplete: Bool + let currentLanes: [OperatorRunStatus] + let currentLanesComplete: Bool let emittedAt: Date var shouldPersistAsSnapshotOverlay: Bool { - activeRuns.isEmpty == false || activeRunsComplete == false + currentLanes.isEmpty == false || currentLanesComplete == false } func merging(into snapshot: OperatorSnapshotResponse) -> OperatorSnapshotResponse { - snapshot.mergingRunActivity(activeRuns, activeRunsComplete: activeRunsComplete) + snapshot.mergingRunActivity(currentLanes, currentLanesComplete: currentLanesComplete) } } diff --git a/apps/decodex/src/agent.rs b/apps/decodex/src/agent.rs index 9a560c155..c4b81ed7b 100644 --- a/apps/decodex/src/agent.rs +++ b/apps/decodex/src/agent.rs @@ -10,11 +10,11 @@ mod tracker_tool_bridge; #[cfg(test)] pub(crate) use self::tracker_tool_bridge::DynamicToolHandler; pub(crate) use self::{ app_server::{ - ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, + AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerPhaseGoalFailure, AppServerRunRequest, AppServerRunResult, AppServerThreadArchiveOutcome, AppServerThreadArchiveRequest, AppServerTurnFailure, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, - TurnContinuationGuard, execute_app_server_run, probe_app_server, + RUN_LEASE_IDLE_TIMEOUT, TurnContinuationGuard, execute_app_server_run, probe_app_server, protocol_activity_idle_timeout, }, codex_accounts::{CodexAccountAuthFailure, CodexAccountPool, CodexAccountProvider}, diff --git a/apps/decodex/src/agent/app_server.rs b/apps/decodex/src/agent/app_server.rs index 775b905ad..8281b98f7 100644 --- a/apps/decodex/src/agent/app_server.rs +++ b/apps/decodex/src/agent/app_server.rs @@ -67,7 +67,7 @@ use crate::{ }, }; -pub(crate) const ACTIVE_RUN_IDLE_TIMEOUT: Duration = Duration::from_secs(300); +pub(crate) const RUN_LEASE_IDLE_TIMEOUT: Duration = Duration::from_secs(300); pub(crate) const MODEL_EXECUTION_IDLE_TIMEOUT: Duration = Duration::from_secs(30 * 60); const PROBE_TIMEOUT: Duration = Duration::from_secs(30); diff --git a/apps/decodex/src/agent/app_server/tests.rs b/apps/decodex/src/agent/app_server/tests.rs index 64718b44c..f925e2edc 100644 --- a/apps/decodex/src/agent/app_server/tests.rs +++ b/apps/decodex/src/agent/app_server/tests.rs @@ -1027,7 +1027,7 @@ fn phase_goal_complete_runs_validation_transition_before_handoff_goal() { } #[test] -fn still_active_phase_goal_stops_at_max_turns_without_terminal_signal() { +fn open_phase_goal_stops_at_max_turns_without_terminal_signal() { let handler = ContinueTokenCompletionHandler; let controller = TestPhaseGoalController::new(PhaseGoalKind::ImplementToValidationReady); let script = phase_goal_fake_codex_script(&["CONTINUE", "DONE"], &["active", "active"], &[]); @@ -1036,7 +1036,7 @@ fn still_active_phase_goal_stops_at_max_turns_without_terminal_signal() { request.dynamic_tool_handler = Some(&handler); request.phase_goal_controller = Some(&controller); }); - let result = result.expect("active goal should allow another bounded turn"); + let result = result.expect("open phase goal should allow another bounded turn"); assert_eq!(result.turn_count, 2); assert_eq!(result.turn_id, "turn-2"); @@ -1052,7 +1052,7 @@ fn still_active_phase_goal_stops_at_max_turns_without_terminal_signal() { } #[test] -fn still_active_phase_goal_stops_at_max_turns_with_continuation_pending() { +fn open_phase_goal_stops_at_max_turns_with_continuation_pending() { let handler = ContinueTokenCompletionHandler; let controller = TestPhaseGoalController::new(PhaseGoalKind::ImplementToValidationReady); let script = phase_goal_fake_codex_script(&["CONTINUE"], &["active"], &[]); @@ -1061,7 +1061,7 @@ fn still_active_phase_goal_stops_at_max_turns_with_continuation_pending() { request.dynamic_tool_handler = Some(&handler); request.phase_goal_controller = Some(&controller); }); - let result = result.expect("active goal should exit cleanly at max_turns"); + let result = result.expect("open phase goal should exit cleanly at max_turns"); assert_eq!(result.turn_count, 1); assert!(result.continuation_pending); @@ -1613,7 +1613,7 @@ fn protocol_activity_idle_timeout_extends_running_model_execution() { assert_eq!( super::protocol_activity_idle_timeout( Some(&protocol_activity), - super::ACTIVE_RUN_IDLE_TIMEOUT + super::RUN_LEASE_IDLE_TIMEOUT ), super::MODEL_EXECUTION_IDLE_TIMEOUT ); @@ -1630,9 +1630,9 @@ fn protocol_activity_idle_timeout_keeps_base_timeout_for_other_waits() { assert_eq!( super::protocol_activity_idle_timeout( Some(&protocol_activity), - super::ACTIVE_RUN_IDLE_TIMEOUT + super::RUN_LEASE_IDLE_TIMEOUT ), - super::ACTIVE_RUN_IDLE_TIMEOUT + super::RUN_LEASE_IDLE_TIMEOUT ); } diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs index 7b4c07ac5..c540ecb19 100644 --- a/apps/decodex/src/cli.rs +++ b/apps/decodex/src/cli.rs @@ -514,7 +514,7 @@ struct LaneInterruptCommand { #[derive(Debug, Args)] struct LaneSteerCommand { - /// Issue identifier or local issue id for the active lane. + /// Issue identifier or local issue id for the current lane. #[arg(value_name = "ISSUE")] issue: String, /// Run id that must own the active turn. diff --git a/apps/decodex/src/execution_program.rs b/apps/decodex/src/execution_program.rs index c41e1b378..4aee65ecb 100644 --- a/apps/decodex/src/execution_program.rs +++ b/apps/decodex/src/execution_program.rs @@ -329,7 +329,7 @@ pub(crate) enum ExecutionProgramNodeLifecycleState { Ready, /// Node was retained in a ready-to-dispatch position. Queued, - /// Node already has an active lane. + /// Node already has a current lane. Active, /// Node is blocked by dependency, conflict, issue, or briefing evidence. Blocked, @@ -1492,7 +1492,7 @@ fn evaluate_node(input: EvaluateNodeInput<'_>) -> Result { state = ExecutionReadinessState::Active; - reasons.push(String::from("node already has an active lane")); + reasons.push(String::from("node already has a current lane")); }, ExecutionQueueIntent::Done | ExecutionQueueIntent::Canceled => { state = ExecutionReadinessState::Completed; diff --git a/apps/decodex/src/maintenance.rs b/apps/decodex/src/maintenance.rs index 922b2e844..4f4db6bb2 100644 --- a/apps/decodex/src/maintenance.rs +++ b/apps/decodex/src/maintenance.rs @@ -556,7 +556,7 @@ fn maintain_runtime_protocol_events( event_count: candidate.event_count, last_event_at: candidate.last_event_at.clone(), reason: format!( - "terminal run has no active lease, retained worktree, or review marker and its latest protocol event is older than {} days", + "terminal run has no run lease, retained worktree, or review marker and its latest protocol event is older than {} days", policy.protocol_event_retention_days ), }); @@ -631,7 +631,7 @@ fn protocol_event_compaction_candidates( JOIN protocol_events last ON last.run_id = totals.run_id AND last.sequence_number = totals.last_sequence_number - LEFT JOIN leases active_lease ON active_lease.issue_id = attempts.issue_id + LEFT JOIN leases run_lease ON run_lease.issue_id = attempts.issue_id LEFT JOIN worktrees retained_worktree ON retained_worktree.issue_id = attempts.issue_id LEFT JOIN review_handoffs review_handoff ON review_handoff.issue_id = attempts.issue_id LEFT JOIN review_orchestrations review_orchestration @@ -648,7 +648,7 @@ fn protocol_event_compaction_candidates( AND human_stop_event.run_id = attempts.run_id WHERE attempts.status IN ('succeeded', 'failed', 'interrupted', 'terminated') AND totals.last_created_at_unix < ?1 - AND active_lease.issue_id IS NULL + AND run_lease.issue_id IS NULL AND retained_worktree.issue_id IS NULL AND review_handoff.issue_id IS NULL AND review_orchestration.issue_id IS NULL @@ -681,7 +681,7 @@ fn protected_protocol_run_count(connection: &Connection) -> Result { "SELECT COUNT(DISTINCT attempts.run_id) FROM run_attempts attempts JOIN protocol_events events ON events.run_id = attempts.run_id - LEFT JOIN leases active_lease ON active_lease.issue_id = attempts.issue_id + LEFT JOIN leases run_lease ON run_lease.issue_id = attempts.issue_id LEFT JOIN worktrees retained_worktree ON retained_worktree.issue_id = attempts.issue_id LEFT JOIN review_handoffs review_handoff ON review_handoff.issue_id = attempts.issue_id LEFT JOIN review_orchestrations review_orchestration @@ -696,7 +696,7 @@ fn protected_protocol_run_count(connection: &Connection) -> Result { ) human_stop_event ON human_stop_event.issue_id = attempts.issue_id AND human_stop_event.run_id = attempts.run_id - WHERE active_lease.issue_id IS NOT NULL + WHERE run_lease.issue_id IS NOT NULL OR retained_worktree.issue_id IS NOT NULL OR review_handoff.issue_id IS NOT NULL OR review_orchestration.issue_id IS NOT NULL @@ -1043,18 +1043,18 @@ mod tests { insert_attempt(&connection, "old-run", "old-issue", "succeeded"); insert_event(&connection, "old-run", 1, old); insert_event(&connection, "old-run", 2, old + 60); - insert_attempt(&connection, "active-run", "active-issue", "running"); - insert_event(&connection, "active-run", 1, old); - insert_attempt(&connection, "old-active-issue-run", "active-issue", "succeeded"); - insert_event(&connection, "old-active-issue-run", 1, old); + insert_attempt(&connection, "leased-run", "leased-issue", "running"); + insert_event(&connection, "leased-run", 1, old); + insert_attempt(&connection, "old-leased-issue-run", "leased-issue", "succeeded"); + insert_event(&connection, "old-leased-issue-run", 1, old); connection .execute( "INSERT INTO leases (issue_id, project_id, run_id, issue_state) - VALUES ('active-issue', 'decodex', 'active-run', 'In Progress')", + VALUES ('leased-issue', 'decodex', 'leased-run', 'In Progress')", [], ) - .expect("active lease should insert"); + .expect("run lease should insert"); insert_attempt(&connection, "retained-run", "retained-issue", "failed"); insert_event(&connection, "retained-run", 1, old); @@ -1106,8 +1106,8 @@ mod tests { assert_eq!(report.runtime.compacted_events, 2); assert_eq!(protocol_event_count(&connection, "old-run"), 0); assert_eq!(protocol_summary_event_count(&connection, "old-run"), Some(2)); - assert_eq!(protocol_event_count(&connection, "active-run"), 1); - assert_eq!(protocol_event_count(&connection, "old-active-issue-run"), 1); + assert_eq!(protocol_event_count(&connection, "leased-run"), 1); + assert_eq!(protocol_event_count(&connection, "old-leased-issue-run"), 1); assert_eq!(protocol_event_count(&connection, "retained-run"), 1); assert_eq!(protocol_event_count(&connection, "review-handoff-run"), 1); assert_eq!(protocol_event_count(&connection, "cleanup-blocked-run"), 1); diff --git a/apps/decodex/src/manual.rs b/apps/decodex/src/manual.rs index aa094fc93..f237572c9 100644 --- a/apps/decodex/src/manual.rs +++ b/apps/decodex/src/manual.rs @@ -1587,8 +1587,8 @@ fn clear_manual_closeout_runtime_state( issue_id: &str, handoff_run_id: &str, ) -> Result<()> { - state_store.succeed_active_run_attempts_for_issue(issue_id).wrap_err_with(|| { - format!("Failed to finalize active runtime attempts for issue `{issue_id}`.") + state_store.succeed_running_run_attempts_for_issue(issue_id).wrap_err_with(|| { + format!("Failed to finalize running runtime attempts for issue `{issue_id}`.") })?; succeed_manual_land_handoff_attempt(state_store, issue_id, handoff_run_id)?; diff --git a/apps/decodex/src/orchestrator.rs b/apps/decodex/src/orchestrator.rs index a638f796c..0d39ba3ce 100644 --- a/apps/decodex/src/orchestrator.rs +++ b/apps/decodex/src/orchestrator.rs @@ -40,7 +40,7 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::{agent, default_branch_sync, git_credentials, maintenance, state}; #[rustfmt::skip] -use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{ReviewLevel, ServiceConfig}, execution_program::{ExecutionNodeEvaluation, ExecutionProgramEvaluation, ExecutionProgramOperatorSummary, ExecutionProgramReadinessContext, ExecutionWorkflowPolicy}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ExecutionProgramRecord, LoopGuardrailCheckpoint, LoopGuardrailCheckpointInput, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; +use crate::{agent::{RUN_LEASE_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{ReviewLevel, ServiceConfig}, execution_program::{ExecutionNodeEvaluation, ExecutionProgramEvaluation, ExecutionProgramOperatorSummary, ExecutionProgramReadinessContext, ExecutionWorkflowPolicy}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ExecutionProgramRecord, LoopGuardrailCheckpoint, LoopGuardrailCheckpointInput, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; use harness_improvement::{ HarnessImprovementCandidateSummary, HarnessOutcomeKind, harness_improvement_candidates_from_private_events, record_harness_outcome_best_effort, diff --git a/apps/decodex/src/orchestrator/agent_evidence.rs b/apps/decodex/src/orchestrator/agent_evidence.rs index 47c3eb589..9d160210b 100644 --- a/apps/decodex/src/orchestrator/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/agent_evidence.rs @@ -55,7 +55,7 @@ struct AgentHandoffIndex { #[derive(Clone, Debug, Eq, PartialEq, Serialize)] struct AgentEvidenceSummary { project_count: usize, - active_run_count: usize, + current_lane_count: usize, recent_run_count: usize, history_lane_count: usize, queued_candidate_count: usize, @@ -156,7 +156,7 @@ struct AgentRunCapsule { terminalization_state: String, lane_control_next_action: String, lane_control_conditions: Vec, - active_lease: bool, + run_lease: bool, continuation_pending: bool, suspected_stall: bool, thread_id: Option, @@ -356,7 +356,7 @@ struct AgentEvidenceProjectView<'a> { warnings: Vec, projects: Vec<&'a OperatorProjectStatus>, connector_backoffs: Vec<&'a OperatorConnectorBackoffStatus>, - active_runs: Vec<&'a OperatorRunStatus>, + current_lanes: Vec<&'a OperatorRunStatus>, recent_runs: Vec<&'a OperatorRunStatus>, history_lanes: Vec<&'a OperatorHistoryLaneStatus>, queued_candidates: Vec<&'a OperatorQueuedIssueStatus>, @@ -376,8 +376,8 @@ impl<'a> AgentEvidenceProjectView<'a> { .iter() .filter(|backoff| backoff.project_id == project_id) .collect::>(); - let active_runs = snapshot - .active_runs + let current_lanes = snapshot + .current_lanes .iter() .filter(|run| run.project_id == project_id) .collect::>(); @@ -419,7 +419,7 @@ impl<'a> AgentEvidenceProjectView<'a> { warnings: snapshot.warnings.clone(), projects, connector_backoffs, - active_runs, + current_lanes, recent_runs, history_lanes, queued_candidates, @@ -479,7 +479,7 @@ fn write_agent_evidence_snapshot( .collect::>(); let summary = AgentEvidenceSummary { project_count: project_view.projects.len(), - active_run_count: project_view.active_runs.len(), + current_lane_count: project_view.current_lanes.len(), recent_run_count: project_view.recent_runs.len(), history_lane_count: project_view.history_lanes.len(), queued_candidate_count: project_view.queued_candidates.len(), @@ -1437,7 +1437,7 @@ fn lane_issue_belongs_to_project( snapshot: &OperatorStatusSnapshot, ) -> bool { snapshot - .active_runs + .current_lanes .iter() .chain(snapshot.recent_runs.iter()) .any(|run| run.project_id == project_id && run.issue_id == issue_id) @@ -1454,7 +1454,7 @@ fn agent_evidence_project_ids(snapshot: &OperatorStatusSnapshot) -> Vec for project in &snapshot.projects { project_ids.insert(project.project_id.clone()); } - for run in snapshot.active_runs.iter().chain(snapshot.recent_runs.iter()) { + for run in snapshot.current_lanes.iter().chain(snapshot.recent_runs.iter()) { project_ids.insert(run.project_id.clone()); } for lane in &snapshot.history_lanes { @@ -1481,7 +1481,7 @@ fn build_run_capsules( let mut capsules = Vec::new(); for run in project_view - .active_runs + .current_lanes .iter() .chain(project_view.recent_runs.iter()) .copied() @@ -1579,7 +1579,7 @@ fn agent_run_capsule( terminalization_state: run.terminalization_state.clone(), lane_control_next_action: run.lane_control_next_action.clone(), lane_control_conditions: run.lane_control_conditions.clone(), - active_lease: run.active_lease, + run_lease: run.run_lease, continuation_pending: run.continuation_pending, suspected_stall: run.suspected_stall, thread_id: run.thread_id.clone(), @@ -1715,7 +1715,7 @@ fn push_run_blockers( blockers_dir: &Path, run_refs: &[AgentRunCapsuleRef], ) { - for run in &project_view.active_runs { + for run in &project_view.current_lanes { if let Some(reason_code) = agent_run_blocker_reason(run) { let issue_key = issue_key(run.issue_identifier.as_deref(), &run.issue_id); diff --git a/apps/decodex/src/orchestrator/daemon.rs b/apps/decodex/src/orchestrator/daemon.rs index 00032bef4..11644f63a 100644 --- a/apps/decodex/src/orchestrator/daemon.rs +++ b/apps/decodex/src/orchestrator/daemon.rs @@ -290,12 +290,12 @@ where child_ref.run_id, )? } else { - inspect_active_daemon_child_reconciliation( + inspect_current_daemon_child_reconciliation( tracker, project, workflow, state_store, - ActiveChildRunContext { + CurrentChildRunContext { child: child_ref, workflow: &active_children[index].workflow, dispatch_mode: active_children[index].dispatch_mode, @@ -357,23 +357,23 @@ where stop_daemon_child(&mut daemon_child.child)?; } - apply_active_run_reconciliation(tracker, project, state_store, worktree_manager, actions)?; + apply_run_lease_reconciliation(tracker, project, state_store, worktree_manager, actions)?; } Ok(()) } -fn inspect_active_daemon_child_reconciliation( +fn inspect_current_daemon_child_reconciliation( tracker: &T, project: &ServiceConfig, workflow: &WorkflowDocument, state_store: &StateStore, - child_context: ActiveChildRunContext<'_>, -) -> Result> + child_context: CurrentChildRunContext<'_>, +) -> Result> where T: IssueTracker, { - inspect_active_daemon_child_reconciliation_at( + inspect_current_daemon_child_reconciliation_at( tracker, project, workflow, @@ -383,14 +383,14 @@ where ) } -fn inspect_active_daemon_child_reconciliation_at( +fn inspect_current_daemon_child_reconciliation_at( tracker: &T, project: &ServiceConfig, workflow: &WorkflowDocument, state_store: &StateStore, - child_context: ActiveChildRunContext<'_>, + child_context: CurrentChildRunContext<'_>, now_unix_epoch: i64, -) -> Result> +) -> Result> where T: IssueTracker, { @@ -404,7 +404,7 @@ where let worktree_mapping = state_store.worktree_for_issue(&issue.id)?; if let Some(disposition) = superseded_run_disposition(state_store, &run_attempt)? { - return Ok(vec![ActiveRunReconciliation { + return Ok(vec![RunLeaseReconciliation { issue: issue.clone(), run_attempt, worktree_mapping, @@ -413,7 +413,7 @@ where }]); } - let action_workflow = active_reconciliation_workflow_for_lease( + let action_workflow = run_lease_reconciliation_workflow( workflow, Some(ActiveWorkflowOverride { child, workflow: child_context.workflow }), &issue, @@ -432,9 +432,9 @@ where let disposition = if !retained_closeout && !completed_closeout_child && is_terminal_issue(&issue, action_workflow) { - Some(ActiveRunDisposition::Terminal) + Some(RunLeaseDisposition::Terminal) } else if !retained_closeout && !completed_closeout_child - && is_issue_nonactive_for_active_dispatch( + && is_issue_not_dispatchable_for_current_dispatch( tracker, &issue, project, @@ -442,7 +442,7 @@ where child_context.dispatch_mode, )? { - Some(ActiveRunDisposition::NonActive) + Some(RunLeaseDisposition::NotDispatchable) } else if let Some(idle_for) = stalled_idle_duration( state_store, &run_attempt, @@ -454,18 +454,18 @@ where &run_attempt, worktree_mapping.as_ref(), )? { - Some(ActiveRunDisposition::RetainedReviewComplete) + Some(RunLeaseDisposition::RetainedReviewComplete) } else if stalled_run_has_retained_partial_progress(worktree_mapping.as_ref()) { - Some(ActiveRunDisposition::StalledRetainedPartialProgress { idle_for }) + Some(RunLeaseDisposition::StalledRetainedPartialProgress { idle_for }) } else { - Some(ActiveRunDisposition::Stalled { idle_for }) + Some(RunLeaseDisposition::Stalled { idle_for }) } } else { None }; Ok(disposition.map_or_else(Vec::new, |disposition| { - vec![ActiveRunReconciliation { + vec![RunLeaseReconciliation { issue: issue.clone(), run_attempt, worktree_mapping, @@ -619,7 +619,7 @@ where issue = summary.issue_identifier, worktree = %daemon_spawn_state.worktree.path.display(), retry = from_retry_queue, - "Spawned control-plane child for active issue lane." + "Spawned control-plane child for current issue lane." ); active_children.push(DaemonRunChild { @@ -1242,7 +1242,7 @@ where return Ok(ChildExitPhaseGoalRecovery::None); } - let recovery = maybe_recover_child_exit_active_phase_goal_continuation( + let recovery = maybe_recover_child_exit_phase_goal_continuation( context, issue, child, @@ -1257,7 +1257,7 @@ where Ok(recovery) } -fn maybe_recover_child_exit_active_phase_goal_continuation( +fn maybe_recover_child_exit_phase_goal_continuation( context: &ChildExitRetryContext<'_, T>, issue: &TrackerIssue, child: ChildRunRef<'_>, @@ -1280,12 +1280,13 @@ where run_id: child.run_id.to_owned(), retry_budget_base: 0, }; - let recovery = match recover_active_phase_goal_continuation( + let recovery = match recover_phase_goal_continuation( context.project, context.workflow, context.state_store, &issue_run, "child_exit_failed", + Some("child_exit_failed"), ) { Ok(recovery) => recovery, Err(error) if run_failure_requires_terminal_attention(&error) => { @@ -1312,7 +1313,7 @@ where attempt = child.attempt_number, source_phase = recovery.source_phase.as_str(), next_phase = recovery.next_phase.as_str(), - "Recovered active phase goal after child exit failure; scheduling continuation." + "Recovered phase goal after child exit failure; scheduling continuation." ); } diff --git a/apps/decodex/src/orchestrator/dispatch_policy.rs b/apps/decodex/src/orchestrator/dispatch_policy.rs index ddcd1f53d..baa7d6379 100644 --- a/apps/decodex/src/orchestrator/dispatch_policy.rs +++ b/apps/decodex/src/orchestrator/dispatch_policy.rs @@ -576,14 +576,14 @@ fn state_name_is_terminal(state_name: &str, workflow: &WorkflowDocument) -> bool workflow.frontmatter().tracker().terminal_states().iter().any(|state| state == state_name) } -fn is_issue_active_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { +fn is_issue_in_progress_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { let tracker_policy = workflow.frontmatter().tracker(); issue.state.name == tracker_policy.in_progress_state() && !issue.has_label(tracker_policy.needs_attention_label()) } -fn is_issue_nonactive_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { +fn is_issue_not_dispatchable_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) -> bool { let tracker_policy = workflow.frontmatter().tracker(); issue.has_label(tracker_policy.opt_out_label()) @@ -592,7 +592,7 @@ fn is_issue_nonactive_for_run(issue: &TrackerIssue, workflow: &WorkflowDocument) && !tracker_policy.startable_states().iter().any(|state| state == &issue.state.name)) } -fn is_issue_nonactive_for_active_dispatch( +fn is_issue_not_dispatchable_for_current_dispatch( tracker: &T, issue: &TrackerIssue, project: &ServiceConfig, @@ -609,7 +609,7 @@ where IssueDispatchMode::Normal | IssueDispatchMode::Program | IssueDispatchMode::Retry - | IssueDispatchMode::Closeout => Ok(is_issue_nonactive_for_run(issue, workflow)), + | IssueDispatchMode::Closeout => Ok(is_issue_not_dispatchable_for_run(issue, workflow)), } } diff --git a/apps/decodex/src/orchestrator/entrypoints.rs b/apps/decodex/src/orchestrator/entrypoints.rs index 0aab49b06..261419c1f 100644 --- a/apps/decodex/src/orchestrator/entrypoints.rs +++ b/apps/decodex/src/orchestrator/entrypoints.rs @@ -560,8 +560,8 @@ fn project_status_snapshot_from_operator_cache( .into_iter() .filter(|backoff| backoff.project_id == project.service_id()) .collect(); - project_snapshot.active_runs = snapshot - .active_runs + project_snapshot.current_lanes = snapshot + .current_lanes .into_iter() .filter(|run| run.project_id == project.service_id()) .collect(); @@ -1262,14 +1262,14 @@ fn control_plane_disabled_project_observer_tick( snapshot_warnings: &mut Vec<&'static str>, ) -> ControlPlaneProjectTick { let project_status = operator_project_status_from_registration(project, 0); - let active_runs = match state_store.list_active_runs(project.service_id()) { - Ok(active_runs) => active_runs, + let current_lanes = match state_store.list_leased_runs(project.service_id()) { + Ok(current_lanes) => current_lanes, Err(error) => { let _ = error; tracing::warn!( project_id = project.service_id(), - "Disabled project active-run lookup failed; sensitive runtime details were withheld." + "Disabled project leased-run lookup failed; sensitive runtime details were withheld." ); snapshot_warnings.push("operator_snapshot_build_failed"); @@ -1281,7 +1281,7 @@ fn control_plane_disabled_project_observer_tick( }, }; - if active_runs.is_empty() { + if current_lanes.is_empty() { return ControlPlaneProjectTick { snapshot: None, project_status: Some(project_status), @@ -1304,7 +1304,7 @@ fn control_plane_disabled_project_observer_tick( tracing::warn!( project_id = project.service_id(), - "Disabled project active-run snapshot build failed; sensitive runtime details were withheld." + "Disabled project leased-run snapshot build failed; sensitive runtime details were withheld." ); snapshot_warnings.push("operator_snapshot_build_failed"); @@ -1339,9 +1339,9 @@ fn hydrate_project_status_from_local_snapshot( if let Some(local_status) = project_snapshot.projects.first() { hydrate_project_status_from_registered_status(project_status, local_status); } else { - project_status.active_run_count = project_snapshot.active_runs.len(); + project_status.current_lane_count = project_snapshot.current_lanes.len(); project_status.running_lane_count = project_snapshot - .active_runs + .current_lanes .iter() .filter(|run| operator_run_counts_as_running(run)) .count(); @@ -1352,7 +1352,7 @@ fn hydrate_project_status_from_registered_status( project_status: &mut OperatorProjectStatus, local_status: &OperatorProjectStatus, ) { - project_status.active_run_count = local_status.active_run_count; + project_status.current_lane_count = local_status.current_lane_count; project_status.running_lane_count = local_status.running_lane_count; project_status.retained_worktree_count = local_status.retained_worktree_count; project_status.waiting_lane_count = local_status.waiting_lane_count; @@ -1392,7 +1392,7 @@ fn append_control_plane_project_snapshot( snapshot.warning_details.extend(project_snapshot.warning_details); snapshot.connector_backoffs.extend(project_snapshot.connector_backoffs); snapshot.accounts.extend(project_snapshot.accounts); - snapshot.active_runs.extend(project_snapshot.active_runs); + snapshot.current_lanes.extend(project_snapshot.current_lanes); snapshot.recent_runs.extend(project_snapshot.recent_runs); snapshot.history_lanes.extend(project_snapshot.history_lanes); snapshot.execution_programs.extend(project_snapshot.execution_programs); @@ -1639,7 +1639,15 @@ fn build_operator_state_snapshot_without_live_observers( )?; hydrate_history_lanes_from_local_ledger(project, state_store, &mut snapshot)?; - apply_terminal_history_ledger_outcomes(&mut snapshot); + + let terminal_projection = + current_lane_terminal_projection_from_local_ledger(project, state_store, &snapshot)?; + + apply_operator_lane_terminal_projection( + &mut snapshot, + terminal_projection, + Some(workflow.frontmatter().tracker().resolved_completed_state()), + ); refresh_worktree_ownership( &mut snapshot, Some(workflow.frontmatter().tracker().resolved_completed_state()), @@ -1932,7 +1940,7 @@ fn empty_control_plane_snapshot(limit: usize) -> OperatorStatusSnapshot { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), recent_runs: Vec::new(), history_lanes: Vec::new(), execution_programs: Vec::new(), @@ -1952,7 +1960,7 @@ fn operator_project_status_from_registration( repo_root: project.repo_root().display().to_string(), enabled: project.enabled(), github_cli_authority: operator_github_cli_authority_from_registration(project), - active_run_count: 0, + current_lane_count: 0, running_lane_count: 0, queued_candidate_count: 0, post_review_lane_count: 0, @@ -1984,7 +1992,7 @@ fn operator_project_status_from_dev_registration( repo_root: project.repo_root().display().to_string(), enabled: project.enabled(), github_cli_authority: operator_github_cli_authority_from_registration(project), - active_run_count: 0, + current_lane_count: 0, running_lane_count: 0, queued_candidate_count: 0, post_review_lane_count: 0, diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 7d66f66e3..c89249ba3 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -470,6 +470,17 @@ struct ArchitectureRecoveryTerminalEventInput<'a> { recovery_attempt_number: usize, } +#[derive(Clone, Copy)] +struct PhaseGoalRecoveryRecord<'a> { + project: &'a ServiceConfig, + state_store: &'a StateStore, + issue_run: &'a IssueRunPlan, + source_phase: PhaseGoalKind, + next_phase: PhaseGoalKind, + source_error_class: &'a str, + source_error_message: Option<&'a str>, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum RunFailureWritebackDisposition { RetryableGeneric, @@ -1114,7 +1125,7 @@ where input.review_context, ), max_turns: input.workflow.frontmatter().execution().max_turns(), - timeout: ACTIVE_RUN_IDLE_TIMEOUT, + timeout: RUN_LEASE_IDLE_TIMEOUT, process_env: input.process_env.clone(), continuation_user_input: Some(build_issue_run_continuation_user_input( input.project, @@ -1148,7 +1159,7 @@ where } if !input.tracker_tool_bridge.has_tracker_exit_signal() - && let Some(summary) = maybe_continue_after_active_phase_goal_recovery( + && let Some(summary) = maybe_continue_after_phase_goal_recovery( input.project, input.workflow, input.state_store, @@ -1262,19 +1273,21 @@ where Ok(Some(run_summary_from_issue_run(project.service_id(), issue_run))) } -fn maybe_continue_after_active_phase_goal_recovery( +fn maybe_continue_after_phase_goal_recovery( project: &ServiceConfig, workflow: &WorkflowDocument, state_store: &StateStore, issue_run: &IssueRunPlan, error: &Report, ) -> Result> { - let Some(recovery) = recover_active_phase_goal_continuation( + let source_error_message = phase_goal_recovery_source_error_message(error); + let Some(recovery) = recover_phase_goal_continuation( project, workflow, state_store, issue_run, phase_goal_recovery_source_error_class(error), + Some(source_error_message.as_str()), )? else { return Ok(None); }; @@ -1291,24 +1304,25 @@ fn maybe_continue_after_active_phase_goal_recovery( source_phase = recovery.source_phase.as_str(), next_phase = recovery.next_phase.as_str(), error = %error, - "Recovered active phase goal after app-server failure; scheduling continuation." + "Recovered phase goal after app-server failure; scheduling continuation." ); Ok(Some(summary)) } -fn recover_active_phase_goal_continuation( +fn recover_phase_goal_continuation( project: &ServiceConfig, workflow: &WorkflowDocument, state_store: &StateStore, issue_run: &IssueRunPlan, source_error_class: &str, + source_error_message: Option<&str>, ) -> Result> { if !worktree_has_tracked_changes(&issue_run.worktree.path) { return Ok(None); } - let Some(source_phase) = latest_active_phase_goal_recovery_candidate( + let Some(source_phase) = latest_phase_goal_recovery_candidate( project, state_store, issue_run, @@ -1326,15 +1340,30 @@ fn recover_active_phase_goal_continuation( PhaseGoalTransition::Continue(next_goal) => next_goal.phase, PhaseGoalTransition::CompleteRun => return Ok(None), }; - - record_phase_goal_recovery_continuation( + let prior_recovery_count = matching_phase_goal_recovery_count( project, state_store, issue_run, source_phase, - next_phase, source_error_class, )?; + let recovery_record = PhaseGoalRecoveryRecord { + project, + state_store, + issue_run, + source_phase, + next_phase, + source_error_class, + source_error_message, + }; + + if prior_recovery_count >= PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT { + record_phase_goal_recovery_blocked(recovery_record, prior_recovery_count)?; + + return Ok(None); + } + + record_phase_goal_recovery_continuation(recovery_record)?; Ok(Some(PhaseGoalRecoveryContinuation { source_phase, next_phase })) } @@ -1343,7 +1372,23 @@ fn phase_goal_recovery_source_error_class(error: &Report) -> &'static str { retained_progress_source_error_class(error).unwrap_or("app_server_run_failed") } -fn latest_active_phase_goal_recovery_candidate( +fn phase_goal_recovery_source_error_message(error: &Report) -> String { + truncate_phase_goal_recovery_error(error.to_string(), 512) +} + +fn truncate_phase_goal_recovery_error(value: String, max_chars: usize) -> String { + if value.chars().count() <= max_chars { + return value; + } + + let mut truncated = value.chars().take(max_chars).collect::(); + + truncated.push_str("..."); + + truncated +} + +fn latest_phase_goal_recovery_candidate( project: &ServiceConfig, state_store: &StateStore, issue_run: &IssueRunPlan, @@ -1424,27 +1469,81 @@ fn phase_goal_recovery_candidate_from_status( } } -fn record_phase_goal_recovery_continuation( +fn matching_phase_goal_recovery_count( project: &ServiceConfig, state_store: &StateStore, issue_run: &IssueRunPlan, source_phase: PhaseGoalKind, - next_phase: PhaseGoalKind, source_error_class: &str, +) -> Result { + let events = + state_store.list_private_execution_events_for_issue(project.service_id(), &issue_run.issue.id)?; + + Ok(events + .iter() + .filter(|event| { + event.event_type() == PHASE_GOAL_RECOVERY_EVENT_TYPE + && phase_goal_recovery_event_source_phase(event.payload()) + .is_some_and(|phase| phase == source_phase.as_str()) + && phase_goal_recovery_event_source_error_class(event.payload()) + .is_some_and(|class| class == source_error_class) + }) + .count() as i64) +} + +fn phase_goal_recovery_event_source_phase(payload: &Value) -> Option<&str> { + payload + .get("phase") + .and_then(Value::as_str) + .or_else(|| payload.get("payload")?.get("sourcePhase")?.as_str()) +} + +fn phase_goal_recovery_event_source_error_class(payload: &Value) -> Option<&str> { + payload.get("payload")?.get("sourceErrorClass")?.as_str() +} + +fn record_phase_goal_recovery_continuation(record: PhaseGoalRecoveryRecord<'_>) -> Result<()> { + record.state_store.append_private_execution_event( + record.project.service_id(), + &record.issue_run.issue.id, + &record.issue_run.run_id, + record.issue_run.attempt_number, + PHASE_GOAL_RECOVERY_EVENT_TYPE, + json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": record.source_phase.as_str(), + "signal": "phase_goal_recovered", + "payload": { + "nextPhase": record.next_phase.as_str(), + "sourceErrorClass": record.source_error_class, + "sourceErrorMessage": record.source_error_message, + }, + }), + )?; + + Ok(()) +} + +fn record_phase_goal_recovery_blocked( + record: PhaseGoalRecoveryRecord<'_>, + prior_recovery_count: i64, ) -> Result<()> { - state_store.append_private_execution_event( - project.service_id(), - &issue_run.issue.id, - &issue_run.run_id, - issue_run.attempt_number, - "phase_goal_recovery", + record.state_store.append_private_execution_event( + record.project.service_id(), + &record.issue_run.issue.id, + &record.issue_run.run_id, + record.issue_run.attempt_number, + PHASE_GOAL_RECOVERY_BLOCKED_EVENT_TYPE, json!({ "schema": "decodex.phase_goal_signal/1", - "phase": source_phase.as_str(), - "signal": "active_goal_recovered", + "phase": record.source_phase.as_str(), + "signal": "continuation_budget_exhausted", "payload": { - "nextPhase": next_phase.as_str(), - "sourceErrorClass": source_error_class, + "nextPhase": record.next_phase.as_str(), + "sourceErrorClass": record.source_error_class, + "sourceErrorMessage": record.source_error_message, + "priorRecoveryCount": prior_recovery_count, + "automaticContinuationLimit": PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT, }, }), )?; diff --git a/apps/decodex/src/orchestrator/lane_control.rs b/apps/decodex/src/orchestrator/lane_control.rs index 743413126..a8cee5410 100644 --- a/apps/decodex/src/orchestrator/lane_control.rs +++ b/apps/decodex/src/orchestrator/lane_control.rs @@ -105,7 +105,7 @@ struct LaneRunInspect { phase: String, wait_reason: Option, current_operation: String, - active_lease: bool, + run_lease: bool, execution_liveness: String, ownership_state: String, liveness_state: String, @@ -140,7 +140,7 @@ impl LaneRunInspect { phase: run.phase.clone(), wait_reason: run.wait_reason.clone(), current_operation: run.current_operation.clone(), - active_lease: run.active_lease, + run_lease: run.run_lease, execution_liveness: run.execution_liveness.clone(), ownership_state: run.ownership_state.clone(), liveness_state: run.liveness_state.clone(), @@ -418,7 +418,7 @@ fn soft_interrupt_allows_hard_fallback( "pending" | "failed" | "unavailable" => soft.error_class.as_deref() != Some("lane_not_active") || run.process_id.is_some() && run.process_alive != Some(false), - "rejected" => soft.error_class.as_deref() == Some("active_lease_missing"), + "rejected" => soft.error_class.as_deref() == Some("run_lease_missing"), _ => false, } } @@ -500,7 +500,7 @@ fn matching_lane_runs( let mut seen_run_ids = HashSet::new(); let mut runs = Vec::new(); - for run in snapshot.active_runs.iter().chain(snapshot.recent_runs.iter()) { + for run in snapshot.current_lanes.iter().chain(snapshot.recent_runs.iter()) { if !seen_run_ids.insert(run.run_id.clone()) { continue; } @@ -529,7 +529,7 @@ fn lane_issue_matches(run: &OperatorRunStatus, issue: &str) -> bool { } fn soft_interrupt_available_for_run(run: &OperatorRunStatus) -> bool { - orchestrator::operator_run_counts_as_active(run) + orchestrator::operator_run_counts_as_current_lane(run) && run.worktree_path.is_some() && run.thread_id.is_some() && run.turn_id.is_some() @@ -563,7 +563,7 @@ fn lane_control_operator_context(run: &OperatorRunStatus) -> Value { "phase": run.phase.as_str(), "wait_reason": run.wait_reason.as_deref(), "current_operation": run.current_operation.as_str(), - "active_lease": run.active_lease, + "run_lease": run.run_lease, "queue_lease_state": run.queue_lease_state.as_str(), "execution_liveness": run.execution_liveness.as_str(), "ownership_state": run.ownership_state.as_str(), @@ -625,7 +625,7 @@ fn attempt_lane_steer( } let Some(worktree_path) = absolute_lane_worktree_path(project, state_store, run)? else { - eyre::bail!("Lane steer was accepted without an active lane worktree."); + eyre::bail!("Lane steer was accepted without a current lane worktree."); }; let Some(thread_id) = run.thread_id.as_deref() else { eyre::bail!("Lane steer was accepted before the active app-server thread id was known."); @@ -827,7 +827,7 @@ fn lane_steer_failure_class_for_reason(reason: &str) -> Option<&'static str> { "app_server_turn_steer_timed_out" => Some("app_server_turn_steer_timed_out"), "app_server_turn_steer_unsupported" => Some("app_server_turn_steer_unsupported"), "app_server_turn_steer_failed" => Some("app_server_turn_steer_failed"), - "active_run_control_channel_resolved" | "queued_wait_timeout" => None, + "run_lease_control_channel_resolved" | "queued_wait_timeout" => None, _ => Some("run_control_action_failed"), } } @@ -847,7 +847,7 @@ fn attempt_soft_lane_interrupt( let Some(worktree_path) = absolute_lane_worktree_path(project, state_store, run)? else { return Ok(LaneSoftInterruptReport::unavailable( "worktree_missing", - "Soft interrupt requires the active lane worktree and run-control directory.", + "Soft interrupt requires the current lane worktree and run-control directory.", )); }; let Some(thread_id) = run.thread_id.as_deref() else { @@ -863,10 +863,10 @@ fn attempt_soft_lane_interrupt( )); }; - if !orchestrator::operator_run_counts_as_active(run) { + if !orchestrator::operator_run_counts_as_current_lane(run) { return Ok(LaneSoftInterruptReport::unavailable( "lane_not_active", - "Soft interrupt only targets active or live local lane runs.", + "Soft interrupt only targets current or live local lane runs.", )); } @@ -1203,12 +1203,12 @@ fn render_lane_inspect_report(report: &LaneInspectReport) -> String { for run in &report.runs { output.push_str(&format!( - "- {} attempt {}: status={}, phase={}, activeLease={}, owner={}, liveness={}\n", + "- {} attempt {}: status={}, phase={}, runLease={}, owner={}, liveness={}\n", run.run_id, run.attempt_number, run.status, run.phase, - run.active_lease, + run.run_lease, run.ownership_state, run.execution_liveness )); diff --git a/apps/decodex/src/orchestrator/operator_dashboard.html b/apps/decodex/src/orchestrator/operator_dashboard.html index d72f7987e..9b199e772 100644 --- a/apps/decodex/src/orchestrator/operator_dashboard.html +++ b/apps/decodex/src/orchestrator/operator_dashboard.html @@ -824,7 +824,7 @@ transform var(--medium) var(--ease); } - #active-panel { + #current-lanes-panel { background: transparent; } @@ -3733,15 +3733,15 @@

Decodex

> Execution -
+
-

Running Lanes

+

Current Lanes

-

+

-
+
@@ -3939,7 +3939,7 @@

Run History

panels: { projects: document.getElementById("projects-panel"), accountPool: document.getElementById("account-pool-panel"), - active: document.getElementById("active-panel"), + currentLanes: document.getElementById("current-lanes-panel"), programs: document.getElementById("programs-panel"), queue: document.getElementById("queue-panel"), recent: document.getElementById("recent-panel"), @@ -3957,8 +3957,8 @@

Run History

accountPool: document.getElementById("account-pool"), queuedCandidates: document.getElementById("queued-candidates"), queuedMeta: document.getElementById("queued-meta"), - activeRuns: document.getElementById("active-runs"), - activeRunsMeta: document.getElementById("active-runs-meta"), + currentLanes: document.getElementById("current-lanes"), + currentLanesMeta: document.getElementById("current-lanes-meta"), executionPrograms: document.getElementById("execution-programs"), programsMeta: document.getElementById("programs-meta"), recentRuns: document.getElementById("recent-runs"), @@ -3973,9 +3973,9 @@

Run History

let dashboardSocket = null; let dashboardSocketReconnectTimer = null; let dashboardLocalClockTimer = null; - let dashboardLiveActiveRuns = []; + let dashboardLiveCurrentLanes = []; let dashboardLiveRunActivitySeen = false; - let dashboardLiveActiveRunsComplete = true; + let dashboardLiveCurrentLanesComplete = true; let dashboardLiveAccounts = null; let dashboardLiveAccountControl = null; let dashboardStreamState = { @@ -4015,7 +4015,7 @@

Run History

{ marker: "aftercare", panels: ["review", "worktrees", "recent"] }, ]; const COPY = { - runningLane: "Running Lanes", + currentLane: "Current Lanes", runningInline: "Running here", runningInlineMeta: "already running", runningInlineMetaPlural: "already running", @@ -4603,7 +4603,7 @@

Run History

} function isPostReviewBlocker(lane) { - if (lane?.shadowed_by_active_run === true) { + if (lane?.shadowed_by_current_lane === true) { return false; } @@ -4635,7 +4635,7 @@

Run History

} function toneForQueuedCandidate(candidate) { - if (candidate.display_classification === "owned_active") { + if (candidate.display_classification === "leased_run") { return "tone-run"; } if (candidate.classification === "ready") { @@ -4661,7 +4661,7 @@

Run History

function queuedCandidateRank(candidate) { switch (candidate.display_classification || candidate.classification) { - case "owned_active": + case "leased_run": return 0; case "ready": return 1; @@ -4690,7 +4690,7 @@

Run History

} function summarizeQueuedCandidate(candidate) { - if (candidate.display_classification === "owned_active") { + if (candidate.display_classification === "leased_run") { return `Shown in ${COPY.runningLane}; excluded from queue.`; } if ( @@ -4718,7 +4718,7 @@

Run History

} function queuedCandidateStatusText(candidate) { - if (candidate.display_classification === "owned_active") { + if (candidate.display_classification === "leased_run") { return COPY.runningInline; } if (candidate.reason === "global_concurrency_exhausted") { @@ -4729,7 +4729,7 @@

Run History

} function queuedCandidateReasonText(candidate) { - if (candidate.display_classification === "owned_active") { + if (candidate.display_classification === "leased_run") { return "running lane claim"; } if ( @@ -5388,8 +5388,8 @@

Run History

return `run:${run.run_id}:more-fields`; } - function activeRunRenderKey(run) { - return `active-run:${run.run_id || issueDisplayKey(run)}`; + function currentLaneRenderKey(run) { + return `current-lane:${run.run_id || issueDisplayKey(run)}`; } function detailsOpenAttribute(detailKey) { @@ -5749,10 +5749,10 @@

Run History

} function rawSessionHistoryRuns(snapshot) { - const activeRunIds = new Set((snapshot?.active_runs ?? []).map((run) => run.run_id)); - const activeIssueKeys = new Set((snapshot?.active_runs ?? []).flatMap(issueIdentityKeys)); + const currentLaneIds = new Set((snapshot?.current_lanes ?? []).map((run) => run.run_id)); + const currentLaneIssueKeys = new Set((snapshot?.current_lanes ?? []).flatMap(issueIdentityKeys)); return (snapshot?.recent_runs ?? []).filter( - (run) => !activeRunIds.has(run.run_id) && !issueMatchesKeySet(run, activeIssueKeys), + (run) => !currentLaneIds.has(run.run_id) && !issueMatchesKeySet(run, currentLaneIssueKeys), ); } @@ -5988,7 +5988,7 @@

Run History

} function runOwnershipSummary(run) { - return displayToken(run.ownership_state || (runCountsAsRunning(run) ? "owned_active" : "unknown")); + return displayToken(run.ownership_state || (runCountsAsRunning(run) ? "leased_run" : "unknown")); } function runLivenessStateSummary(run) { @@ -6008,8 +6008,23 @@

Run History

return conditions.length ? conditions.map(displayToken).join(", ") : "none"; } + function runContinuationRecoverySummary(run) { + const recovery = run.continuation_recovery; + if (!recovery) { + return "none"; + } + + const count = `${recovery.recovery_count ?? 0}/${recovery.automatic_continuation_limit ?? 0}`; + const exceeded = recovery.budget_exceeded ? "budget exceeded" : "within budget"; + const message = recovery.source_error_message + ? `; ${recovery.source_error_message}` + : ""; + + return `${displayToken(recovery.state)} · ${displayToken(recovery.source_phase)} -> ${displayToken(recovery.next_phase)} · ${displayToken(recovery.source_error_class)} · ${count} · ${exceeded}${message}`; + } + function runQueueLeaseSummary(run) { - const leaseState = run.queue_lease_state || (run.active_lease ? "held" : "not_held"); + const leaseState = run.queue_lease_state || (run.run_lease ? "held" : "not_held"); if (leaseState === "held") { return "held"; @@ -6147,7 +6162,7 @@

Run History

return issueIdentityKeys(item).some((key) => keySet.has(key)); } - function activeRunFreshness(run) { + function currentLaneFreshness(run) { if (run?.last_run_activity_at) { return { label: "Lane activity", @@ -6180,8 +6195,8 @@

Run History

}; } - function activeRunFreshnessFact(run, formatter = formatTimestamp) { - const freshness = activeRunFreshness(run); + function currentLaneFreshnessFact(run, formatter = formatTimestamp) { + const freshness = currentLaneFreshness(run); return [ freshness.label, freshness.timestamp ? formatter(freshness.timestamp) : "not captured", @@ -6344,9 +6359,9 @@

Run History

return missing > 0 ? `${captured}/${attempts} captured · ${missing} missing` : `${captured}/${attempts} captured`; } - function activeRunTelemetryFacts(run) { + function currentLaneTelemetryFacts(run) { const facts = []; - const freshness = activeRunFreshness(run); + const freshness = currentLaneFreshness(run); const focus = protocolActivityFocus(run); if (freshness.timestamp) { @@ -6377,7 +6392,7 @@

Run History

} function renderRunTelemetryMetaItems(run) { - const facts = activeRunTelemetryFacts(run); + const facts = currentLaneTelemetryFacts(run); if (!facts.length) { return ""; @@ -6386,7 +6401,7 @@

Run History

return facts.map(([label, value]) => renderRunMetaFact(label, value)).join(""); } - function activeRunLifecycleMetrics(run, summary = childAgentActivity(run)) { + function currentLaneLifecycleMetrics(run, summary = childAgentActivity(run)) { const provided = run?.lifecycle_metrics || {}; const providedPhases = Array.isArray(provided.phases) ? provided.phases.map(normalizeLifecyclePhaseMetrics) @@ -8082,7 +8097,7 @@

Run History

const accountsByIdentity = new Map(); const configuredAccounts = configuredDashboardAccounts(snapshot); const runs = [ - ...(snapshot?.active_runs ?? []).map((run) => [run, true]), + ...(snapshot?.current_lanes ?? []).map((run) => [run, true]), ...(snapshot?.recent_runs ?? []).map((run) => [run, false]), ]; @@ -8096,7 +8111,7 @@

Run History

accountsByIdentity.set(identity, existing ? { ...existing, ...account } : account); } - for (const [run, isActiveRun] of runs) { + for (const [run, isCurrentLane] of runs) { const selected = codexAccount(run); for (const account of codexAccounts(run)) { const identity = codexAccountIdentity(account); @@ -8112,7 +8127,7 @@

Run History

...existing, ...account, status: - isActiveRun && selectedForRun + isCurrentLane && selectedForRun ? "selected" : accountStatus === "selected" ? "available" @@ -8746,7 +8761,7 @@

Run History

return `${escapeHtml(String(value))}`; } - function childAgentContextRows(run, summary, lifecycle = activeRunLifecycleMetrics(run, summary)) { + function childAgentContextRows(run, summary, lifecycle = currentLaneLifecycleMetrics(run, summary)) { const rows = []; const contextFacts = childAgentContextFacts(summary); const overview = renderChildLifecycleOverview(lifecycle, contextFacts); @@ -8769,7 +8784,7 @@

Run History

return ""; } - const lifecycle = activeRunLifecycleMetrics(run, summary); + const lifecycle = currentLaneLifecycleMetrics(run, summary); const buckets = (lifecycle.buckets || []).length ? lifecycle.buckets : childAgentBuckets(summary); const totalWall = Math.max( 1, @@ -8912,7 +8927,7 @@

Run History

return `Latest run ${displayToken(run.status || run.phase)}; lifecycle cost is grouped by phase.`; } if (isSuccessfulTerminalRun(run)) { - return "Finished; no active lane."; + return "Finished; no current lane."; } if (run.status === "interrupted") { return "Stopped before completion; replaced after a later success."; @@ -9039,7 +9054,7 @@

Run History

`; } - function activeRunSummary(run) { + function currentLaneSummary(run) { if (runTelemetryMissingNeedsAttention(run)) { if (runHasChildAgentActivity(run)) { return `Metadata missing ${formatDuration(run.idle_for_seconds)}.`; @@ -9073,7 +9088,7 @@

Run History

if (run.wait_reason) { return `Waiting on ${displayToken(run.wait_reason)}.`; } - if (!run.active_lease) { + if (!run.run_lease) { return "No queue lease."; } const childCurrent = childAgentCurrentSummary(childAgentActivity(run)); @@ -9141,7 +9156,7 @@

Run History

if (run.wait_reason) { return displayToken(run.wait_reason); } - if (!run.active_lease && runCountsAsRunning(run)) { + if (!run.run_lease && runCountsAsRunning(run)) { return "live_no_queue_lease"; } if (run.thread_status) { @@ -9156,7 +9171,7 @@

Run History

} return ( - (snapshot.active_runs?.length ?? 0) === 0 && + (snapshot.current_lanes?.length ?? 0) === 0 && (snapshot.recent_runs?.length ?? 0) === 0 && (snapshot.worktrees?.length ?? 0) === 0 && (snapshot.post_review_lanes?.length ?? 0) === 0 @@ -9442,31 +9457,31 @@

Run History

} function buildDerivedState(snapshot) { - const activeRuns = snapshot?.active_runs ?? []; + const currentLanes = snapshot?.current_lanes ?? []; const recentRuns = snapshot?.recent_runs ?? []; const historyRuns = sessionHistoryRuns(snapshot); const executionPrograms = snapshot?.execution_programs ?? []; const postReviewLanes = snapshot?.post_review_lanes ?? []; const postReviewIssueKeys = new Set(postReviewLanes.flatMap(issueIdentityKeys)); - const activeRunByIssue = new Map(); - for (const run of activeRuns) { + const currentLaneByIssue = new Map(); + for (const run of currentLanes) { for (const key of issueIdentityKeys(run)) { - if (!activeRunByIssue.has(key)) { - activeRunByIssue.set(key, run); + if (!currentLaneByIssue.has(key)) { + currentLaneByIssue.set(key, run); } } } const queuedCandidates = [...(snapshot?.queued_candidates ?? [])] .map((candidate) => { - const activeRun = issueIdentityKeys(candidate) - .map((key) => activeRunByIssue.get(key)) + const currentLane = issueIdentityKeys(candidate) + .map((key) => currentLaneByIssue.get(key)) .find(Boolean); - if (activeRun) { + if (currentLane) { return { ...candidate, - display_classification: "owned_active", - active_run_id: activeRun.run_id, + display_classification: "leased_run", + current_lane_run_id: currentLane.run_id, }; } @@ -9475,13 +9490,13 @@

Run History

.sort(compareQueuedCandidates); const queueBacklogCandidates = queuedCandidates.filter( (candidate) => - candidate.display_classification !== "owned_active" && + candidate.display_classification !== "leased_run" && candidate.classification !== "closed" && !issueMatchesKeySet(candidate, postReviewIssueKeys), ); const reviewOwnedQueueCount = queuedCandidates.filter( (candidate) => - candidate.display_classification !== "owned_active" && + candidate.display_classification !== "leased_run" && candidate.classification !== "closed" && issueMatchesKeySet(candidate, postReviewIssueKeys), ).length; @@ -9499,7 +9514,7 @@

Run History

const readyItems = []; const attentionItems = []; - for (const run of activeRuns) { + for (const run of currentLanes) { const tone = toneForRun(run); const issueKey = issueDisplayKey(run); @@ -9523,7 +9538,7 @@

Run History

["Codex thread", runThreadSummary(run)], ["Thread flags", runThreadFlagSummary(run)], ["Process", runProcessSummary(run)], - activeRunFreshnessFact(run), + currentLaneFreshnessFact(run), ["Protocol idle", formatDuration(run.protocol_idle_for_seconds)], ["Last progress", formatTimestamp(run.last_progress_at)], [COPY.protocolEvent, protocolEventSummary(run)], @@ -9552,7 +9567,7 @@

Run History

facts: [ ["Codex thread", runThreadSummary(run)], ["Thread flags", runThreadFlagSummary(run)], - activeRunFreshnessFact(run), + currentLaneFreshnessFact(run), ["Lane idle", formatDuration(run.idle_for_seconds)], ["Protocol idle", formatDuration(run.protocol_idle_for_seconds)], [COPY.protocolEvent, protocolEventSummary(run)], @@ -9565,23 +9580,19 @@

Run History

for (const lane of postReviewLanes) { const tone = toneForLane(lane); - const activeRun = issueIdentityKeys(lane) - .map((key) => activeRunByIssue.get(key)) + const currentLane = issueIdentityKeys(lane) + .map((key) => currentLaneByIssue.get(key)) .find(Boolean); - const shadowedByActiveRun = - lane.shadowed_by_active_run === true || - (typeof lane.shadowed_by_active_run !== "boolean" && - activeRun && - lane.classification === "needs_review_repair"); + const shadowedByCurrentLane = lane.shadowed_by_current_lane === true; const issueKey = issueDisplayKey(lane); - if (shadowedByActiveRun) { - const activeRunFacts = activeRun + if (shadowedByCurrentLane) { + const currentLaneFacts = currentLane ? [ - ["Run", activeRun.run_id], + ["Run", currentLane.run_id], [ "Operation", - displayToken(activeRun.current_operation || activeRun.phase), + displayToken(currentLane.current_operation || currentLane.phase), ], ] : [["Run", "active"]]; @@ -9592,9 +9603,9 @@

Run History

issue: issueKey, title: "Repair running", summary: "", - status: activeRun ? `run ${displayToken(activeRun.phase)}` : "active run", + status: currentLane ? `run ${displayToken(currentLane.phase)}` : "current lane", facts: [ - ...activeRunFacts, + ...currentLaneFacts, ["Checks", compactStateToken(lane.check_state)], ["Threads", reviewThreadToken(lane.unresolved_review_threads)], ["PR", optionalCardToken(lane.pr_url)], @@ -9682,7 +9693,7 @@

Run History

).length; const cleanupCount = cleanupIssueKeys.size; const runningAttentionCount = attentionItems.filter((item) => item.scope === "Running").length; - const liveRuns = activeRuns.filter(runCountsAsRunning).length; + const liveRuns = currentLanes.filter(runCountsAsRunning).length; const intakeAttentionCount = queueBacklogCandidates.filter(queuedCandidateNeedsAttention).length; const programAttentionCount = executionPrograms.filter((program) => ["attention", "blocked", "stale"].includes(program.status), @@ -9699,14 +9710,14 @@

Run History

return { projects: snapshot?.projects ?? [], liveRuns, - activeLaneCount: activeRuns.length, - liveLeases: activeRuns.filter((run) => run.active_lease).length, + currentLaneCount: currentLanes.length, + liveLeases: currentLanes.filter((run) => run.run_lease).length, readyCount: readyItems.length, queuedCandidates, queueBacklogCandidates, queuedReady: queueBacklogCandidates.filter((candidate) => candidate.classification === "ready").length, queuedWaiting: queueBacklogCandidates.filter((candidate) => candidate.classification === "waiting").length, - queuedActiveOwned: queuedCandidates.filter((candidate) => candidate.display_classification === "owned_active").length, + queuedActiveOwned: queuedCandidates.filter((candidate) => candidate.display_classification === "leased_run").length, queuedBlocked: queueBacklogCandidates.filter((candidate) => candidate.classification === "blocked").length, queuedBlockedWithoutAttention: queueBacklogCandidates.filter( (candidate) => @@ -9774,7 +9785,7 @@

Run History

function projectHasActiveWork(project) { const workCount = - (project.active_run_count ?? 0) + + (project.current_lane_count ?? 0) + (project.queued_candidate_count ?? 0) + (project.waiting_lane_count ?? 0) + (project.attention_count ?? 0) + @@ -9786,7 +9797,7 @@

Run History

} function projectRunningLaneCount(project) { - return project.running_lane_count ?? project.active_run_count ?? 0; + return project.running_lane_count ?? project.current_lane_count ?? 0; } function activeProjects(projects) { @@ -10275,17 +10286,17 @@

Run History

return; } - const activeLaneCount = (snapshot.active_runs ?? []).length; + const currentLaneCount = (snapshot.current_lanes ?? []).length; const retainedCount = (snapshot.post_review_lanes ?? []).length; setFlowCounts( pluralize(derived.queueBacklogCandidates.length, "issue"), - pluralize(activeLaneCount, "lane"), + pluralize(currentLaneCount, "lane"), pluralize(retainedCount, "PR"), pluralize(derived.readyCount, "PR"), ); setFlowActivity({ queue: derived.queueBacklogCandidates.length > 0, - run: activeLaneCount > 0, + run: currentLaneCount > 0, review: derived.reviewBlockerCount > 0 || derived.reviewWaitingCount > 0 || @@ -10599,31 +10610,31 @@

${escapeHtml(item.title)}

); } - function renderActiveRuns(snapshot, derived) { - const runs = snapshot?.active_runs ?? []; + function renderCurrentLanes(snapshot, derived) { + const runs = snapshot?.current_lanes ?? []; setPanelMeta( - nodes.activeRunsMeta, + nodes.currentLanesMeta, runningLaneMetaText(derived), derived.runningAttentionCount ? "attention" : "", ); if (!runs.length) { - renderRoutineEmptyList(nodes.activeRuns); + renderRoutineEmptyList(nodes.currentLanes); return; } renderStableList( - nodes.activeRuns, + nodes.currentLanes, runs .map((run) => { const tone = toneForRun(run); const statusBits = [statusLabel(runStageLabel(run), tone)]; const detailKey = runDetailKey(run); - const renderKey = activeRunRenderKey(run); + const renderKey = currentLaneRenderKey(run); const issueKey = issueDisplayKey(run); const issueTitle = runIssueTitle(run, derived); - const summary = activeRunSummary(run); - if (run.ownership_state && run.ownership_state !== "owned_active") { + const summary = currentLaneSummary(run); + if (run.ownership_state && run.ownership_state !== "leased_run") { statusBits.push(inlineStatusFact("Owner", displayToken(run.ownership_state))); } if (run.policy_state && !["allowed", "review_findings", "review_pending"].includes(run.policy_state)) { @@ -10704,6 +10715,7 @@

${escapeHtml(issueTitle)}

${field("Terminalization", runTerminalizationSummary(run))} ${field("Lane next action", capturedValue(run.lane_control_next_action))} ${field("Lane conditions", runLaneControlConditionsSummary(run))} + ${field("Continuation recovery", runContinuationRecoverySummary(run))} ${field("Loop", run.loop_status?.summary || "none")} ${field("Review loop", loopStatusFacts(run.loop_status).map(([label, value]) => `${label}: ${value}`).join("; ") || "none")} ${field("Model", runModelSummary(run))} @@ -10856,7 +10868,7 @@

${escapeHtml(title)}

function worktreeRoleMeta(worktree, snapshot) { const hygiene = worktree.hygiene; - const activeMatch = (snapshot?.active_runs ?? []).find( + const currentLaneMatch = (snapshot?.current_lanes ?? []).find( (run) => (run.worktree_path === worktree.worktree_path || run.branch_name === worktree.branch_name || @@ -10870,11 +10882,11 @@

${escapeHtml(title)}

lane.issue_identifier === worktree.issue_id, ); - if (activeMatch) { + if (currentLaneMatch) { return { sortRank: 0, tone: "tone-run", - label: "active lane", + label: "current lane", summary: worktree.ownership_reason || "Leased by a running lane.", }; } @@ -10924,11 +10936,11 @@

${escapeHtml(title)}

"Owned by Intake Queue attention; recover there before cleanup.", }; } - if (worktree.ownership === "active_lane") { + if (worktree.ownership === "current_lane") { return { sortRank: 0, tone: "tone-run", - label: "active lane", + label: "current lane", summary: worktree.ownership_reason || "Leased by a running lane.", }; } @@ -11265,20 +11277,20 @@

${escapeHtml(worktree.branch_name)}

function dashboardRunShouldRetainDuringPartialActivity(run) { return ( - run?.active_lease === true || + run?.run_lease === true || run?.process_alive === true || runHasFreshExecution(run) || runCountsAsRunning(run) ); } - function mergeDashboardActiveRuns(snapshot, activeRunRows, activeRunsComplete = true) { - const snapshotActiveRuns = snapshot.active_runs || []; - const snapshotRunLookup = dashboardRunLookup(snapshotActiveRuns); + function mergeDashboardCurrentLanes(snapshot, currentLaneRows, currentLanesComplete = true) { + const snapshotCurrentLanes = snapshot.current_lanes || []; + const snapshotRunLookup = dashboardRunLookup(snapshotCurrentLanes); const seenSnapshotRuns = new Set(); const mergedRuns = []; - for (const activityRun of activeRunRows) { + for (const activityRun of currentLaneRows) { const snapshotRun = dashboardRunIdentityKeys(activityRun) .map((key) => snapshotRunLookup.get(key)) .find(Boolean); @@ -11290,9 +11302,9 @@

${escapeHtml(worktree.branch_name)}

mergedRuns.push(mergeDashboardRunRecord(snapshotRun, activityRun)); } - for (const snapshotRun of snapshotActiveRuns) { + for (const snapshotRun of snapshotCurrentLanes) { if ( - !activeRunsComplete && + !currentLanesComplete && !seenSnapshotRuns.has(snapshotRun) && dashboardRunShouldRetainDuringPartialActivity(snapshotRun) ) { @@ -11303,20 +11315,20 @@

${escapeHtml(worktree.branch_name)}

return mergedRuns; } - function mergeDashboardRunActivity(snapshot, activeRuns, activityPayload = {}) { - const activeRunRows = activeRuns + function mergeDashboardRunActivity(snapshot, currentLanes, activityPayload = {}) { + const currentLaneRows = currentLanes .filter((run) => run && typeof run === "object") .map((run) => ({ ...run })); - const activeRunsComplete = - activityPayload.activeRunsComplete !== false && - activityPayload.activeRunScope !== "filtered"; - const mergedActiveRuns = mergeDashboardActiveRuns( + const currentLanesComplete = + activityPayload.currentLanesComplete !== false && + activityPayload.currentLaneScope !== "filtered"; + const mergedCurrentLanes = mergeDashboardCurrentLanes( snapshot, - activeRunRows, - activeRunsComplete, + currentLaneRows, + currentLanesComplete, ); const activeById = new Map( - mergedActiveRuns + mergedCurrentLanes .filter((run) => run.run_id) .map((run) => [run.run_id, run]), ); @@ -11327,19 +11339,19 @@

${escapeHtml(worktree.branch_name)}

} seenRecentIds.add(run.run_id); - const activeRun = activeById.get(run.run_id); + const currentLane = activeById.get(run.run_id); - return activeRun ? mergeDashboardRunRecord(run, activeRun) : run; + return currentLane ? mergeDashboardRunRecord(run, currentLane) : run; }); - for (const run of mergedActiveRuns) { + for (const run of mergedCurrentLanes) { if (run.run_id && !seenRecentIds.has(run.run_id)) { recentRuns.unshift(run); seenRecentIds.add(run.run_id); } } - const activeCountsByProject = mergedActiveRuns.reduce((counts, run) => { + const currentLaneCountsByProject = mergedCurrentLanes.reduce((counts, run) => { const projectId = run.project_id || snapshot.project_id; if (!projectId) { return counts; @@ -11349,7 +11361,7 @@

${escapeHtml(worktree.branch_name)}

return counts; }, new Map()); - const runningCountsByProject = mergedActiveRuns.reduce((counts, run) => { + const runningCountsByProject = mergedCurrentLanes.reduce((counts, run) => { const projectId = run.project_id || snapshot.project_id; if (!projectId || !runCountsAsRunning(run)) { return counts; @@ -11368,7 +11380,7 @@

${escapeHtml(worktree.branch_name)}

return { ...project, - active_run_count: activeCountsByProject.get(project.project_id) || 0, + current_lane_count: currentLaneCountsByProject.get(project.project_id) || 0, running_lane_count: runningCountsByProject.get(project.project_id) || 0, }; }) @@ -11385,7 +11397,7 @@

${escapeHtml(worktree.branch_name)}

...snapshot, account_control: accountControl, accounts, - active_runs: mergedActiveRuns, + current_lanes: mergedCurrentLanes, recent_runs: recentRuns, projects, }; @@ -11395,7 +11407,7 @@

${escapeHtml(worktree.branch_name)}

if (!dashboardLiveRunActivitySeen) { return false; } - if (dashboardLiveActiveRuns.length || !dashboardLiveActiveRunsComplete) { + if (dashboardLiveCurrentLanes.length || !dashboardLiveCurrentLanesComplete) { return true; } @@ -11405,12 +11417,12 @@

${escapeHtml(worktree.branch_name)}

function clearDashboardLiveRunActivityOverlayIfCompleteEmpty() { if ( dashboardLiveRunActivitySeen && - dashboardLiveActiveRunsComplete && - !dashboardLiveActiveRuns.length + dashboardLiveCurrentLanesComplete && + !dashboardLiveCurrentLanes.length ) { dashboardLiveRunActivitySeen = false; - dashboardLiveActiveRuns = []; - dashboardLiveActiveRunsComplete = true; + dashboardLiveCurrentLanes = []; + dashboardLiveCurrentLanesComplete = true; dashboardLiveAccounts = null; dashboardLiveAccountControl = null; } @@ -11425,15 +11437,15 @@

${escapeHtml(worktree.branch_name)}

return snapshot; } - return mergeDashboardRunActivity(snapshot, dashboardLiveActiveRuns, { + return mergeDashboardRunActivity(snapshot, dashboardLiveCurrentLanes, { accountControl: dashboardLiveAccountControl, accounts: dashboardLiveAccounts, - activeRunsComplete: dashboardLiveActiveRunsComplete, + currentLanesComplete: dashboardLiveCurrentLanesComplete, }); } function applyDashboardRunActivity(payload) { - if (!Array.isArray(payload?.activeRuns)) { + if (!Array.isArray(payload?.currentLanes)) { updateDashboardStreamState({ connected: true, error: false, @@ -11443,12 +11455,12 @@

${escapeHtml(worktree.branch_name)}

return; } - dashboardLiveActiveRuns = payload.activeRuns + dashboardLiveCurrentLanes = payload.currentLanes .filter((run) => run && typeof run === "object") .map((run) => ({ ...run })); dashboardLiveRunActivitySeen = true; - dashboardLiveActiveRunsComplete = - payload.activeRunsComplete !== false && payload.activeRunScope !== "filtered"; + dashboardLiveCurrentLanesComplete = + payload.currentLanesComplete !== false && payload.currentLaneScope !== "filtered"; dashboardLiveAccounts = Array.isArray(payload.accounts) && payload.accounts.length ? payload.accounts.filter(Boolean).map((account) => ({ ...account })) : null; @@ -11612,9 +11624,9 @@

${escapeHtml(worktree.branch_name)}

syncDashboardSubscriptionToSocket(); }; dashboardSocket.onclose = () => { - dashboardLiveActiveRuns = []; + dashboardLiveCurrentLanes = []; dashboardLiveRunActivitySeen = false; - dashboardLiveActiveRunsComplete = true; + dashboardLiveCurrentLanesComplete = true; dashboardLiveAccounts = null; dashboardLiveAccountControl = null; updateDashboardStreamState({ @@ -11624,9 +11636,9 @@

${escapeHtml(worktree.branch_name)}

scheduleDashboardSocketReconnect(); }; dashboardSocket.onerror = () => { - dashboardLiveActiveRuns = []; + dashboardLiveCurrentLanes = []; dashboardLiveRunActivitySeen = false; - dashboardLiveActiveRunsComplete = true; + dashboardLiveCurrentLanesComplete = true; dashboardLiveAccounts = null; dashboardLiveAccountControl = null; updateDashboardStreamState({ @@ -11666,7 +11678,7 @@

${escapeHtml(worktree.branch_name)}

renderFlow(snapshot, derived); renderProjects(snapshot, derived); renderAccountPool(snapshot); - renderActiveRuns(snapshot, derived); + renderCurrentLanes(snapshot, derived); renderExecutionPrograms(snapshot, derived); renderQueuedCandidates( nodes.queuedCandidates, diff --git a/apps/decodex/src/orchestrator/operator_http.rs b/apps/decodex/src/orchestrator/operator_http.rs index 968261386..4cdd5e0a4 100644 --- a/apps/decodex/src/orchestrator/operator_http.rs +++ b/apps/decodex/src/orchestrator/operator_http.rs @@ -566,7 +566,7 @@ fn build_operator_run_activity_event( let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); let account_control = global_codex_account_control_status(); let mut accounts = Vec::new(); - let mut active_runs = Vec::new(); + let mut current_lanes = Vec::new(); for registration in state_store.list_projects()? { let project = match ServiceConfig::from_path(registration.config_path()) { @@ -582,58 +582,47 @@ fn build_operator_run_activity_event( continue; }, }; - let (leased_runs, recent_runs) = - state_store.list_project_runs(project.service_id(), DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT)?; - let loop_evidence = state_store.project_loop_evidence_snapshot(project.service_id())?; - let project_display_name = operator_project_display_name(&project); - let recent_runs = recent_runs - .into_iter() - .map(|run| { - operator_run_status( - &project, - &loop_evidence, - &project_display_name, - run, - now_unix_epoch, - ) - }) - .collect::>>()?; - let project_active_runs = operator_active_run_statuses( + let workflow = match WorkflowDocument::from_path(project.workflow_path()) { + Ok(workflow) => workflow, + Err(error) => { + tracing::debug!( + project_id = project.service_id(), + workflow_path = %project.workflow_path().display(), + ?error, + "Skipped dashboard run activity for a project with an unreadable workflow." + ); + + continue; + }, + }; + let project_snapshot = build_operator_state_snapshot_without_live_observers( &project, - &loop_evidence, - &project_display_name, - leased_runs, - &recent_runs, - now_unix_epoch, + &workflow, + state_store, + DEFAULT_OPERATOR_DASHBOARD_RUN_LIMIT, )?; - if project_active_runs.is_empty() { + if project_snapshot.current_lanes.is_empty() { continue; } - let mut account_warnings = Vec::new(); - - accounts.extend(codex_account_activity_summaries( - &project, - &mut account_warnings, - AccountActivityMode::Snapshot, - )); - active_runs.extend(project_active_runs); + accounts.extend(project_snapshot.accounts); + current_lanes.extend(project_snapshot.current_lanes); } let fingerprint_payload = dashboard_run_activity_fingerprint_payload( &account_control, &accounts, - &active_runs, + ¤t_lanes, ); let fingerprint = serde_json::to_vec(&fingerprint_payload)?; let payload = json!({ "emittedAtUnixEpoch": now_unix_epoch, "accountControl": &account_control, "accounts": &accounts, - "activeRuns": &active_runs, - "activeRunsComplete": true, - "activeRunScope": "complete", + "currentLanes": ¤t_lanes, + "currentLanesComplete": true, + "currentLaneScope": "complete", }); Ok(DashboardRunActivityEvent { @@ -645,14 +634,14 @@ fn build_operator_run_activity_event( fn dashboard_run_activity_fingerprint_payload( account_control: &OperatorCodexAccountControlStatus, accounts: &[CodexAccountActivitySummary], - active_runs: &[OperatorRunStatus], + current_lanes: &[OperatorRunStatus], ) -> Value { let mut fingerprint_payload = json!({ "accountControl": account_control, "accounts": accounts, - "activeRuns": active_runs, - "activeRunsComplete": true, - "activeRunScope": "complete", + "currentLanes": current_lanes, + "currentLanesComplete": true, + "currentLaneScope": "complete", }); strip_dashboard_run_activity_volatile_fields(&mut fingerprint_payload); @@ -720,7 +709,7 @@ fn write_cached_dashboard_run_activity_event( subscription: &DashboardClientSubscription, ) { match dashboard_events.cached_run_activity_event(subscription) { - Some(event) if dashboard_run_activity_event_has_active_runs(&event) => { + Some(event) if dashboard_run_activity_event_has_current_lanes(&event) => { if let Err(error) = write_dashboard_websocket_event(stream, event.event_type, &event.payload) { tracing::warn!( @@ -733,8 +722,8 @@ fn write_cached_dashboard_run_activity_event( } } -fn dashboard_run_activity_event_has_active_runs(event: &DashboardBroadcastEvent) -> bool { - event.payload.get("activeRuns").and_then(Value::as_array).is_some_and(|runs| !runs.is_empty()) +fn dashboard_run_activity_event_has_current_lanes(event: &DashboardBroadcastEvent) -> bool { + event.payload.get("currentLanes").and_then(Value::as_array).is_some_and(|runs| !runs.is_empty()) } fn write_dashboard_websocket_event( @@ -1185,7 +1174,7 @@ fn dashboard_event_for_subscription( return Some(event.clone()); } - let active_runs = event.payload.get("activeRuns").and_then(Value::as_array).map(|runs| { + let current_lanes = event.payload.get("currentLanes").and_then(Value::as_array).map(|runs| { runs.iter() .filter(|run| dashboard_run_matches_subscription(run, subscription)) .cloned() @@ -1193,9 +1182,9 @@ fn dashboard_event_for_subscription( })?; let mut payload = event.payload.clone(); - payload["activeRuns"] = Value::Array(active_runs); - payload["activeRunsComplete"] = Value::Bool(false); - payload["activeRunScope"] = Value::String(String::from("filtered")); + payload["currentLanes"] = Value::Array(current_lanes); + payload["currentLanesComplete"] = Value::Bool(false); + payload["currentLaneScope"] = Value::String(String::from("filtered")); Some(DashboardBroadcastEvent { event_type: event.event_type, payload }) } diff --git a/apps/decodex/src/orchestrator/reconciliation.rs b/apps/decodex/src/orchestrator/reconciliation.rs index 72985c951..abf00f260 100644 --- a/apps/decodex/src/orchestrator/reconciliation.rs +++ b/apps/decodex/src/orchestrator/reconciliation.rs @@ -1,12 +1,12 @@ #[cfg(test)] -fn inspect_active_run_reconciliation_at( +fn inspect_run_lease_reconciliation_at( tracker: &T, project: &ServiceConfig, workflow: &WorkflowDocument, state_store: &StateStore, active_workflow_override: Option>, now_unix_epoch: i64, -) -> Result> +) -> Result> where T: IssueTracker, { @@ -30,7 +30,7 @@ where continue; }; let worktree_mapping = state_store.worktree_for_issue(&issue.id)?; - let action_workflow = active_reconciliation_workflow_for_lease( + let action_workflow = run_lease_reconciliation_workflow( workflow, active_workflow_override, &issue, @@ -47,11 +47,11 @@ where if let Some(disposition) = superseded_run_disposition(state_store, &run_attempt)? { Some(disposition) } else if !retained_closeout && is_terminal_issue(&issue, action_workflow) { - Some(ActiveRunDisposition::Terminal) + Some(RunLeaseDisposition::Terminal) } else if !retained_closeout - && is_issue_nonactive_for_run(&issue, action_workflow) + && is_issue_not_dispatchable_for_run(&issue, action_workflow) { - Some(ActiveRunDisposition::NonActive) + Some(RunLeaseDisposition::NotDispatchable) } else if let Some(idle_for) = stalled_idle_duration( state_store, &run_attempt, @@ -63,18 +63,18 @@ where &run_attempt, worktree_mapping.as_ref(), )? { - Some(ActiveRunDisposition::RetainedReviewComplete) + Some(RunLeaseDisposition::RetainedReviewComplete) } else if stalled_run_has_retained_partial_progress(worktree_mapping.as_ref()) { - Some(ActiveRunDisposition::StalledRetainedPartialProgress { idle_for }) + Some(RunLeaseDisposition::StalledRetainedPartialProgress { idle_for }) } else { - Some(ActiveRunDisposition::Stalled { idle_for }) + Some(RunLeaseDisposition::Stalled { idle_for }) } } else { None }; if let Some(disposition) = disposition { - actions.push(ActiveRunReconciliation { + actions.push(RunLeaseReconciliation { issue: issue.clone(), run_attempt, worktree_mapping, @@ -94,7 +94,7 @@ fn inspect_exited_daemon_child_reconciliation( state_store: &StateStore, issue_id: &str, run_id: &str, -) -> Result> +) -> Result> where T: IssueTracker, { @@ -117,7 +117,7 @@ fn inspect_exited_daemon_child_reconciliation_at( issue_id: &str, run_id: &str, now_unix_epoch: i64, -) -> Result> +) -> Result> where T: IssueTracker, { @@ -130,7 +130,7 @@ where let worktree_mapping = state_store.worktree_for_issue(issue_id)?; if let Some(disposition) = superseded_run_disposition(state_store, &run_attempt)? { - return Ok(vec![ActiveRunReconciliation { + return Ok(vec![RunLeaseReconciliation { issue, run_attempt, worktree_mapping, @@ -139,7 +139,7 @@ where }]); } - if run_attempt.status() != "failed" || !is_issue_active_for_run(&issue, workflow) { + if run_attempt.status() != "failed" || !is_issue_in_progress_for_run(&issue, workflow) { return Ok(Vec::new()); } @@ -153,12 +153,12 @@ where return Ok(Vec::new()); }; let disposition = if stalled_run_has_retained_partial_progress(worktree_mapping.as_ref()) { - ActiveRunDisposition::StalledRetainedPartialProgress { idle_for } + RunLeaseDisposition::StalledRetainedPartialProgress { idle_for } } else { - ActiveRunDisposition::Stalled { idle_for } + RunLeaseDisposition::Stalled { idle_for } }; - Ok(vec![ActiveRunReconciliation { + Ok(vec![RunLeaseReconciliation { issue, run_attempt, worktree_mapping, @@ -167,7 +167,7 @@ where }]) } -fn active_reconciliation_workflow_for_lease<'a>( +fn run_lease_reconciliation_workflow<'a>( current_workflow: &'a WorkflowDocument, active_workflow_override: Option>, issue: &TrackerIssue, @@ -182,26 +182,26 @@ fn active_reconciliation_workflow_for_lease<'a>( } } -fn apply_active_run_reconciliation( +fn apply_run_lease_reconciliation( tracker: &T, project: &ServiceConfig, state_store: &StateStore, worktree_manager: &WorktreeManager, - actions: Vec, + actions: Vec, ) -> Result<()> where T: IssueTracker, { for action in actions { match &action.disposition { - ActiveRunDisposition::RetainedReviewComplete => { - reconcile_retained_review_complete_active_run(project, state_store, &action)?; + RunLeaseDisposition::RetainedReviewComplete => { + reconcile_retained_review_complete_run_lease(project, state_store, &action)?; }, - ActiveRunDisposition::Superseded { + RunLeaseDisposition::Superseded { newer_run_id, newer_attempt_number, } => { - reconcile_superseded_active_run( + reconcile_superseded_run_lease( project, state_store, &action, @@ -209,14 +209,14 @@ where *newer_attempt_number, )?; }, - ActiveRunDisposition::Terminal => { + RunLeaseDisposition::Terminal => { tracing::info!( project_id = project.service_id(), issue_id = action.issue.id, issue = action.issue.identifier, run_id = action.run_attempt.run_id(), disposition = "terminal", - "Reconciling terminal active run." + "Reconciling terminal run lease." ); mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "terminated")?; @@ -235,11 +235,11 @@ where )?; } }, - ActiveRunDisposition::NonActive => { - reconcile_nonactive_active_run(project, state_store, worktree_manager, &action)?; + RunLeaseDisposition::NotDispatchable => { + reconcile_not_dispatchable_run_lease(project, state_store, worktree_manager, &action)?; }, - ActiveRunDisposition::Stalled { idle_for } => { - reconcile_stalled_active_run( + RunLeaseDisposition::Stalled { idle_for } => { + reconcile_stalled_run_lease( tracker, project, state_store, @@ -248,7 +248,7 @@ where *idle_for, )?; }, - ActiveRunDisposition::StalledRetainedPartialProgress { idle_for } => { + RunLeaseDisposition::StalledRetainedPartialProgress { idle_for } => { reconcile_stalled_retained_partial_progress_run( tracker, project, @@ -258,8 +258,8 @@ where *idle_for, )?; }, - ActiveRunDisposition::StalledAlreadyNeedsAttention { idle_for } => { - reconcile_stalled_attention_run(project, state_store, &action, *idle_for)?; + RunLeaseDisposition::StalledAlreadyNeedsAttention { idle_for } => { + reconcile_stalled_attention_run_lease(project, state_store, &action, *idle_for)?; }, } } @@ -267,10 +267,10 @@ where Ok(()) } -fn reconcile_superseded_active_run( +fn reconcile_superseded_run_lease( project: &ServiceConfig, state_store: &StateStore, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, newer_run_id: &str, newer_attempt_number: i64, ) -> Result<()> { @@ -283,7 +283,7 @@ fn reconcile_superseded_active_run( superseded_by_run_id = newer_run_id, superseded_by_attempt = newer_attempt_number, disposition = "superseded", - "Reconciling superseded active run without tracker writeback." + "Reconciling superseded run lease without tracker writeback." ); mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "interrupted")?; @@ -297,10 +297,10 @@ fn reconcile_superseded_active_run( Ok(()) } -fn reconcile_retained_review_complete_active_run( +fn reconcile_retained_review_complete_run_lease( project: &ServiceConfig, state_store: &StateStore, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, ) -> Result<()> { tracing::info!( project_id = project.service_id(), @@ -318,19 +318,19 @@ fn reconcile_retained_review_complete_active_run( Ok(()) } -fn reconcile_nonactive_active_run( +fn reconcile_not_dispatchable_run_lease( project: &ServiceConfig, state_store: &StateStore, worktree_manager: &WorktreeManager, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, ) -> Result<()> { tracing::info!( project_id = project.service_id(), issue_id = action.issue.id, issue = action.issue.identifier, run_id = action.run_attempt.run_id(), - disposition = "non_active", - "Reconciling non-active run." + disposition = "not_dispatchable", + "Reconciling run lease for issue that no longer matches dispatch policy." ); mark_run_attempt_if_active(state_store, action.run_attempt.run_id(), "interrupted")?; @@ -354,12 +354,12 @@ fn reconcile_nonactive_active_run( Ok(()) } -fn reconcile_stalled_active_run( +fn reconcile_stalled_run_lease( tracker: &T, project: &ServiceConfig, state_store: &StateStore, worktree_manager: &WorktreeManager, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, idle_for: Duration, ) -> Result<()> where @@ -407,7 +407,7 @@ fn reconcile_stalled_retained_partial_progress_run( project: &ServiceConfig, state_store: &StateStore, worktree_manager: &WorktreeManager, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, idle_for: Duration, ) -> Result<()> where @@ -496,12 +496,13 @@ fn try_recover_stalled_retained_phase_goal( RUN_OPERATION_RECONCILIATION, ); - let recovery = recover_active_phase_goal_continuation( + let recovery = recover_phase_goal_continuation( project, workflow, state_store, issue_run, "stalled_run_detected", + Some("stalled_run_detected"), )?; let Some(recovery) = recovery else { return Ok(false); @@ -550,7 +551,7 @@ fn write_stalled_phase_goal_continuation_retry_marker( fn stalled_reconciliation_issue_run( state_store: &StateStore, worktree_manager: &WorktreeManager, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, ) -> Result { let worktree = action.worktree_mapping.as_ref().map_or_else( || worktree_manager.plan_for_issue(&action.issue.identifier), @@ -583,10 +584,10 @@ fn stalled_reconciliation_issue_run( }) } -fn reconcile_stalled_attention_run( +fn reconcile_stalled_attention_run_lease( project: &ServiceConfig, state_store: &StateStore, - action: &ActiveRunReconciliation, + action: &RunLeaseReconciliation, idle_for: Duration, ) -> Result<()> { tracing::warn!( @@ -660,7 +661,7 @@ fn retained_review_handoff_matches_run( fn superseded_run_disposition( state_store: &StateStore, run_attempt: &RunAttempt, -) -> Result> { +) -> Result> { let Some(latest_attempt) = state_store.latest_run_attempt_for_issue(run_attempt.issue_id())? else { return Ok(None); @@ -670,7 +671,7 @@ fn superseded_run_disposition( return Ok(None); } - Ok(Some(ActiveRunDisposition::Superseded { + Ok(Some(RunLeaseDisposition::Superseded { newer_run_id: latest_attempt.run_id().to_owned(), newer_attempt_number: latest_attempt.attempt_number(), })) @@ -697,7 +698,7 @@ fn stalled_idle_duration( let Some(idle_for) = observed_idle_duration(last_activity, now_unix_epoch) else { return Ok(None); }; - let idle_timeout = active_run_idle_timeout(run_attempt, worktree_mapping)?; + let idle_timeout = run_lease_idle_timeout(run_attempt, worktree_mapping)?; if idle_for >= idle_timeout { return Ok(Some(idle_for)); @@ -706,27 +707,27 @@ fn stalled_idle_duration( Ok(None) } -fn active_run_idle_timeout( +fn run_lease_idle_timeout( run_attempt: &RunAttempt, worktree_mapping: Option<&WorktreeMapping>, ) -> Result { let Some(worktree_mapping) = worktree_mapping else { - return Ok(ACTIVE_RUN_IDLE_TIMEOUT); + return Ok(RUN_LEASE_IDLE_TIMEOUT); }; let Some(marker) = state::read_run_activity_marker_snapshot(worktree_mapping.worktree_path())? else { - return Ok(ACTIVE_RUN_IDLE_TIMEOUT); + return Ok(RUN_LEASE_IDLE_TIMEOUT); }; if marker.run_id() != run_attempt.run_id() || marker.attempt_number() != run_attempt.attempt_number() { - return Ok(ACTIVE_RUN_IDLE_TIMEOUT); + return Ok(RUN_LEASE_IDLE_TIMEOUT); } Ok(agent::protocol_activity_idle_timeout( marker.protocol_activity(), - ACTIVE_RUN_IDLE_TIMEOUT, + RUN_LEASE_IDLE_TIMEOUT, )) } @@ -771,7 +772,7 @@ fn stalled_protocol_idle_duration( return Ok(None); }; - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT { + if idle_for >= RUN_LEASE_IDLE_TIMEOUT { return Ok(Some(idle_for)); } diff --git a/apps/decodex/src/orchestrator/run_cycle.rs b/apps/decodex/src/orchestrator/run_cycle.rs index 60a2bb1d8..b28657191 100644 --- a/apps/decodex/src/orchestrator/run_cycle.rs +++ b/apps/decodex/src/orchestrator/run_cycle.rs @@ -1383,7 +1383,7 @@ fn select_recovered_retry_issue_candidate( recovered_state: RecoveredRuntimeState, excluded_issue_ids: &[&str], ) -> Result> { - for issue in recovered_state.active_issues { + for issue in recovered_state.recoverable_issues { if excluded_issue_ids.contains(&issue.id.as_str()) { continue; } @@ -2854,7 +2854,7 @@ where orphaned_actions.push(action); } - apply_active_run_reconciliation( + apply_run_lease_reconciliation( context.tracker, context.project, context.state_store, @@ -2907,7 +2907,7 @@ fn inspect_orphaned_active_worktree_reconciliation( issue: &TrackerIssue, worktree_mapping: &WorktreeMapping, now_unix_epoch: i64, -) -> Result> +) -> Result> where T: IssueTracker, { @@ -2924,7 +2924,7 @@ where return Ok(None); }; let Some(idle_for) = - orphaned_active_run_idle_duration( + orphaned_run_lease_idle_duration( context.state_store, &run_attempt, worktree_mapping, @@ -2934,18 +2934,18 @@ where return Ok(None); }; let disposition = if needs_attention { - ActiveRunDisposition::StalledAlreadyNeedsAttention { idle_for } - } else if is_issue_active_for_run(issue, context.workflow) + RunLeaseDisposition::StalledAlreadyNeedsAttention { idle_for } + } else if is_issue_in_progress_for_run(issue, context.workflow) && worktree_has_tracked_changes(worktree_mapping.worktree_path()) { - ActiveRunDisposition::StalledRetainedPartialProgress { idle_for } - } else if is_issue_active_for_run(issue, context.workflow) { - ActiveRunDisposition::Stalled { idle_for } + RunLeaseDisposition::StalledRetainedPartialProgress { idle_for } + } else if is_issue_in_progress_for_run(issue, context.workflow) { + RunLeaseDisposition::Stalled { idle_for } } else { return Ok(None); }; - Ok(Some(ActiveRunReconciliation { + Ok(Some(RunLeaseReconciliation { issue: issue.clone(), run_attempt, worktree_mapping: Some(worktree_mapping.clone()), @@ -2954,7 +2954,7 @@ where })) } -fn orphaned_active_run_idle_duration( +fn orphaned_run_lease_idle_duration( state_store: &StateStore, run_attempt: &RunAttempt, worktree_mapping: &WorktreeMapping, diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index e310a742a..c951342af 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -51,7 +51,7 @@ enum AccountActivityMode { #[derive(Clone, Copy)] enum RunIssueMetadataHydration { AllRows, - ActiveRowsOnly, + CurrentLaneRowsOnly, } enum TrackerObserverOutcome { @@ -228,7 +228,7 @@ struct OperatorRunLifecycleProjection { current_operation: String, suspected_stall: bool, execution_liveness: String, - active_lease: bool, + run_lease: bool, retry_kind: Option, retry_ready_at_unix_epoch: Option, } @@ -290,6 +290,11 @@ struct OperatorLaneControlProjection { conditions: Vec, } +#[derive(Default)] +struct OperatorLaneTerminalProjection { + outcomes_by_issue_key: HashMap, +} + pub(crate) fn ensure_project_has_no_merged_worktree_cleanup_debt( project: &ServiceConfig, ) -> crate::prelude::Result<()> { @@ -341,7 +346,7 @@ fn build_operator_status_snapshot_with_account_mode( ) }) .collect::>>()?; - let active_runs = operator_active_run_statuses( + let current_lanes = operator_current_lane_statuses( project, &loop_evidence, &project_display_name, @@ -349,7 +354,7 @@ fn build_operator_status_snapshot_with_account_mode( &recent_runs, now_unix_epoch, )?; - let history_lanes = operator_history_lanes(&active_runs, &recent_runs); + let history_lanes = operator_history_lanes(¤t_lanes, &recent_runs); let (worktrees, mut warnings, warning_details) = operator_status_worktrees(project, state_store)?; let accounts = codex_account_activity_summaries(project, &mut warnings, account_activity_mode); @@ -367,8 +372,8 @@ fn build_operator_status_snapshot_with_account_mode( repo_root: project.repo_root().display().to_string(), enabled: true, github_cli_authority: operator_github_cli_authority(project), - active_run_count: active_runs.len(), - running_lane_count: active_runs.len(), + current_lane_count: current_lanes.len(), + running_lane_count: current_lanes.len(), queued_candidate_count: 0, post_review_lane_count: 0, retained_worktree_count: 0, @@ -382,7 +387,7 @@ fn build_operator_status_snapshot_with_account_mode( }], account_control: global_codex_account_control_status(), accounts, - active_runs, + current_lanes, recent_runs, history_lanes, execution_programs: Vec::new(), @@ -405,13 +410,13 @@ fn build_lane_inspect_operator_runs( limit: usize, ) -> crate::prelude::Result> { let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); - let (active_runs, recent_runs) = state_store.list_project_runs(project.service_id(), limit)?; + let (current_lanes, recent_runs) = state_store.list_project_runs(project.service_id(), limit)?; let loop_evidence = state_store.project_loop_evidence_snapshot(project.service_id())?; let project_display_name = operator_project_display_name(project); let mut seen_run_ids = HashSet::new(); let mut runs = Vec::new(); - for run in active_runs.into_iter().chain(recent_runs) { + for run in current_lanes.into_iter().chain(recent_runs) { if !seen_run_ids.insert(run.run_id().to_owned()) { continue; } @@ -450,7 +455,7 @@ fn project_run_status_issue_matches(run: &ProjectRunStatus, issue: &str) -> bool .is_some_and(|identifier| identifier.eq_ignore_ascii_case(issue)) } -fn operator_active_run_statuses( +fn operator_current_lane_statuses( project: &ServiceConfig, loop_evidence: &ProjectLoopEvidenceSnapshot, project_display_name: &str, @@ -458,7 +463,7 @@ fn operator_active_run_statuses( recent_runs: &[OperatorRunStatus], now_unix_epoch: i64, ) -> crate::prelude::Result> { - let mut active_runs = leased_runs + let mut current_lanes = leased_runs .into_iter() .map(|run| { operator_run_status( @@ -471,21 +476,21 @@ fn operator_active_run_statuses( }) .collect::>>()? .into_iter() - .filter(operator_run_counts_as_active) + .filter(operator_run_counts_as_current_lane) .collect::>(); - let mut active_run_ids = - active_runs.iter().map(|run| run.run_id.clone()).collect::>(); + let mut current_lane_run_ids = + current_lanes.iter().map(|run| run.run_id.clone()).collect::>(); for run in recent_runs { - if !active_run_ids.contains(&run.run_id) && operator_run_has_live_execution(run) { - active_run_ids.insert(run.run_id.clone()); - active_runs.push(run.clone()); + if !current_lane_run_ids.contains(&run.run_id) && operator_run_has_live_execution(run) { + current_lane_run_ids.insert(run.run_id.clone()); + current_lanes.push(run.clone()); } } - hydrate_active_run_lifecycle_metrics(&mut active_runs, recent_runs); + hydrate_current_lane_lifecycle_metrics(&mut current_lanes, recent_runs); - Ok(active_runs) + Ok(current_lanes) } fn global_codex_account_control_status() -> OperatorCodexAccountControlStatus { @@ -623,7 +628,7 @@ where limit, LiveOperatorStatusSnapshotOptions { hydrate_history_ledger: false, - run_issue_metadata_hydration: RunIssueMetadataHydration::ActiveRowsOnly, + run_issue_metadata_hydration: RunIssueMetadataHydration::CurrentLaneRowsOnly, account_activity_mode: AccountActivityMode::Snapshot, }, ) @@ -673,9 +678,17 @@ where }, &mut snapshot, )?; - apply_terminal_history_ledger_outcomes(&mut snapshot); + + let terminal_projection = + current_lane_terminal_projection_from_local_ledger(project, state_store, &snapshot)?; + + apply_operator_lane_terminal_projection( + &mut snapshot, + terminal_projection, + Some(workflow.frontmatter().tracker().resolved_completed_state()), + ); suppress_terminal_attention_queue_echoes(&mut snapshot); - hydrate_post_review_lane_active_run_shadowing(&mut snapshot); + hydrate_post_review_lane_current_lane_shadowing(&mut snapshot); refresh_worktree_ownership( &mut snapshot, Some(workflow.frontmatter().tracker().resolved_completed_state()), @@ -1051,6 +1064,190 @@ fn hydrate_history_lanes_from_local_ledger( Ok(()) } +fn current_lane_terminal_projection_from_local_ledger( + project: &ServiceConfig, + state_store: &StateStore, + snapshot: &OperatorStatusSnapshot, +) -> crate::prelude::Result { + let mut projection = OperatorLaneTerminalProjection::default(); + + for run in &snapshot.current_lanes { + let records = state_store.list_linear_execution_events(project.service_id(), &run.issue_id)?; + + if records.is_empty() { + continue; + } + + let records = local_history_ledger_records(records); + let outcome = operator_history_ledger_outcome(&records); + + if history_ledger_outcome_is_terminal(&outcome) { + projection + .outcomes_by_issue_key + .insert(operator_run_group_key(run), outcome); + } + } + + Ok(projection) +} + +fn apply_operator_lane_terminal_projection( + snapshot: &mut OperatorStatusSnapshot, + projection: OperatorLaneTerminalProjection, + completed_state: Option<&str>, +) { + apply_terminal_history_ledger_outcomes(snapshot); + + if projection.outcomes_by_issue_key.is_empty() { + return; + } + + let current_attention_worktree_keys = snapshot + .worktrees + .iter() + .map(|worktree| operator_issue_attention_key(&worktree.issue_id, worktree.issue_identifier.as_deref())) + .collect::>(); + let mut retained_current_lanes = Vec::new(); + let mut demoted_lanes = Vec::new(); + + for run in snapshot.current_lanes.drain(..) { + if let Some(outcome) = projection.outcomes_by_issue_key.get(&operator_run_group_key(&run)) + && current_lane_terminal_outcome_supersedes( + &run, + outcome, + completed_state, + ¤t_attention_worktree_keys, + ) + { + demoted_lanes.push((run, outcome.clone())); + } else { + retained_current_lanes.push(run); + } + } + + snapshot.current_lanes = retained_current_lanes; + + for (run, outcome) in demoted_lanes { + append_current_lane_to_history(snapshot, run, outcome); + } + + apply_terminal_history_ledger_outcomes(snapshot); +} + +fn current_lane_terminal_outcome_supersedes( + run: &OperatorRunStatus, + outcome: &OperatorHistoryLedgerOutcome, + completed_state: Option<&str>, + current_attention_worktree_keys: &HashSet, +) -> bool { + if !history_ledger_outcome_is_terminal(outcome) { + return false; + } + if current_lane_has_authoritative_live_owner(run) { + return false; + } + if history_ledger_outcome_requires_attention(outcome) { + return !current_lane_has_current_attention_signal(run, current_attention_worktree_keys); + } + + current_lane_tracker_terminal_is_clean(run, completed_state) +} + +fn current_lane_has_authoritative_live_owner(run: &OperatorRunStatus) -> bool { + run.run_lease + || run.process_alive == Some(true) + || matches!(run.thread_status.as_deref(), Some("active")) + || !run.thread_active_flags.is_empty() +} + +fn current_lane_has_current_attention_signal( + run: &OperatorRunStatus, + current_attention_worktree_keys: &HashSet, +) -> bool { + run.needs_attention_label_present == Some(true) + || run.active_label_present == Some(true) + || current_attention_worktree_keys.contains(&operator_run_group_key(run)) +} + +fn current_lane_tracker_terminal_is_clean( + run: &OperatorRunStatus, + completed_state: Option<&str>, +) -> bool { + let has_tracker_metadata = run.issue_state.is_some() + || run.active_label_present.is_some() + || run.needs_attention_label_present.is_some(); + + if !has_tracker_metadata { + return true; + } + + let Some(completed_state) = completed_state else { + return false; + }; + + run.issue_state.as_deref() == Some(completed_state) + && run.active_label_present == Some(false) + && run.needs_attention_label_present == Some(false) +} + +fn append_current_lane_to_history( + snapshot: &mut OperatorStatusSnapshot, + run: OperatorRunStatus, + outcome: OperatorHistoryLedgerOutcome, +) { + let group_key = operator_run_group_key(&run); + + if let Some(lane) = snapshot + .history_lanes + .iter_mut() + .find(|lane| history_lane_group_key(lane) == group_key) + { + if !lane.attempts.iter().any(|attempt| attempt.run_id == run.run_id) { + hydrate_history_lane_from_run(lane, &run); + + if run.attempt_number > lane.latest_run.attempt_number { + lane.latest_run = run.clone(); + } + + lane.attempts.push(run); + + lane.attempt_count = lane.attempts.len(); + lane.lifecycle_metrics = operator_lane_lifecycle_metrics(&lane.attempts); + } + + lane.ledger_outcome = outcome; + + apply_terminal_history_ledger_outcome_to_latest_run(lane); + + return; + } + + let attempts = vec![run.clone()]; + let lifecycle_metrics = operator_lane_lifecycle_metrics(&attempts); + let issue_identifier = run.issue_identifier.clone(); + let issue_key = operator_run_issue_key(&run); + let mut lane = OperatorHistoryLaneStatus { + project_id: run.project_id.clone(), + issue_id: run.issue_id.clone(), + issue_identifier, + title: run.title.clone(), + author: run.author.clone(), + issue_state: run.issue_state.clone(), + active_label_present: run.active_label_present, + needs_attention_label_present: run.needs_attention_label_present, + issue_key, + attempt_count: 1, + ledger_outcome: outcome, + lifecycle_metrics, + latest_run: run, + attempts, + }; + + apply_terminal_history_ledger_outcome_to_latest_run(&mut lane); + + snapshot.history_lanes.push(lane); +} + fn apply_terminal_history_ledger_outcomes(snapshot: &mut OperatorStatusSnapshot) { let mut terminal_history_keys = HashSet::new(); @@ -1068,13 +1265,13 @@ fn apply_terminal_history_ledger_outcomes(snapshot: &mut OperatorStatusSnapshot) return; } - let active_run_ids = snapshot - .active_runs + let current_lane_run_ids = snapshot + .current_lanes .iter() .map(|run| run.run_id.clone()) .collect::>(); - let active_issue_keys = snapshot - .active_runs + let current_lane_issue_keys = snapshot + .current_lanes .iter() .map(operator_run_group_key) .collect::>(); @@ -1082,8 +1279,8 @@ fn apply_terminal_history_ledger_outcomes(snapshot: &mut OperatorStatusSnapshot) snapshot.recent_runs.retain(|run| { let run_group_key = operator_run_group_key(run); - active_run_ids.contains(&run.run_id) - || active_issue_keys.contains(&run_group_key) + current_lane_run_ids.contains(&run.run_id) + || current_lane_issue_keys.contains(&run_group_key) || !terminal_history_keys.contains(&run_group_key) }); } @@ -1141,7 +1338,7 @@ fn apply_terminal_history_ledger_outcome_to_latest_run(lane: &mut OperatorHistor lane.latest_run.wait_reason = None; lane.latest_run.current_operation = String::from("ledger_outcome"); lane.latest_run.continuation_pending = false; - lane.latest_run.active_lease = false; + lane.latest_run.run_lease = false; lane.latest_run.queue_lease_state = String::from("not_held"); lane.latest_run.execution_liveness = String::from("not_running"); lane.latest_run.suspected_stall = false; @@ -1211,11 +1408,11 @@ fn worktree_ownership( snapshot: &OperatorStatusSnapshot, completed_state: Option<&str>, ) -> WorktreeOwnership { - if let Some(run) = worktree_active_run_owner(worktree, snapshot) { + if let Some(run) = worktree_current_lane_owner(worktree, snapshot) { return match run.ownership_state.as_str() { - "owned_active" => WorktreeOwnership { - kind: "active_lane", - reason: format!("Active lane `{}` owns this worktree.", run.run_id), + "leased_run" => WorktreeOwnership { + kind: "current_lane", + reason: format!("Current lane `{}` owns this worktree.", run.run_id), next_action: None, audit_required: false, }, @@ -1310,18 +1507,18 @@ fn worktree_ownership( } } -fn worktree_active_run_owner<'a>( +fn worktree_current_lane_owner<'a>( worktree: &OperatorWorktreeStatus, snapshot: &'a OperatorStatusSnapshot, ) -> Option<&'a OperatorRunStatus> { snapshot - .active_runs + .current_lanes .iter() .chain(snapshot.recent_runs.iter()) .find(|run| { matches!( run.ownership_state.as_str(), - "owned_active" | "retained_attention" | "orphaned_live_thread" | "terminalizing" + "leased_run" | "retained_attention" | "orphaned_live_thread" | "terminalizing" ) && (run.worktree_path.as_deref() == Some(worktree.worktree_path.as_str()) || run.branch_name.as_deref() == Some(worktree.branch_name.as_str()) || run.issue_id == worktree.issue_id) @@ -1395,7 +1592,7 @@ fn worktree_cleanup_only_reason( } String::from( - "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", + "No current lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", ) } @@ -1412,9 +1609,9 @@ fn refresh_operator_project_summary( snapshot: &mut OperatorStatusSnapshot, completed_state: Option<&str>, ) { - let active_run_count = snapshot.active_runs.len(); + let current_lane_count = snapshot.current_lanes.len(); let running_lane_count = - snapshot.active_runs.iter().filter(|run| operator_run_counts_as_running(run)).count(); + snapshot.current_lanes.iter().filter(|run| operator_run_counts_as_running(run)).count(); let queued_candidate_count = snapshot .queued_candidates .iter() @@ -1423,7 +1620,7 @@ fn refresh_operator_project_summary( let post_review_lane_count = snapshot .post_review_lanes .iter() - .filter(|lane| !lane.shadowed_by_active_run) + .filter(|lane| !lane.shadowed_by_current_lane) .count(); let retained_worktree_count = rendered_recovery_worktrees(snapshot).len(); let waiting_lane_count = project_waiting_lane_count(snapshot); @@ -1435,7 +1632,7 @@ fn refresh_operator_project_summary( let warning_count = snapshot.warnings.len(); if let Some(project_status) = snapshot.projects.first_mut() { - project_status.active_run_count = active_run_count; + project_status.current_lane_count = current_lane_count; project_status.running_lane_count = running_lane_count; project_status.queued_candidate_count = queued_candidate_count; project_status.post_review_lane_count = post_review_lane_count; @@ -1465,14 +1662,14 @@ fn project_waiting_lane_count(snapshot: &OperatorStatusSnapshot) -> usize { let review_waiting = snapshot .post_review_lanes .iter() - .filter(|lane| !lane.shadowed_by_active_run && lane.classification == "wait_for_review") + .filter(|lane| !lane.shadowed_by_current_lane && lane.classification == "wait_for_review") .count(); waiting_run_count + queued_waiting + review_waiting } fn project_summary_runs(snapshot: &OperatorStatusSnapshot) -> Vec<&OperatorRunStatus> { - let mut runs = snapshot.active_runs.iter().collect::>(); + let mut runs = snapshot.current_lanes.iter().collect::>(); runs.extend(snapshot.history_lanes.iter().map(|lane| &lane.latest_run)); @@ -1494,7 +1691,7 @@ fn project_attention_count( let mut attention_keys = HashSet::new(); for run in snapshot - .active_runs + .current_lanes .iter() .filter(|run| operator_run_counts_as_attention(run)) { @@ -1544,7 +1741,7 @@ fn queued_candidate_counts_as_attention(candidate: &OperatorQueuedIssueStatus) - } fn post_review_lane_counts_as_attention(lane: &OperatorPostReviewLaneStatus) -> bool { - if lane.shadowed_by_active_run { + if lane.shadowed_by_current_lane { return false; } @@ -1676,7 +1873,7 @@ fn project_cleanup_blocked_count(snapshot: &OperatorStatusSnapshot) -> usize { for lane in snapshot .post_review_lanes .iter() - .filter(|lane| !lane.shadowed_by_active_run && lane.classification == "cleanup_blocked") + .filter(|lane| !lane.shadowed_by_current_lane && lane.classification == "cleanup_blocked") { cleanup_keys.insert(post_review_lane_cleanup_key(lane)); } @@ -1713,19 +1910,20 @@ fn post_review_lane_cleanup_key(lane: &OperatorPostReviewLaneStatus) -> String { lane.issue_identifier.clone() } -fn hydrate_post_review_lane_active_run_shadowing(snapshot: &mut OperatorStatusSnapshot) { - let active_issue_keys = snapshot - .active_runs +fn hydrate_post_review_lane_current_lane_shadowing(snapshot: &mut OperatorStatusSnapshot) { + let current_lane_issue_keys = snapshot + .current_lanes .iter() .filter(|run| run.counts_as_running) .map(|run| operator_run_group_key(run).to_ascii_uppercase()) .collect::>(); for lane in &mut snapshot.post_review_lanes { - lane.shadowed_by_active_run = active_issue_keys.contains(&operator_issue_attention_key( - &lane.issue_id, - Some(&lane.issue_identifier), - )); + lane.shadowed_by_current_lane = + current_lane_issue_keys.contains(&operator_issue_attention_key( + &lane.issue_id, + Some(&lane.issue_identifier), + )); } } @@ -1736,8 +1934,8 @@ fn worktree_cleanup_key(worktree: &OperatorWorktreeStatus) -> String { .unwrap_or_else(|| worktree.issue_id.clone()) } -fn operator_run_counts_as_active(run: &OperatorRunStatus) -> bool { - (run.active_lease || operator_run_has_live_execution(run)) +fn operator_run_counts_as_current_lane(run: &OperatorRunStatus) -> bool { + (run.run_lease || operator_run_has_live_execution(run)) && !matches!(run.phase.as_str(), "completed" | "failed" | "terminated") } @@ -1750,7 +1948,7 @@ fn operator_run_has_live_execution(run: &OperatorRunStatus) -> bool { } fn operator_run_counts_as_running(run: &OperatorRunStatus) -> bool { - run.ownership_state == "owned_active" + run.ownership_state == "leased_run" && matches!(run.status.as_str(), "starting" | "running") && run.phase == "executing" && (run.process_alive != Some(false) || run.has_fresh_execution) @@ -1762,7 +1960,10 @@ fn operator_run_counts_as_attention(run: &OperatorRunStatus) -> bool { || run.ownership_state == "retained_attention" || matches!( run.policy_state.as_str(), - "review_churn_exceeded" | "authority_boundary_required" | "human_attention_required" + "review_churn_exceeded" + | "continuation_recovery_churn_exceeded" + | "authority_boundary_required" + | "human_attention_required" ) } @@ -1788,7 +1989,7 @@ fn operator_run_has_recent_app_server_execution(run: &OperatorRunStatus) -> bool || !run.thread_active_flags.is_empty() || run.protocol_idle_for_seconds.is_some_and(|idle_for| { u64::try_from(idle_for) - .is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + .is_ok_and(|idle_for| idle_for < RUN_LEASE_IDLE_TIMEOUT.as_secs()) }) } @@ -1800,7 +2001,7 @@ fn operator_run_has_stale_execution_without_known_process(run: &OperatorRunStatu && !run.has_fresh_execution && [run.idle_for_seconds, run.protocol_idle_for_seconds].iter().any(|idle_for| { idle_for.is_some_and(|idle_for| { - u64::try_from(idle_for).is_ok_and(|idle_for| idle_for >= ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + u64::try_from(idle_for).is_ok_and(|idle_for| idle_for >= RUN_LEASE_IDLE_TIMEOUT.as_secs()) }) }) } @@ -1826,7 +2027,7 @@ fn project_connector_state(snapshot: &OperatorStatusSnapshot) -> String { fn project_last_activity_at(snapshot: &OperatorStatusSnapshot) -> Option { snapshot - .active_runs + .current_lanes .iter() .chain(snapshot.recent_runs.iter()) .chain(snapshot.history_lanes.iter().map(|lane| &lane.latest_run)) @@ -1865,7 +2066,7 @@ fn operator_status_worktrees( worktree_path: relative_worktree_path_for_path(project, mapping.worktree_path()), ownership: String::from("cleanup_only"), ownership_reason: String::from( - "No active lane, queued recovery, or post-review lane currently owns this worktree.", + "No current lane, queued recovery, or post-review lane currently owns this worktree.", ), provenance: operator_worktree_provenance_from_mapping(&mapping), recovery_next_action: None, @@ -1899,7 +2100,7 @@ fn operator_status_worktrees( worktree_path: relative_path, ownership: String::from("cleanup_only"), ownership_reason: String::from( - "No active lane, queued recovery, or post-review lane currently owns this worktree.", + "No current lane, queued recovery, or post-review lane currently owns this worktree.", ), provenance: operator_worktree_provenance( WORKTREE_PROVENANCE_FILESYSTEM_SCAN, @@ -2214,7 +2415,7 @@ fn operator_snapshot_run_issue_ids( ) -> Vec { let mut issue_ids = BTreeSet::new(); - for run in &snapshot.active_runs { + for run in &snapshot.current_lanes { append_operator_run_issue_id(&mut issue_ids, run); } @@ -2246,7 +2447,7 @@ fn hydrate_operator_snapshot_run_rows( snapshot: &mut OperatorStatusSnapshot, metadata_by_issue_id: &HashMap, ) { - for run in snapshot.active_runs.iter_mut().chain(snapshot.recent_runs.iter_mut()) { + for run in snapshot.current_lanes.iter_mut().chain(snapshot.recent_runs.iter_mut()) { hydrate_operator_run_row_from_issue_metadata(run, metadata_by_issue_id); } for lane in &mut snapshot.history_lanes { @@ -2322,6 +2523,19 @@ fn apply_run_issue_metadata( if let Some(author) = metadata.author.as_ref().filter(|author| !author.trim().is_empty()) { run.author = Some(author.clone()); } + if let Some(issue_state) = metadata + .issue_state + .as_ref() + .filter(|issue_state| !issue_state.trim().is_empty()) + { + run.issue_state = Some(issue_state.clone()); + } + if let Some(active_label_present) = metadata.active_label_present { + run.active_label_present = Some(active_label_present); + } + if let Some(needs_attention_label_present) = metadata.needs_attention_label_present { + run.needs_attention_label_present = Some(needs_attention_label_present); + } } fn fill_missing_history_lane_issue_metadata( @@ -2371,6 +2585,23 @@ fn fill_missing_run_issue_metadata( { run.author = Some(author.clone()); } + if run + .issue_state + .as_ref() + .is_none_or(|issue_state| issue_state.trim().is_empty()) + && let Some(issue_state) = metadata + .issue_state + .as_ref() + .filter(|issue_state| !issue_state.trim().is_empty()) + { + run.issue_state = Some(issue_state.clone()); + } + if run.active_label_present.is_none() { + run.active_label_present = metadata.active_label_present; + } + if run.needs_attention_label_present.is_none() { + run.needs_attention_label_present = metadata.needs_attention_label_present; + } } fn hydrate_history_lanes_from_linear_ledger( @@ -3290,7 +3521,7 @@ fn operator_queued_issue_attention_summary( } return String::from( - "Linear active ownership is still present without a matching local active lease; reconcile before dispatch.", + "Linear active ownership is still present without a matching local run lease; reconcile before dispatch.", ); } if worktree_has_tracked_changes { @@ -3551,7 +3782,7 @@ fn degraded_post_review_lane_status_from_classification( mergeable: classification.mergeable, check_state: classification.check_state, unresolved_review_threads: classification.unresolved_review_threads, - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: classification.readback_warning, readback_root_cause: classification.readback_root_cause, loop_status: Some(loop_status), @@ -3764,7 +3995,7 @@ fn post_review_lane_status_from_classification( mergeable: classification.mergeable, check_state: classification.check_state, unresolved_review_threads: classification.unresolved_review_threads, - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: classification.readback_warning, readback_root_cause: classification.readback_root_cause, loop_status, @@ -4838,7 +5069,7 @@ fn blocked_post_review_lane_status( mergeable: None, check_state: None, unresolved_review_threads: None, - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: None, readback_root_cause: post_review_readback_root_cause_for_reason(reason) .map(|root_cause| root_cause.as_str().to_owned()), @@ -5059,10 +5290,10 @@ where } let now_unix_epoch = OffsetDateTime::now_utc().unix_timestamp(); - let mut active_issues = Vec::new(); + let mut recoverable_issues = Vec::new(); for issue in issues { - if let Some(active_issue) = recover_issue_runtime_state( + if let Some(recoverable_issue) = recover_issue_runtime_state( tracker, project, workflow, @@ -5071,13 +5302,13 @@ where issue, now_unix_epoch, )? { - active_issues.push(active_issue); + recoverable_issues.push(recoverable_issue); } } - active_issues.sort_by(compare_issue_candidates); + recoverable_issues.sort_by(compare_issue_candidates); - Ok(RecoveredRuntimeState { active_issues }) + Ok(RecoveredRuntimeState { recoverable_issues }) } fn recover_issue_runtime_state( @@ -5353,7 +5584,7 @@ fn worktree_activity_marker_is_fresh(marker: &RunActivityMarker, now_unix_epoch: fn run_activity_idle_timeout(marker: Option<&RunActivityMarker>) -> Duration { agent::protocol_activity_idle_timeout( marker.and_then(RunActivityMarker::protocol_activity), - ACTIVE_RUN_IDLE_TIMEOUT, + RUN_LEASE_IDLE_TIMEOUT, ) } @@ -5571,6 +5802,7 @@ fn operator_run_status( worktree_path.as_deref(), ); let private_evidence = operator_run_private_evidence(project, &run, issue_identifier.as_deref()); + let continuation_recovery = operator_run_continuation_recovery_status(loop_evidence, &run); let loop_status = operator_run_loop_status( project, @@ -5599,6 +5831,7 @@ fn operator_run_status( worktree_path, issue_identifier, private_evidence, + continuation_recovery, loop_status, ))) } @@ -5622,6 +5855,7 @@ fn operator_run_status_from_parts( worktree_path: Option, issue_identifier: Option, private_evidence: AgentPrivateEvidenceRef, + continuation_recovery: Option, loop_status: OperatorLoopStatus, ) -> OperatorRunStatus { OperatorRunStatus { @@ -5632,6 +5866,9 @@ fn operator_run_status_from_parts( issue_identifier, title: None, author: None, + issue_state: None, + active_label_present: None, + needs_attention_label_present: None, attempt_number: run.attempt_number(), status: lifecycle.status, attempt_status: run.status().to_owned(), @@ -5652,8 +5889,9 @@ fn operator_run_status_from_parts( thread_active_flags: app_server_state.thread_active_flags, interactive_requested: app_server_state.interactive_requested, continuation_pending: app_server_state.continuation_pending, - active_lease: lifecycle.active_lease, - queue_lease_state: operator_run_queue_lease_state(lifecycle.active_lease), + continuation_recovery, + run_lease: lifecycle.run_lease, + queue_lease_state: operator_run_queue_lease_state(lifecycle.run_lease), execution_liveness: lifecycle.execution_liveness, has_fresh_execution: false, counts_as_running: false, @@ -5727,8 +5965,8 @@ fn operator_lane_control_state(run: &OperatorRunStatus) -> OperatorLaneControlPr ); let mut conditions = operator_run_lane_control_conditions(run, &liveness_state, &policy_state); - if ownership_state == "owned_active" && !run.active_lease { - conditions.push(String::from("invalid_owned_active_without_lease")); + if ownership_state == "leased_run" && !run.run_lease { + conditions.push(String::from("invalid_leased_run_without_lease")); } OperatorLaneControlProjection { @@ -5747,24 +5985,30 @@ fn operator_run_ownership_state( policy_state: &str, terminalization_state: &str, ) -> String { - if run.active_lease + if run.run_lease && matches!(run.attempt_status.as_str(), "starting" | "running" | "continuation_pending") && !matches!( policy_state, - "review_churn_exceeded" | "authority_boundary_required" | "human_attention_required" + "review_churn_exceeded" + | "continuation_recovery_churn_exceeded" + | "authority_boundary_required" + | "human_attention_required" ) { - return String::from("owned_active"); + return String::from("leased_run"); } if matches!( policy_state, - "review_churn_exceeded" | "authority_boundary_required" | "human_attention_required" + "review_churn_exceeded" + | "continuation_recovery_churn_exceeded" + | "authority_boundary_required" + | "human_attention_required" ) || run.needs_attention - || (!run.active_lease && liveness_state == "host_boot_mismatch") + || (!run.run_lease && liveness_state == "host_boot_mismatch") { return String::from("retained_attention"); } - if !run.active_lease + if !run.run_lease && matches!(liveness_state, "process_alive" | "thread_active" | "protocol_recent") { return String::from("orphaned_live_thread"); @@ -5805,6 +6049,14 @@ fn operator_run_liveness_state(run: &OperatorRunStatus) -> String { } fn operator_run_policy_state(run: &OperatorRunStatus) -> String { + if run + .continuation_recovery + .as_ref() + .is_some_and(|recovery| recovery.budget_exceeded) + { + return String::from("continuation_recovery_churn_exceeded"); + } + let Some(loop_status) = run.loop_status.as_ref() else { return String::from("allowed"); }; @@ -5855,7 +6107,7 @@ fn operator_run_terminalization_state(run: &OperatorRunStatus, liveness_state: & return String::from("cleanup_complete"); } if matches!(run.phase.as_str(), "completed" | "failed" | "terminated") - && !run.active_lease + && !run.run_lease && matches!(liveness_state, "not_running" | "unknown") { return String::from("cleanup_complete"); @@ -5874,10 +6126,10 @@ fn operator_run_lane_control_conditions( ) -> Vec { let mut conditions = Vec::new(); - if !run.active_lease + if !run.run_lease && matches!(run.attempt_status.as_str(), "starting" | "running" | "continuation_pending") { - conditions.push(String::from("active_lease_missing")); + conditions.push(String::from("run_lease_missing")); } if matches!( run.attempt_status.as_str(), @@ -5892,6 +6144,9 @@ fn operator_run_lane_control_conditions( if policy_state == "review_churn_exceeded" { conditions.push(String::from("review_churn_threshold_exceeded")); } + if policy_state == "continuation_recovery_churn_exceeded" { + conditions.push(String::from("continuation_recovery_budget_exceeded")); + } if matches!( policy_state, "authority_boundary_required" | "human_attention_required" @@ -5912,6 +6167,9 @@ fn operator_run_lane_control_next_action( if policy_state == "review_churn_exceeded" { return String::from("start_architecture_recovery_or_stop_for_human_attention"); } + if policy_state == "continuation_recovery_churn_exceeded" { + return String::from("stop_auto_continuation_and_request_architecture_recovery"); + } if matches!( policy_state, "authority_boundary_required" | "human_attention_required" @@ -5927,7 +6185,7 @@ fn operator_run_lane_control_next_action( if terminalization_state != "none" && terminalization_state != "cleanup_complete" { return String::from("finish_terminalization"); } - if ownership_state == "owned_active" { + if ownership_state == "leased_run" { if let Some(next_action) = run.loop_status.as_ref().and_then(|loop_status| loop_status.next_action.clone()) { @@ -6014,7 +6272,7 @@ fn operator_run_lifecycle_projection( } else { operator_run_execution_liveness(&status, timing, app_server_state, protocol_summary) }; - let active_lease = terminal_finalize_projection.is_none() && run.active_lease(); + let run_lease = terminal_finalize_projection.is_none() && run.run_lease(); OperatorRunLifecycleProjection { status, @@ -6024,7 +6282,7 @@ fn operator_run_lifecycle_projection( current_operation, suspected_stall, execution_liveness, - active_lease, + run_lease, retry_kind, retry_ready_at_unix_epoch, } @@ -6743,6 +7001,86 @@ fn operator_run_terminal_finalize_projection( } } +fn operator_run_continuation_recovery_status( + loop_evidence: &ProjectLoopEvidenceSnapshot, + run: &ProjectRunStatus, +) -> Option { + let recovery_events = loop_evidence + .private_events_for_issue(run.issue_id()) + .into_iter() + .filter(|event| event.attempt_number() <= run.attempt_number()) + .filter_map(operator_continuation_recovery_event_status) + .collect::>(); + let latest = recovery_events.last()?.clone(); + let recovery_count = recovery_events + .iter() + .filter(|event| { + event.source_phase == latest.source_phase + && event.source_error_class == latest.source_error_class + && event.state == "continuation_scheduled" + }) + .count() as i64; + let budget_exceeded = latest.state == "continuation_blocked" + || recovery_count > PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT; + + Some(OperatorContinuationRecoveryStatus { + state: latest.state, + source_phase: latest.source_phase, + next_phase: latest.next_phase, + source_error_class: latest.source_error_class, + source_error_message: latest.source_error_message, + recorded_at: latest.recorded_at, + run_id: latest.run_id, + attempt_number: latest.attempt_number, + recovery_count, + automatic_continuation_limit: PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT, + budget_exceeded, + next_action: if budget_exceeded { + String::from("stop_auto_continuation_and_request_architecture_recovery") + } else { + String::from("monitor_continuation_recovery") + }, + }) +} + +fn operator_continuation_recovery_event_status( + event: &PrivateExecutionEvent, +) -> Option { + let state = match event.event_type() { + PHASE_GOAL_RECOVERY_EVENT_TYPE => "continuation_scheduled", + PHASE_GOAL_RECOVERY_BLOCKED_EVENT_TYPE => "continuation_blocked", + _ => return None, + }; + let payload = event.payload(); + let event_payload = payload.get("payload").unwrap_or(payload); + let source_phase = payload + .get("phase") + .and_then(Value::as_str) + .or_else(|| event_payload.get("sourcePhase").and_then(Value::as_str))? + .to_owned(); + let next_phase = event_payload.get("nextPhase")?.as_str()?.to_owned(); + let source_error_class = event_payload.get("sourceErrorClass")?.as_str()?.to_owned(); + let source_error_message = event_payload + .get("sourceErrorMessage") + .and_then(Value::as_str) + .map(str::to_owned); + + Some(OperatorContinuationRecoveryStatus { + state: String::from(state), + source_phase, + next_phase, + source_error_class, + source_error_message, + recorded_at: event.recorded_at().to_owned(), + run_id: event.run_id().to_owned(), + attempt_number: event.attempt_number(), + recovery_count: 0, + automatic_continuation_limit: PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT, + budget_exceeded: false, + next_action: String::new(), + }) +} + fn operator_run_visible_status( attempt_status: &str, app_server_state: &OperatorRunAppServerState, @@ -6823,7 +7161,7 @@ fn operator_run_has_recent_protocol_execution_evidence( operator_protocol_event_counts_as_live_execution(protocol_summary.last_event_type.as_deref()) && timing.protocol_idle_for_seconds.is_some_and(|idle_for| { u64::try_from(idle_for) - .is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + .is_ok_and(|idle_for| idle_for < RUN_LEASE_IDLE_TIMEOUT.as_secs()) }) } @@ -6852,12 +7190,12 @@ fn operator_run_has_app_server_execution_evidence( || protocol_summary.last_event_type.is_some() || timing.protocol_idle_for_seconds.is_some_and(|idle_for| { u64::try_from(idle_for) - .is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + .is_ok_and(|idle_for| idle_for < RUN_LEASE_IDLE_TIMEOUT.as_secs()) }) } -fn operator_run_queue_lease_state(active_lease: bool) -> String { - if active_lease { +fn operator_run_queue_lease_state(run_lease: bool) -> String { + if run_lease { String::from("held") } else { String::from("not_held") @@ -6959,7 +7297,7 @@ fn operator_run_protocol_activity( if is_running && summary.waiting_reason.is_none() && protocol_idle_for_seconds.is_some_and(|idle_for| { - u64::try_from(idle_for).is_ok_and(|idle_for| idle_for < ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + u64::try_from(idle_for).is_ok_and(|idle_for| idle_for < RUN_LEASE_IDLE_TIMEOUT.as_secs()) }) { summary.waiting_reason = Some(String::from("protocol_idleness")); } @@ -7154,13 +7492,13 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { .iter() .map(|lane| lane.attempt_count) .sum::(); - let hides_running_lanes = session_history_attempt_count < snapshot.recent_runs.len(); - let (running_inline_claims, non_running_queued_candidates): (Vec<_>, Vec<_>) = snapshot + let hides_current_lanes = session_history_attempt_count < snapshot.recent_runs.len(); + let (current_lane_claims, backlog_or_stale_queue_candidates): (Vec<_>, Vec<_>) = snapshot .queued_candidates .iter() - .partition(|queued_issue| queue_claim_belongs_to_active_run(queued_issue, snapshot)); + .partition(|queued_issue| queue_claim_belongs_to_current_lane(queued_issue, snapshot)); let (stale_closed_queue_labels, backlog_candidates) = - rendered_backlog_queue_groups(non_running_queued_candidates); + rendered_backlog_queue_groups(backlog_or_stale_queue_candidates); let recovery_worktrees = rendered_recovery_worktrees(snapshot); let hides_owned_worktrees = recovery_worktrees.len() < snapshot.worktrees.len(); let mut output = String::new(); @@ -7182,17 +7520,24 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { append_rendered_github_cli_authority(&mut output, snapshot); - output.push_str(&format!("Running lanes: {}\n", snapshot.active_runs.len())); + let running_lane_count = snapshot + .current_lanes + .iter() + .filter(|run| operator_run_counts_as_running(run)) + .count(); + + output.push_str(&format!("Current lanes: {}\n", snapshot.current_lanes.len())); + output.push_str(&format!("Running lanes: {running_lane_count}\n")); output.push_str(&format!( "Run ledger shown: {} issue lanes from {} history attempts{}\n", snapshot.history_lanes.len(), session_history_attempt_count, - if hides_running_lanes { " (running lanes inline)" } else { "" }, + if hides_current_lanes { " (current lanes inline)" } else { "" }, )); output.push_str(&format!("Backlog: {}\n", backlog_candidates.len())); output.push_str(&format!( - "Active queue echoes: {}\n", - running_inline_claims.len() + "Claimed queue echoes: {}\n", + current_lane_claims.len() )); output.push_str(&format!( "Stale closed queue labels: {}\n", @@ -7208,12 +7553,12 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { append_rendered_attention_summary(&mut output, snapshot); append_rendered_execution_programs(&mut output, snapshot); - output.push_str("\nRunning Lanes\n"); + output.push_str("\nCurrent Lanes\n"); - if snapshot.active_runs.is_empty() { + if snapshot.current_lanes.is_empty() { output.push_str("- none\n"); } else { - for run in &snapshot.active_runs { + for run in &snapshot.current_lanes { append_rendered_run(&mut output, run); } } @@ -7221,8 +7566,8 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { output.push_str("\nRun Ledger\n"); if snapshot.history_lanes.is_empty() { - if hides_running_lanes { - output.push_str("- none (running lanes are shown above)\n"); + if hides_current_lanes { + output.push_str("- none (current lanes are shown above)\n"); } else { output.push_str("- none\n"); } @@ -7233,10 +7578,10 @@ fn render_operator_status(snapshot: &OperatorStatusSnapshot) -> String { } append_rendered_queued_issue_section(&mut output, "Backlog", &backlog_candidates, snapshot, false); - append_rendered_queued_issue_section( +append_rendered_queued_issue_section( &mut output, - "Active Queue Echoes", - &running_inline_claims, + "Claimed Queue Echoes", + ¤t_lane_claims, snapshot, true, ); @@ -7385,13 +7730,13 @@ fn append_rendered_post_review_lanes(output: &mut String, snapshot: &OperatorSta let loop_boundary = render_loop_boundary_summary(lane.loop_status.as_ref()); output.push_str(&format!( - "- issue_id: {}\n issue: {}\n state: {}\n classification: {}\n reason: {}\n shadowed_by_active_run: {}\n branch: {}\n worktree_path: {}\n pr_url: {}\n pr_head_sha: {}\n pr_state: {}\n review_decision: {}\n mergeable: {}\n check_state: {}\n unresolved_review_threads: {}\n readback_warning: {}\n readback_root_cause: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n", + "- issue_id: {}\n issue: {}\n state: {}\n classification: {}\n reason: {}\n shadowed_by_current_lane: {}\n branch: {}\n worktree_path: {}\n pr_url: {}\n pr_head_sha: {}\n pr_state: {}\n review_decision: {}\n mergeable: {}\n check_state: {}\n unresolved_review_threads: {}\n readback_warning: {}\n readback_root_cause: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n", lane.issue_id, lane.issue_identifier, lane.issue_state, lane.classification, lane.reason, - if lane.shadowed_by_active_run { "yes" } else { "no" }, + if lane.shadowed_by_current_lane { "yes" } else { "no" }, lane.branch_name, lane.worktree_path, lane.pr_url.as_deref().unwrap_or("none"), @@ -7616,7 +7961,7 @@ fn append_rendered_queued_issue_section( for queued_issue in queued_issues { let running_owner = show_running_owner - .then(|| active_run_id_for_queue_candidate(queued_issue, snapshot)) + .then(|| current_lane_run_id_for_queue_candidate(queued_issue, snapshot)) .flatten(); append_rendered_queued_issue(output, queued_issue, running_owner); @@ -7624,14 +7969,14 @@ fn append_rendered_queued_issue_section( } fn operator_history_lanes( - active_runs: &[OperatorRunStatus], + current_lanes: &[OperatorRunStatus], recent_runs: &[OperatorRunStatus], ) -> Vec { - let active_run_ids = active_runs + let current_lane_run_ids = current_lanes .iter() .map(|run| run.run_id.as_str()) .collect::>(); - let active_issue_ids = active_runs + let current_lane_issue_ids = current_lanes .iter() .map(|run| run.issue_id.as_str()) .collect::>(); @@ -7639,8 +7984,8 @@ fn operator_history_lanes( let mut lanes = Vec::new(); for run in recent_runs { - if active_run_ids.contains(run.run_id.as_str()) - || active_issue_ids.contains(run.issue_id.as_str()) + if current_lane_run_ids.contains(run.run_id.as_str()) + || current_lane_issue_ids.contains(run.issue_id.as_str()) { continue; } @@ -7709,35 +8054,35 @@ fn hydrate_history_lane_from_run(lane: &mut OperatorHistoryLaneStatus, run: &Ope } } -fn hydrate_active_run_lifecycle_metrics( - active_runs: &mut [OperatorRunStatus], +fn hydrate_current_lane_lifecycle_metrics( + current_lanes: &mut [OperatorRunStatus], recent_runs: &[OperatorRunStatus], ) { - for active_run in active_runs { - let group_key = operator_run_group_key(active_run); - let active_run_id = active_run.run_id.clone(); - let active_snapshot = active_run.clone(); + for current_lane in current_lanes { + let group_key = operator_run_group_key(current_lane); + let current_lane_run_id = current_lane.run_id.clone(); + let current_lane_snapshot = current_lane.clone(); let mut attempts = Vec::new(); - let mut captured_active_snapshot = false; + let mut captured_current_lane_snapshot = false; for run in recent_runs .iter() .filter(|run| operator_run_group_key(run) == group_key) { - if run.run_id == active_run_id { - captured_active_snapshot = true; + if run.run_id == current_lane_run_id { + captured_current_lane_snapshot = true; - attempts.push(active_snapshot.clone()); + attempts.push(current_lane_snapshot.clone()); } else { attempts.push(run.clone()); } } - if !captured_active_snapshot { - attempts.push(active_snapshot); + if !captured_current_lane_snapshot { + attempts.push(current_lane_snapshot); } - active_run.lifecycle_metrics = operator_lane_lifecycle_metrics(&attempts); + current_lane.lifecycle_metrics = operator_lane_lifecycle_metrics(&attempts); } } @@ -8038,20 +8383,20 @@ fn issue_identifier_in_text(value: &str) -> Option { None } -fn queue_claim_belongs_to_active_run( +fn queue_claim_belongs_to_current_lane( queued_issue: &OperatorQueuedIssueStatus, snapshot: &OperatorStatusSnapshot, ) -> bool { queued_issue.classification == "claimed" - && active_run_id_for_queue_candidate(queued_issue, snapshot).is_some() + && current_lane_run_id_for_queue_candidate(queued_issue, snapshot).is_some() } -fn active_run_id_for_queue_candidate<'a>( +fn current_lane_run_id_for_queue_candidate<'a>( queued_issue: &OperatorQueuedIssueStatus, snapshot: &'a OperatorStatusSnapshot, ) -> Option<&'a str> { snapshot - .active_runs + .current_lanes .iter() .find(|run| run.issue_id == queued_issue.issue_id && run.counts_as_running) .map(|run| run.run_id.as_str()) @@ -8060,7 +8405,7 @@ fn active_run_id_for_queue_candidate<'a>( fn append_rendered_queued_issue( output: &mut String, queued_issue: &OperatorQueuedIssueStatus, - active_run_id: Option<&str>, + current_lane_run_id: Option<&str>, ) { let priority = queued_issue .priority @@ -8070,7 +8415,7 @@ fn append_rendered_queued_issue( } else { queued_issue.blocker_identifiers.join(", ") }; - let running_owner = active_run_id.unwrap_or("none"); + let running_owner = current_lane_run_id.unwrap_or("none"); output.push_str(&format!( "- issue_id: {}\n issue: {}\n title: {}\n state: {}\n priority: {}\n created_at: {}\n classification: {}\n reason: {}\n running_owner_run: {}\n blockers: {}\n", @@ -8136,13 +8481,13 @@ fn rendered_worktree_role<'a>( if !worktree.ownership.trim().is_empty() { return worktree.ownership.as_str(); } - if snapshot.active_runs.iter().any(|run| { - run.ownership_state == "owned_active" + if snapshot.current_lanes.iter().any(|run| { + run.ownership_state == "leased_run" && (run.worktree_path.as_deref() == Some(worktree.worktree_path.as_str()) || run.branch_name.as_deref() == Some(worktree.branch_name.as_str()) || run.issue_id == worktree.issue_id) }) { - return "active_lane"; + return "current_lane"; } if snapshot.post_review_lanes.iter().any(|lane| { lane.worktree_path == worktree.worktree_path @@ -8173,7 +8518,7 @@ fn rendered_worktree_role<'a>( fn rendered_worktree_role_rank(role: &str) -> u8 { match role { - "active_lane" | "running_lane" | "blocked_queue_issue" | "queued_attention" => 0, + "current_lane" | "running_lane" | "blocked_queue_issue" | "queued_attention" => 0, "post_review_lane" => 1, _ => 2, } @@ -8596,12 +8941,7 @@ fn history_ledger_outcome_has_records(outcome: &OperatorHistoryLedgerOutcome) -> fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { let (freshness_source, freshness_at) = operator_run_freshness(run); - let protocol_event = match (&run.last_event_type, &run.last_event_at) { - (Some(event_type), Some(timestamp)) => format!("{event_type} @ {timestamp}"), - (Some(event_type), None) => event_type.clone(), - (None, Some(timestamp)) => timestamp.clone(), - (None, None) => String::from("none"), - }; + let protocol_event = render_run_protocol_event(run); let thread_id = run.thread_id.as_deref().unwrap_or("none"); let turn_id = run.turn_id.as_deref().unwrap_or("none"); let thread_status = run.thread_status.as_deref().unwrap_or("none"); @@ -8632,9 +8972,11 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { let loop_boundary = render_loop_boundary_summary(run.loop_status.as_ref()); let control_capability = render_control_capability_summary(run.control_capability.as_ref()); let lane_control_conditions = render_lane_control_conditions(run); + let continuation_recovery = + render_continuation_recovery_summary(run.continuation_recovery.as_ref()); output.push_str(&format!( - "- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n status_projection_reason: {}\n ownership_state: {}\n liveness_state: {}\n policy_state: {}\n terminalization_state: {}\n lane_control_next_action: {}\n lane_control_conditions: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n active_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n has_fresh_execution: {}\n counts_as_running: {}\n needs_attention: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n progress_diagnostic: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n", + "- run_id: {}\n project_id: {}\n issue_id: {}\n issue_identifier: {}\n title: {}\n attempt: {}\n status: {}\n attempt_status: {}\n status_projection_reason: {}\n ownership_state: {}\n liveness_state: {}\n policy_state: {}\n terminalization_state: {}\n lane_control_next_action: {}\n lane_control_conditions: {}\n phase: {}\n wait_reason: {}\n current_operation: {}\n run_lease: {}\n queue_lease_state: {}\n queue_lease: {}\n execution_liveness: {}\n has_fresh_execution: {}\n counts_as_running: {}\n needs_attention: {}\n freshness_at: {}\n freshness_source: {}\n timing: run_idle={} protocol_idle={} last_progress={} protocol_event={} events={}\n account: {}\n accounts: {}\n child_agent_activity: {}\n protocol_activity: {}\n context_pressure: {}\n private_evidence: {}\n loop_status: {}\n loop_review: {}\n loop_architecture_recovery: {}\n loop_boundary: {}\n control_capability: {}\n thread_id: {}\n turn_id: {}\n thread_status: {}\n thread_active_flags: {}\n interactive_requested: {}\n continuation_pending: {}\n continuation_recovery: {}\n branch: {}\n worktree_path: {}\n updated_at: {}\n last_run_activity_at: {}\n last_protocol_activity_at: {}\n last_progress_at: {}\n idle_for_seconds: {}\n protocol_idle_for_seconds: {}\n suspected_stall: {}\n progress_diagnostic: {}\n process_id: {}\n process_alive: {}\n process_liveness_reason: {}\n retry_kind: {}\n next_retry_at: {}\n effective_model: {}\n effective_model_provider: {}\n effective_cwd: {}\n effective_approval_policy: {}\n effective_approvals_reviewer: {}\n effective_sandbox_mode: {}\n protocol_event: {}\n event_count: {}\n", run.run_id, run.project_id, run.issue_id, @@ -8653,7 +8995,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { run.phase, run.wait_reason.as_deref().unwrap_or("none"), run.current_operation, - if run.active_lease { "yes" } else { "no" }, + if run.run_lease { "yes" } else { "no" }, run.queue_lease_state, queue_lease, run.execution_liveness, @@ -8684,6 +9026,7 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { thread_active_flags, if run.interactive_requested { "yes" } else { "no" }, if run.continuation_pending { "yes" } else { "no" }, + continuation_recovery, branch_name, worktree_path, run.updated_at, @@ -8713,6 +9056,15 @@ fn append_rendered_run(output: &mut String, run: &OperatorRunStatus) { )); } +fn render_run_protocol_event(run: &OperatorRunStatus) -> String { + match (&run.last_event_type, &run.last_event_at) { + (Some(event_type), Some(timestamp)) => format!("{event_type} @ {timestamp}"), + (Some(event_type), None) => event_type.clone(), + (None, Some(timestamp)) => timestamp.clone(), + (None, None) => String::from("none"), + } +} + fn render_lane_control_conditions(run: &OperatorRunStatus) -> String { if run.lane_control_conditions.is_empty() { String::from("none") @@ -8721,8 +9073,41 @@ fn render_lane_control_conditions(run: &OperatorRunStatus) -> String { } } +fn render_continuation_recovery_summary( + recovery: Option<&OperatorContinuationRecoveryStatus>, +) -> String { + let Some(recovery) = recovery else { + return String::from("none"); + }; + let message = recovery + .source_error_message + .as_deref() + .map(single_line_status_value) + .unwrap_or_else(|| String::from("none")); + + format!( + "state={} source_phase={} next_phase={} source_error_class={} source_error_message={} count={}/{} budget_exceeded={} recorded_at={} run_id={} attempt={} next_action={}", + recovery.state, + recovery.source_phase, + recovery.next_phase, + recovery.source_error_class, + message, + recovery.recovery_count, + recovery.automatic_continuation_limit, + if recovery.budget_exceeded { "yes" } else { "no" }, + recovery.recorded_at, + recovery.run_id, + recovery.attempt_number, + recovery.next_action, + ) +} + +fn single_line_status_value(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + fn operator_run_queue_lease_summary(run: &OperatorRunStatus) -> String { - if run.active_lease { + if run.run_lease { return String::from("held"); } @@ -8742,7 +9127,7 @@ fn operator_run_queue_lease_summary(run: &OperatorRunStatus) -> String { } fn operator_run_freshness(run: &OperatorRunStatus) -> (&'static str, &str) { - if operator_run_counts_as_active(run) { + if operator_run_counts_as_current_lane(run) { if let Some(timestamp) = run.last_run_activity_at.as_deref() { return ("last_run_activity_at", timestamp); } diff --git a/apps/decodex/src/orchestrator/tests.rs b/apps/decodex/src/orchestrator/tests.rs index 23ce167ea..553384d46 100644 --- a/apps/decodex/src/orchestrator/tests.rs +++ b/apps/decodex/src/orchestrator/tests.rs @@ -22,7 +22,7 @@ use time::OffsetDateTime; use crate::{orchestrator::RepoGatePhaseGoalController, tracker::records}; #[rustfmt::skip] use crate::agent::{ - ACTIVE_RUN_IDLE_TIMEOUT, MODEL_EXECUTION_IDLE_TIMEOUT, + RUN_LEASE_IDLE_TIMEOUT, MODEL_EXECUTION_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerTransportFailure, AppServerTurnFailure, @@ -36,7 +36,7 @@ use crate::config::{ReviewLevel, ServiceConfig}; use crate::github; use crate::loop_contract::DecisionContract; #[rustfmt::skip] -use crate::orchestrator::{self, ActiveChildRunContext, ActiveRunDisposition, ActiveRunReconciliation, ActiveWorkflowOverride, AgentEvidenceSource, AuthorityBoundaryCheckInput, AuthorityBoundaryDisposition, AuthorityDecisionRequestInput, ChildExitRetryContext, ChildRunRef, ControlPlaneProjectTick, CONTINUATION_PENDING_RUN_STATUS, DaemonRunChild, DaemonTickRuntimeContext, DashboardEventHub, EvidenceRequest, GhPullRequestReviewStateInspector, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, IssueDispatchMode, IssueRunPlan, IssueTurnContinuationGuard, ManualAttentionRequested, OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH, OPERATOR_DASHBOARD_ENDPOINT_PATH, OperatorCodexAccountControlStatus, OperatorExecutionProgramNodeStatus, OperatorExecutionProgramStatus, OperatorGitHubCliAuthority, OperatorProjectStatus, OperatorStatusSnapshot, PostReviewLaneClassification, PostReviewLaneDecision, PostReviewLaneSnapshot, PreferredRunIdentity, PrepareIssueRunContext, PublishedOperatorSnapshot, PullRequestCommitConnection, PullRequestCommitNode, PullRequestCommitPayload, PullRequestIssueCommentConnection, PullRequestIssueCommentState, PullRequestIssueCommentsNode, PullRequestPageInfo, PullRequestReactionGroup, PullRequestReactionUsersConnection, PullRequestActor, PullRequestRepository, PullRequestRepositoryOwner, PullRequestReviewConnection, PullRequestIssueCommentNode, PullRequestReviewNode, PullRequestReviewRequestConnection, PullRequestReviewState, PullRequestReviewStateInspector, PullRequestReviewStateNode, PullRequestReviewStateRepository, PullRequestReviewSummaryState, PullRequestReviewThreadConnection, PullRequestReviewThreadNode, PullRequestStatusCheckRollup, RecoveredRuntimeState, RetainedPartialProgress, RetainedReviewRunIdentity, RetryComment, RetryDispatchDecision, RetryEntry, RetryKind, RetryQueue, RunCompletionDisposition, RunSummary, RepoGateFailure, TERMINAL_GUARD_MARKER_FILE, TERMINAL_GUARDED_RUN_STATUS, TRACKER_RATE_LIMIT_WARNING, TargetIssueRunContext, EXTERNAL_REVIEW_ACTOR_LOGIN, EXTERNAL_REVIEW_PASS_PHRASE, EXTERNAL_REVIEW_REQUEST_BODY}; +use crate::orchestrator::{self, CurrentChildRunContext, RunLeaseDisposition, RunLeaseReconciliation, ActiveWorkflowOverride, AgentEvidenceSource, AuthorityBoundaryCheckInput, AuthorityBoundaryDisposition, AuthorityDecisionRequestInput, ChildExitRetryContext, ChildRunRef, ControlPlaneProjectTick, CONTINUATION_PENDING_RUN_STATUS, DaemonRunChild, DaemonTickRuntimeContext, DashboardEventHub, EvidenceRequest, GhPullRequestReviewStateInspector, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, IssueDispatchMode, IssueRunPlan, IssueTurnContinuationGuard, ManualAttentionRequested, OPERATOR_DASHBOARD_ALIAS_ENDPOINT_PATH, OPERATOR_DASHBOARD_ENDPOINT_PATH, PHASE_GOAL_RECOVERY_BLOCKED_EVENT_TYPE, PHASE_GOAL_RECOVERY_EVENT_TYPE, OperatorCodexAccountControlStatus, OperatorExecutionProgramNodeStatus, OperatorExecutionProgramStatus, OperatorGitHubCliAuthority, OperatorProjectStatus, OperatorStatusSnapshot, PostReviewLaneClassification, PostReviewLaneDecision, PostReviewLaneSnapshot, PreferredRunIdentity, PrepareIssueRunContext, PublishedOperatorSnapshot, PullRequestCommitConnection, PullRequestCommitNode, PullRequestCommitPayload, PullRequestIssueCommentConnection, PullRequestIssueCommentState, PullRequestIssueCommentsNode, PullRequestPageInfo, PullRequestReactionGroup, PullRequestReactionUsersConnection, PullRequestActor, PullRequestRepository, PullRequestRepositoryOwner, PullRequestReviewConnection, PullRequestIssueCommentNode, PullRequestReviewNode, PullRequestReviewRequestConnection, PullRequestReviewState, PullRequestReviewStateInspector, PullRequestReviewStateNode, PullRequestReviewStateRepository, PullRequestReviewSummaryState, PullRequestReviewThreadConnection, PullRequestReviewThreadNode, PullRequestStatusCheckRollup, RecoveredRuntimeState, RetainedPartialProgress, RetainedReviewRunIdentity, RetryComment, RetryDispatchDecision, RetryEntry, RetryKind, RetryQueue, RunCompletionDisposition, RunSummary, RepoGateFailure, TERMINAL_GUARD_MARKER_FILE, TERMINAL_GUARDED_RUN_STATUS, TRACKER_RATE_LIMIT_WARNING, TargetIssueRunContext, EXTERNAL_REVIEW_ACTOR_LOGIN, EXTERNAL_REVIEW_PASS_PHRASE, EXTERNAL_REVIEW_REQUEST_BODY}; #[rustfmt::skip] use crate::prelude::Result; #[rustfmt::skip] diff --git a/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs index 5dcea1924..47d5bae09 100644 --- a/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs +++ b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs @@ -349,7 +349,7 @@ fn targeted_identifier_dispatch_accepts_stopped_active_closeout_lease() { state_store .record_run_attempt("run-1", &issue.id, 1, "running") - .expect("stopped active run attempt should record"); + .expect("stopped running run attempt should record"); state_store .upsert_lease(config.service_id(), &issue.id, "run-1", "Done") .expect("stopped closeout lease should record"); @@ -390,8 +390,8 @@ fn targeted_identifier_dispatch_accepts_stopped_active_closeout_lease() { assert_eq!(lane.classification, "continue"); assert_eq!(lane.reason, "pull_request_merged_closeout_pending"); assert_eq!(lane.pr_state.as_deref(), Some("MERGED")); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].process_alive, Some(false)); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].process_alive, Some(false)); assert!( orchestrator::issue_passes_closeout_dispatch_policy( &tracker, diff --git a/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs b/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs index b00e23e0a..5366ccc49 100644 --- a/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs +++ b/apps/decodex/src/orchestrator/tests/intake/workflow_reload.rs @@ -119,7 +119,7 @@ fn active_child_reconciliation_keeps_spawn_time_workflow_until_exit() { .upsert_lease("pubfi", &stale_issue.id, "run-stale", "In Progress") .expect("stale lease should record"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, ¤t_workflow, @@ -134,7 +134,7 @@ fn active_child_reconciliation_keeps_spawn_time_workflow_until_exit() { }), OffsetDateTime::now_utc().unix_timestamp() + 1, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!( actions.iter().all(|action| action.issue.id != child_issue.id), @@ -142,7 +142,7 @@ fn active_child_reconciliation_keeps_spawn_time_workflow_until_exit() { ); assert!(actions.iter().any(|action| { action.issue.id == stale_issue.id - && matches!(action.disposition, orchestrator::ActiveRunDisposition::NonActive) + && matches!(action.disposition, orchestrator::RunLeaseDisposition::NotDispatchable) })); } diff --git a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs index e82115002..3568a6590 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs @@ -6,12 +6,12 @@ use orchestrator::PrivateEvidenceReadback; fn agent_evidence_snapshot_writes_index_blockers_capsules_and_event_stream() { let temp_dir = TempDir::new().expect("temp dir should create"); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("temp path should be utf-8")); - let mut active_run = operator_status_text_active_run(); + let mut current_lane = operator_status_text_current_lane(); - active_run.suspected_stall = true; - active_run.phase = String::from("stalled"); - active_run.counts_as_running = false; - active_run.needs_attention = true; + current_lane.suspected_stall = true; + current_lane.phase = String::from("stalled"); + current_lane.counts_as_running = false; + current_lane.needs_attention = true; let snapshot = OperatorStatusSnapshot { project_id: String::from(TEST_SERVICE_ID), @@ -27,8 +27,8 @@ fn agent_evidence_snapshot_writes_index_blockers_capsules_and_event_stream() { account_selector: None, }, accounts: Vec::new(), - active_runs: vec![active_run.clone()], - recent_runs: vec![active_run], + current_lanes: vec![current_lane.clone()], + recent_runs: vec![current_lane], history_lanes: Vec::new(), execution_programs: Vec::new(), queued_candidates: vec![agent_evidence_blocked_candidate()], @@ -164,7 +164,7 @@ fn agent_evidence_project_status_with_configured_gh() -> OperatorProjectStatus { "No action needed; Decodex will use the configured GitHub CLI path.", ), }, - active_run_count: 0, + current_lane_count: 0, running_lane_count: 0, queued_candidate_count: 0, post_review_lane_count: 0, diff --git a/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs b/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs index 2ff00098b..27a1df188 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs @@ -29,14 +29,14 @@ fn control_plane_snapshot_lists_disabled_registered_projects() { assert_eq!(snapshot.account_control.mode, "balanced"); assert_eq!(snapshot.account_control.account_selector, None); assert_eq!(project.connector_state, "disabled"); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.retained_worktree_count, 0); assert!(snapshot.warnings.contains(&String::from("no_enabled_projects"))); assert!(project_runtimes.is_empty(), "disabled projects should not be ticked"); } #[test] -fn control_plane_snapshot_includes_disabled_project_active_runs_without_ticking() { +fn control_plane_snapshot_includes_disabled_project_current_lanes_without_ticking() { let (temp_dir, config, _workflow) = temp_project_layout(); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("home should be utf-8")); @@ -55,10 +55,10 @@ fn control_plane_snapshot_includes_disabled_project_active_runs_without_ticking( observer_store.upsert_project(®istration).expect("project should register"); writer_store .record_run_attempt("run-disabled-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); writer_store .upsert_lease(config.service_id(), &issue.id, "run-disabled-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); let mut project_runtimes = HashMap::new(); let snapshot = @@ -71,11 +71,11 @@ fn control_plane_snapshot_includes_disabled_project_active_runs_without_ticking( assert_eq!(project.project_id, "pubfi"); assert!(!project.enabled); assert_eq!(project.connector_state, "disabled"); - assert_eq!(project.active_run_count, 1); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].run_id, "run-disabled-active"); - assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); - assert_eq!(snapshot.active_runs[0].phase, "executing"); + assert_eq!(project.current_lane_count, 1); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].run_id, "run-disabled-active"); + assert_eq!(snapshot.current_lanes[0].project_id, "pubfi"); + assert_eq!(snapshot.current_lanes[0].phase, "executing"); assert!(snapshot.warnings.contains(&String::from("no_enabled_projects"))); assert!(project_runtimes.is_empty(), "disabled projects should not be ticked"); } @@ -202,10 +202,10 @@ fn control_plane_dev_snapshot_does_not_tick_enabled_projects() { assert_eq!(project.project_id, "pubfi"); assert!(project.enabled); assert_eq!(project.connector_state, "dev"); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.queued_candidate_count, 0); assert_eq!(project.warning_count, 1); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert!(snapshot.queued_candidates.is_empty()); assert!(snapshot.warnings.contains(&String::from("automation_disabled"))); assert!(!snapshot.warnings.contains(&String::from("no_enabled_projects"))); @@ -237,13 +237,13 @@ fn control_plane_dev_snapshot_marks_unloadable_project_config() { assert!(project.enabled); assert_eq!(project.connector_state, "config_error"); assert_eq!(project.warning_count, 2); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert!(snapshot.warnings.contains(&String::from("automation_disabled"))); assert!(snapshot.warnings.contains(&String::from("operator_snapshot_build_failed"))); } #[test] -fn control_plane_dev_snapshot_includes_local_active_runs() { +fn control_plane_dev_snapshot_includes_local_current_lanes() { let (temp_dir, config, _workflow) = temp_project_layout(); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("home should be utf-8")); @@ -260,10 +260,10 @@ fn control_plane_dev_snapshot_includes_local_active_runs() { state_store.upsert_project(®istration).expect("project should register"); state_store .record_run_attempt("run-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(config.service_id(), &issue.id, "run-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); let snapshot = orchestrator::run_control_plane_dev_tick(&state_store).expect("dev snapshot should build"); @@ -272,18 +272,18 @@ fn control_plane_dev_snapshot_includes_local_active_runs() { assert_eq!(snapshot.projects.len(), 1); assert_eq!(project.project_id, "pubfi"); assert_eq!(project.connector_state, "dev"); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 1); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].run_id, "run-active"); - assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); - assert_eq!(snapshot.active_runs[0].phase, "executing"); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].run_id, "run-active"); + assert_eq!(snapshot.current_lanes[0].project_id, "pubfi"); + assert_eq!(snapshot.current_lanes[0].phase, "executing"); assert!(snapshot.queued_candidates.is_empty()); assert!(snapshot.warnings.contains(&String::from("automation_disabled"))); } #[test] -fn control_plane_dev_snapshot_separates_visible_active_runs_from_running_lanes() { +fn control_plane_dev_snapshot_separates_visible_current_lanes_from_running_lanes() { let (temp_dir, config, _workflow) = temp_project_layout(); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("home should be utf-8")); @@ -301,10 +301,10 @@ fn control_plane_dev_snapshot_separates_visible_active_runs_from_running_lanes() state_store.upsert_project(®istration).expect("project should register"); state_store .record_run_attempt("run-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(config.service_id(), &issue.id, "run-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store .upsert_worktree( config.service_id(), @@ -321,10 +321,10 @@ fn control_plane_dev_snapshot_separates_visible_active_runs_from_running_lanes() let snapshot = orchestrator::run_control_plane_dev_tick(&state_store).expect("dev snapshot should build"); let project = snapshot.projects.first().expect("enabled project should be listed"); - let run = snapshot.active_runs.first().expect("stopped active run should stay visible"); + let run = snapshot.current_lanes.first().expect("stopped current lane should stay visible"); - assert_eq!(project.active_run_count, snapshot.active_runs.len()); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, snapshot.current_lanes.len()); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); assert_eq!(run.run_id, "run-active"); @@ -374,10 +374,10 @@ fn control_plane_snapshot_aggregates_top_level_lanes_for_all_registered_projects state_store .record_run_attempt("run-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(active_config.service_id(), &issue.id, "run-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); let active_snapshot = orchestrator::build_operator_status_snapshot(&active_config, &state_store, 10) @@ -416,18 +416,18 @@ fn control_plane_snapshot_aggregates_top_level_lanes_for_all_registered_projects assert_eq!(snapshot.project_id, "all"); assert_eq!(snapshot.projects.len(), 2); assert_eq!( - project_by_id.get("pubfi").expect("active project summary should exist").active_run_count, + project_by_id.get("pubfi").expect("active project summary should exist").current_lane_count, 1, ); assert_eq!( - project_by_id.get("rsnap").expect("idle project summary should exist").active_run_count, + project_by_id.get("rsnap").expect("idle project summary should exist").current_lane_count, 0, ); assert_eq!(snapshot.account_control.mode, "balanced"); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].run_id, "run-active"); - assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); - assert_eq!(snapshot.active_runs[0].phase, "executing"); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].run_id, "run-active"); + assert_eq!(snapshot.current_lanes[0].project_id, "pubfi"); + assert_eq!(snapshot.current_lanes[0].phase, "executing"); } #[test] @@ -472,10 +472,10 @@ fn status_cache_projects_aggregate_snapshot_to_requested_project() { state_store .record_run_attempt("run-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(active_config.service_id(), &issue.id, "run-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store .upsert_worktree( active_config.service_id(), @@ -530,8 +530,8 @@ fn status_cache_projects_aggregate_snapshot_to_requested_project() { assert_eq!(cached.snapshot_age_seconds, Some(5)); assert_eq!(cached.projects.len(), 1); assert_eq!(cached.projects[0].project_id, "pubfi"); - assert_eq!(cached.active_runs.len(), 1); - assert_eq!(cached.active_runs[0].project_id, "pubfi"); + assert_eq!(cached.current_lanes.len(), 1); + assert_eq!(cached.current_lanes[0].project_id, "pubfi"); assert!(cached.worktrees.iter().all(|worktree| worktree.project_id == "pubfi")); assert!(cached.recent_runs.iter().all(|run| run.project_id == "pubfi")); } diff --git a/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs b/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs index c5e7af63f..14122dd55 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs @@ -287,21 +287,21 @@ fn operator_dashboard_cards_and_accounts_share_running_lane_typography() { } #[test] -fn operator_dashboard_patches_active_run_cards_without_replacing_the_list() { +fn operator_dashboard_patches_current_lane_cards_without_replacing_the_list() { let response = dashboard_response(); assert!(response.contains("function renderStableList(container, html)")); assert!(response.contains("function animateStableListSize(container, startHeight)")); assert!(response.contains("function markStableListEnter(node)")); assert!(response.contains("function patchChildNodes(current, next)")); - assert!(response.contains("function activeRunRenderKey(run)")); + assert!(response.contains("function currentLaneRenderKey(run)")); assert!(response.contains("data-render-key=\"${escapeHtml(renderKey)}\"")); - assert!(response.contains("renderStableList(\n\t\t\t\t\tnodes.activeRuns,")); + assert!(response.contains("renderStableList(\n\t\t\t\t\tnodes.currentLanes,")); assert!(response.contains("markStableListEnter(clone);")); assert!(response.contains("container.style.height = `${startHeight}px`;")); assert!(response.contains(".is-list-entering")); assert!(response.contains("@keyframes stable-list-item-enter")); - assert!(!response.contains("nodes.activeRuns.innerHTML = runs")); + assert!(!response.contains("nodes.currentLanes.innerHTML = runs")); assert!(response.contains("return node.dataset.renderKey || node.dataset.detailKey || \"\";")); assert!(response.contains("current.closest(\"details.is-animating\")")); assert!(response.contains("width var(--slow) var(--ease),")); @@ -482,7 +482,7 @@ fn assert_running_lane_meta_contract(response: &str) { "runningLaneMetaText", "const parts = [`${derived.liveRuns ?? 0} running`];", "attentionCount === 1", - "nodes.activeRunsMeta,", + "nodes.currentLanesMeta,", "runningLaneMetaText(derived),", ], ); @@ -514,7 +514,7 @@ fn assert_liveness_and_cleanup_contract(response: &str) { "runLaneControlConditionsSummary", "runQueueLeaseSummary", "return displayToken(run.execution_liveness || \"liveness_unknown\");", - "return displayToken(run.ownership_state || (runCountsAsRunning(run) ? \"owned_active\" : \"unknown\"));", + "return displayToken(run.ownership_state || (runCountsAsRunning(run) ? \"leased_run\" : \"unknown\"));", "field(\"Attempt status\", run.attempt_status || run.status)", "field(\"Queue lease\", runQueueLeaseSummary(run))", "field(\"Execution liveness\", runExecutionLivenessSummary(run))", @@ -592,7 +592,7 @@ fn operator_dashboard_keys_child_bucket_rows_for_stable_patching() { } #[test] -fn operator_dashboard_active_run_status_copy_stays_concise() { +fn operator_dashboard_current_lane_status_copy_stays_concise() { let response = dashboard_response(); assert!(response.contains("runNeedsAttention")); @@ -724,7 +724,7 @@ fn operator_dashboard_renders_account_usage_controls() { assert!(!response.contains("Running · Intake")); assert!(!response.contains("Review · Recovery · History")); assert!(!response.contains("data-fold-key=\"panel:projects\"")); - assert!(response.contains("panel section-execution\" id=\"active-panel\"")); + assert!(response.contains("panel section-execution\" id=\"current-lanes-panel\"")); assert!(response.contains("panel section-aftercare\" id=\"review-panel\"")); assert!(!response.contains("section-group-start")); assert!(response.contains("#queue-panel .panel-head")); @@ -736,7 +736,7 @@ fn operator_dashboard_renders_account_usage_controls() { assert!(response.contains("primary: [\"accountPool\", \"projects\", \"active\", \"programs\", \"queue\", \"review\", \"worktrees\", \"recent\"]")); assert!(!response.contains("#account-pool-panel {")); assert!(!response.contains("No accounts")); - assert!(response.contains("#active-panel {\n\t\t\t\tbackground: transparent;")); + assert!(response.contains("#current-lanes-panel {\n\t\t\t\tbackground: transparent;")); assert!(!response.contains("account-pool-title")); assert!(response.contains("account-privacy-toggle")); assert!(response.contains("account-eye-open")); @@ -853,7 +853,7 @@ fn operator_dashboard_account_errors_route_to_notice_dock_with_privacy() { fn operator_dashboard_uses_expanded_section_titles() { let response = dashboard_response(); - assert!(response.contains("

Running Lanes

")); + assert!(response.contains("

Current Lanes

")); assert!(response.contains("

Intake Queue

")); assert!(response.contains("

Review & Landing

")); assert!(response.contains("

Recovery Worktrees

")); @@ -1502,8 +1502,8 @@ fn operator_dashboard_projects_show_compact_activity_work_and_location() { assert!(response.contains("`${cleanup} cleanup`")); assert!(response.contains("run.process_alive !== false")); assert!(response.contains("running_lane_count: runningCountsByProject.get(project.project_id) || 0")); - assert!(!response.contains("const running = project.active_run_count ?? 0;")); - assert!(!response.contains("`${project.active_run_count ?? 0} running`")); + assert!(!response.contains("const running = project.current_lane_count ?? 0;")); + assert!(!response.contains("`${project.current_lane_count ?? 0} running`")); assert!(response.contains("projectNumber(project.cleanup_blocked_count)")); assert!(response.contains("projectNumber(project.cleanup_pending_count)")); assert!(!response.contains("[project.post_review_lane_count ?? 0, \"review/land\"]")); @@ -1559,8 +1559,8 @@ fn operator_dashboard_normalizes_review_state_tokens() { fn operator_dashboard_review_cards_omit_static_summary_copy() { let response = dashboard_response(); - assert!(response.contains("const shadowedByActiveRun =")); - assert!(response.contains("status: activeRun ? `run ${displayToken(activeRun.phase)}` : \"active run\"")); + assert!(response.contains("const shadowedByCurrentLane =")); + assert!(response.contains("status: currentLane ? `run ${displayToken(currentLane.phase)}` : \"current lane\"")); assert!(response.contains("function postReviewBlockerStatus(lane, blockerScope)")); assert!(response.contains("status: postReviewBlockerStatus(lane, blockerScope)")); assert!(response.contains("summary: \"\",\n\t\t\t\t\t\t\tstatus: lane.check_state")); @@ -1660,11 +1660,11 @@ fn operator_dashboard_flow_counts_distinguish_intake_attention() { fn operator_dashboard_does_not_hide_claimed_queue_without_local_lane() { let response = dashboard_response(); - assert!(response.contains("const activeRunByIssue = new Map();")); + assert!(response.contains("const currentLaneByIssue = new Map();")); assert!(response.contains("for (const key of issueIdentityKeys(run))")); - assert!(response.contains("const activeRun = issueIdentityKeys(candidate)")); - assert!(response.contains("if (activeRun) {")); - assert!(!response.contains("activeRun && candidate.classification === \"claimed\"")); + assert!(response.contains("const currentLane = issueIdentityKeys(candidate)")); + assert!(response.contains("if (currentLane) {")); + assert!(!response.contains("currentLane && candidate.classification === \"claimed\"")); assert!(!response.contains("candidate.classification !== \"claimed\" &&")); } @@ -1763,7 +1763,7 @@ fn operator_dashboard_header_shows_endpoint_and_snapshot_freshness() { fn operator_dashboard_active_freshness_prefers_live_activity_source() { let response = dashboard_response(); - assert!(response.contains("function activeRunFreshness(run)")); + assert!(response.contains("function currentLaneFreshness(run)")); assert!(response.contains("source: \"last_run_activity_at\"")); assert!(response.contains("source: \"none\"")); assert!(!response.contains("source: \"updated_at\"")); @@ -1778,12 +1778,12 @@ fn operator_dashboard_active_freshness_prefers_live_activity_source() { assert!(response.contains("facts.push([\"lane idle\", formatDuration(run.idle_for_seconds)]);")); assert!(response.contains("facts.push([\"agent idle\", formatDuration(run.protocol_idle_for_seconds)]);")); assert!(response.contains("facts.push([\"focus\", detailLabel(focus)]);")); - assert!(response.contains("function activeRunLifecycleMetrics(run, summary = childAgentActivity(run))")); + assert!(response.contains("function currentLaneLifecycleMetrics(run, summary = childAgentActivity(run))")); assert!(response.contains("function lifecycleMetricFacts(metrics, { includeAttempts = false } = {})")); assert!(response.contains("facts.push([\"tokens\", tokenSummary]);")); assert!(response.contains("facts.push([\"tools\", formatCompactCount(metrics.tool_call_count)]);")); assert!(response.contains("\"max output\",")); - assert!(response.contains("function childAgentContextRows(run, summary, lifecycle = activeRunLifecycleMetrics(run, summary))")); + assert!(response.contains("function childAgentContextRows(run, summary, lifecycle = currentLaneLifecycleMetrics(run, summary))")); assert!(response.contains("renderChildLifecycleOverview(lifecycle, contextFacts)")); assert!(response.contains("renderChildLifecyclePhaseTable(lifecycle.phases || [])")); assert!(!response.contains("rows.push(renderChildContextRow(\"Total\", totalFacts, \"is-total\"));")); @@ -1801,10 +1801,10 @@ fn operator_dashboard_active_freshness_prefers_live_activity_source() { assert!(!response.contains("m ago")); assert!(!response.contains("h ago")); assert!(!response.contains("d ago")); - assert!(response.contains("function activeRunTelemetryFacts(run)")); + assert!(response.contains("function currentLaneTelemetryFacts(run)")); assert!(response.contains("function renderRunTelemetryMetaItems(run)")); assert!(response.contains("function renderRunMetaFact(label, value, valueClass = \"\", title = \"\")")); - assert!(!response.contains("renderActiveRunActivityStrip(run)")); + assert!(!response.contains("renderCurrentLaneActivityStrip(run)")); assert!(!response.contains("run-activity-strip")); assert!(!response.contains("function renderActiveTelemetryLine(run)")); assert!(!response.contains("activity-line")); @@ -1815,8 +1815,8 @@ fn operator_dashboard_active_freshness_prefers_live_activity_source() { assert!(!response.contains("Last ${freshness.sourceLabel}")); assert!(!response.contains("Latest ${freshness.sourceLabel}")); assert!(!response.contains("renderTimingStrip(run)")); - assert!(!response.contains("activeRunFreshnessSource(run)")); - assert!(!response.contains("field(\"Freshness source\", activeRunFreshnessSource(run))")); + assert!(!response.contains("currentLaneFreshnessSource(run)")); + assert!(!response.contains("field(\"Freshness source\", currentLaneFreshnessSource(run))")); assert!(response.contains("field(\"Updated\", formatTimestamp(run.updated_at))")); } @@ -1846,15 +1846,15 @@ fn operator_dashboard_run_activity_preserves_snapshot_detail_fields() { assert!(response.contains("function mergeDashboardRunRecord(snapshotRun, activityRun)")); assert!( - response.contains("function mergeDashboardActiveRuns(snapshot, activeRunRows, activeRunsComplete = true)") + response.contains("function mergeDashboardCurrentLanes(snapshot, currentLaneRows, currentLanesComplete = true)") ); assert!(response.contains("function dashboardRunTitleIsOperationFallback(run)")); assert!(response.contains("const operationFallback = displayToken(run.current_operation || run.phase);")); assert!(response.contains("!(fallback !== \"unknown\" && title === operationFallback)")); assert!(response.contains("return fallback !== \"unknown\" ? fallback : operationFallback;")); - assert!(response.contains("let dashboardLiveActiveRuns = [];")); + assert!(response.contains("let dashboardLiveCurrentLanes = [];")); assert!(response.contains("let dashboardLiveRunActivitySeen = false;")); - assert!(response.contains("let dashboardLiveActiveRunsComplete = true;")); + assert!(response.contains("let dashboardLiveCurrentLanesComplete = true;")); assert!(response.contains("let dashboardLiveAccounts = null;")); assert!(response.contains("let dashboardLiveAccountControl = null;")); assert!(response @@ -1877,14 +1877,14 @@ fn operator_dashboard_run_activity_preserves_snapshot_detail_fields() { assert!(response.contains("dashboardRunTitleIsOperationFallback(activityRun)")); assert!(response.contains("merged.title = snapshotRun.title;")); assert!( - response.contains("const activeRunsComplete =\n\t\t\t\t\tactivityPayload.activeRunsComplete !== false") + response.contains("const currentLanesComplete =\n\t\t\t\t\tactivityPayload.currentLanesComplete !== false") ); assert!( - response.contains("const mergedActiveRuns = mergeDashboardActiveRuns(\n\t\t\t\t\tsnapshot,\n\t\t\t\t\tactiveRunRows,\n\t\t\t\t\tactiveRunsComplete,\n\t\t\t\t);") + response.contains("const mergedCurrentLanes = mergeDashboardCurrentLanes(\n\t\t\t\t\tsnapshot,\n\t\t\t\t\tcurrentLaneRows,\n\t\t\t\t\tcurrentLanesComplete,\n\t\t\t\t);") ); - assert!(response.contains("dashboardLiveActiveRuns = payload.activeRuns")); + assert!(response.contains("dashboardLiveCurrentLanes = payload.currentLanes")); assert!(response.contains("dashboardLiveRunActivitySeen = true;")); - assert!(response.contains("dashboardLiveActiveRunsComplete =")); + assert!(response.contains("dashboardLiveCurrentLanesComplete =")); assert!(response.contains("snapshot: snapshotWithLiveRunActivity(payload.snapshot),")); assert!(response.contains( "snapshot: snapshotWithLiveRunActivity(lastDashboardRender.snapshot, {\n\t\t\t\t\t\tincludeCompletedEmpty: true,\n\t\t\t\t\t})," @@ -1892,8 +1892,8 @@ fn operator_dashboard_run_activity_preserves_snapshot_detail_fields() { assert!(response.contains("clearDashboardLiveRunActivityOverlayIfCompleteEmpty();")); assert!(response.contains("account_control: accountControl,")); assert!(response.contains("accounts,")); - assert!(response.contains("active_runs: mergedActiveRuns,")); - assert!(!response.contains("active_runs: activeRunRows,")); + assert!(response.contains("current_lanes: mergedCurrentLanes,")); + assert!(!response.contains("current_lanes: currentLaneRows,")); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/operator/status/history.rs b/apps/decodex/src/orchestrator/tests/operator/status/history.rs index 95eb4e03d..a15ab88a3 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/history.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/history.rs @@ -1,5 +1,5 @@ #[test] -fn operator_status_history_limit_applies_after_active_runs_are_split_out() { +fn operator_status_history_limit_applies_after_current_lanes_are_split_out() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let active_issue = sample_issue_with_sort_fields( @@ -21,10 +21,10 @@ fn operator_status_history_limit_applies_after_active_runs_are_split_out() { state_store .record_run_attempt("run-active", &active_issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease("pubfi", &active_issue.id, "run-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store .upsert_worktree( "pubfi", @@ -50,12 +50,12 @@ fn operator_status_history_limit_applies_after_active_runs_are_split_out() { let rendered = orchestrator::render_operator_status(&snapshot); assert_eq!(snapshot.run_limit, 1); - assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.current_lanes.len(), 1); assert_eq!(snapshot.recent_runs.len(), 2); assert_eq!(snapshot.history_lanes.len(), 1); assert_eq!(snapshot.history_lanes[0].attempt_count, 1); assert!(rendered.contains( - "Run ledger shown: 1 issue lanes from 1 history attempts (running lanes inline)" + "Run ledger shown: 1 issue lanes from 1 history attempts (current lanes inline)" )); assert_eq!(rendered.matches("run_id: run-active").count(), 1); assert_eq!(rendered.matches("run_id: run-failed").count(), 1); @@ -65,7 +65,7 @@ fn operator_status_history_limit_applies_after_active_runs_are_split_out() { assert!( failed_index > history_index, - "non-running history run should remain visible after running lane overlap is hidden" + "history-only run should remain visible after current lane overlap is hidden" ); } @@ -527,6 +527,84 @@ fn local_operator_history_lanes_prefer_terminal_ledger_outcome() { ); } +#[test] +fn live_status_terminal_cleanup_demotes_unleased_protocol_observed_current_lane() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue_with_sort_fields( + "issue-xy-952", + "XY-952", + "Done", + &[], + Some(3), + "2026-06-16T08:50:00Z", + ); + let tracker = FakeTracker::new(vec![issue.clone()]); + let worktree_path = config.worktree_root().join("XY-952"); + + state_store + .record_run_attempt("xy-952-attempt-2-1781598614", &issue.id, 2, "running") + .expect("stale running attempt should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "y/elf-xy-952", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + state_store + .append_event( + "xy-952-attempt-2-1781598614", + 1, + "item/tool/call", + "{\"tool\":\"issue_progress_checkpoint\"}", + ) + .expect("protocol evidence should record"); + + seed_local_linear_execution_events( + &state_store, + &successful_linear_execution_history_comments_with_cleanup(&issue), + ); + + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let rendered = orchestrator::render_operator_status(&snapshot); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert!(snapshot.current_lanes.is_empty()); + assert_eq!(snapshot.projects[0].current_lane_count, 0); + assert_eq!(snapshot.projects[0].running_lane_count, 0); + assert_eq!(snapshot.projects[0].attention_count, 0); + assert_eq!(snapshot.history_lanes.len(), 1); + assert_eq!(snapshot.history_lanes[0].issue_identifier.as_deref(), Some("XY-952")); + assert_eq!( + snapshot.history_lanes[0].latest_run.run_id, + "xy-952-attempt-2-1781598614" + ); + assert_eq!(snapshot.history_lanes[0].latest_run.status, "cleanup_complete"); + assert_eq!(snapshot.history_lanes[0].latest_run.phase, "completed"); + assert_eq!( + snapshot.history_lanes[0].latest_run.current_operation, + "ledger_outcome" + ); + assert_eq!( + snapshot.history_lanes[0].ledger_outcome.final_outcome, + "cleanup_complete" + ); + assert_eq!(snapshot_json["current_lanes"].as_array().map(Vec::len), Some(0)); + assert!(rendered.contains("Current lanes: 0")); + assert!(rendered.contains("Running lanes: 0")); + assert!(rendered.contains("\nCurrent Lanes\n- none\n")); + assert!(rendered.contains("outcome: cleanup_complete")); +} + #[test] fn local_status_summary_counts_terminal_history_needs_attention_without_queue_candidate() { let (_temp_dir, config, workflow) = temp_project_layout(); @@ -638,11 +716,11 @@ fn local_status_summary_ignores_history_only_terminal_attention_without_current_ let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); let rendered = orchestrator::render_operator_status(&snapshot); - assert_eq!(snapshot.active_runs.len(), 0); + assert_eq!(snapshot.current_lanes.len(), 0); assert_eq!(snapshot.queued_candidates.len(), 0); assert_eq!(snapshot.post_review_lanes.len(), 0); assert_eq!(snapshot.worktrees.len(), 0); - assert_eq!(snapshot.projects[0].active_run_count, 0); + assert_eq!(snapshot.projects[0].current_lane_count, 0); assert_eq!(snapshot.projects[0].running_lane_count, 0); assert_eq!(snapshot.projects[0].queued_candidate_count, 0); assert_eq!(snapshot.projects[0].post_review_lane_count, 0); @@ -654,7 +732,7 @@ fn local_status_summary_ignores_history_only_terminal_attention_without_current_ assert_eq!(lane.ledger_outcome.final_outcome, "needs_attention"); assert_eq!(lane.latest_run.status, "needs_attention"); assert_eq!(lane.latest_run.phase, "needs_attention"); - assert!(!lane.latest_run.active_lease); + assert!(!lane.latest_run.run_lease); assert_eq!(snapshot_json["projects"][0]["attention_count"], 0); assert_eq!(snapshot_json["worktrees"].as_array().map(Vec::len), Some(0)); assert_eq!( @@ -845,7 +923,7 @@ fn live_status_treats_adopted_ready_to_land_history_attention_as_history_only() mergeable: Some(String::from("MERGEABLE")), check_state: Some(String::from("SUCCESS")), unresolved_review_threads: Some(0), - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: None, readback_root_cause: None, loop_status: None, diff --git a/apps/decodex/src/orchestrator/tests/operator/status/http.rs b/apps/decodex/src/orchestrator/tests/operator/status/http.rs index 2e78fb427..587120c7b 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/http.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/http.rs @@ -13,14 +13,14 @@ use crate::runtime; const OPERATOR_DASHBOARD_TEST_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(unix)] -struct ActiveLeaseMissingControlFixture { +struct RunLeaseMissingControlFixture { issue: TrackerIssue, channel_path: PathBuf, child: Child, child_process_id: u32, } #[cfg(unix)] -impl Drop for ActiveLeaseMissingControlFixture { +impl Drop for RunLeaseMissingControlFixture { fn drop(&mut self) { if matches!(self.child.try_wait(), Ok(None)) { let _ = self.child.kill(); @@ -80,7 +80,7 @@ fn operator_state_endpoint_serves_dashboard_html_from_root_and_dashboard_route() assert!(!response.contains("error-banner")); assert!(!response.contains("metric-active")); assert!(!response.contains("Queued issue -> reviewed change -> landed branch")); - assert!(response.contains("Running Lanes")); + assert!(response.contains("Current Lanes")); assert!(response.contains("Intake Queue")); assert!(!response.contains("At capacity")); assert!(response.contains("Review & Landing")); @@ -321,7 +321,7 @@ fn operator_dashboard_websocket_sends_current_snapshot_on_connect() { let listener = TcpListener::bind("127.0.0.1:0").expect("listener should bind"); let address = listener.local_addr().expect("listener address should resolve"); let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot { - snapshot_json: Some(br#"{"project_id":"pubfi","active_runs":[]}"#.to_vec()), + snapshot_json: Some(br#"{"project_id":"pubfi","current_lanes":[]}"#.to_vec()), last_publish_unix_epoch: Some(SNAPSHOT_UNIX_EPOCH), })); let state_store = Arc::new(StateStore::open_in_memory().expect("state store should open")); @@ -368,7 +368,7 @@ fn operator_dashboard_websocket_sends_cached_run_activity_on_connect() { let address = listener.local_addr().expect("listener address should resolve"); let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot { snapshot_json: Some( - br#"{"project_id":"pubfi","active_runs":[],"account_control":{"mode":"balanced","account_selector":null}}"#.to_vec(), + br#"{"project_id":"pubfi","current_lanes":[],"account_control":{"mode":"balanced","account_selector":null}}"#.to_vec(), ), last_publish_unix_epoch: Some(1_774_000_000), })); @@ -448,13 +448,13 @@ fn operator_dashboard_websocket_sends_cached_run_activity_on_connect() { payload["type"] == "runActivity" }); - assert_eq!(activity["payload"]["activeRuns"][0]["run_id"], "run-1"); + assert_eq!(activity["payload"]["currentLanes"][0]["run_id"], "run-1"); assert_eq!( - activity["payload"]["activeRuns"][0]["account"]["account_fingerprint"], + activity["payload"]["currentLanes"][0]["account"]["account_fingerprint"], "acct-1" ); assert_eq!( - activity["payload"]["activeRuns"][0]["accounts"][0]["account_fingerprint"], + activity["payload"]["currentLanes"][0]["accounts"][0]["account_fingerprint"], "acct-1" ); @@ -471,7 +471,7 @@ fn operator_dashboard_websocket_accepts_subscription_and_account_selection_contr let address = listener.local_addr().expect("listener address should resolve"); let snapshot = Arc::new(Mutex::new(PublishedOperatorSnapshot { snapshot_json: Some( - br#"{"project_id":"pubfi","active_runs":[],"account_control":{"mode":"balanced","account_selector":null}}"#.to_vec(), + br#"{"project_id":"pubfi","current_lanes":[],"account_control":{"mode":"balanced","account_selector":null}}"#.to_vec(), ), last_publish_unix_epoch: Some(1_774_000_000), })); @@ -728,7 +728,7 @@ fn operator_dashboard_websocket_filters_run_activity_by_subscription() { "runActivity", serde_json::json!({ "emittedAtUnixEpoch": 1_774_000_010_i64, - "activeRuns": [ + "currentLanes": [ { "project_id": "pubfi", "issue_id": "PUB-101", "run_id": "run-1" }, { "project_id": "pubfi", "issue_id": "PUB-102", "run_id": "run-2" }, { "project_id": "rsnap", "issue_id": "RS-1", "run_id": "run-2" } @@ -739,16 +739,16 @@ fn operator_dashboard_websocket_filters_run_activity_by_subscription() { let activity = read_websocket_json_until(&mut client, &mut frame, |payload| { payload["type"] == "runActivity" }); - let active_runs = activity["payload"]["activeRuns"] + let current_lanes = activity["payload"]["currentLanes"] .as_array() - .expect("active runs should list"); + .expect("current lanes should list"); - assert_eq!(active_runs.len(), 1); - assert_eq!(active_runs[0]["project_id"], "pubfi"); - assert_eq!(active_runs[0]["issue_id"], "PUB-102"); - assert_eq!(active_runs[0]["run_id"], "run-2"); - assert_eq!(activity["payload"]["activeRunsComplete"], false); - assert_eq!(activity["payload"]["activeRunScope"], "filtered"); + assert_eq!(current_lanes.len(), 1); + assert_eq!(current_lanes[0]["project_id"], "pubfi"); + assert_eq!(current_lanes[0]["issue_id"], "PUB-102"); + assert_eq!(current_lanes[0]["run_id"], "run-2"); + assert_eq!(activity["payload"]["currentLanesComplete"], false); + assert_eq!(activity["payload"]["currentLaneScope"], "filtered"); drop(client); @@ -989,7 +989,7 @@ fn read_websocket_json_until( } #[test] -fn operator_dashboard_run_activity_event_summarizes_active_runs() { +fn operator_dashboard_run_activity_event_summarizes_current_lanes() { let temp_dir = TempDir::new().expect("temp dir should exist"); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("temp path should be UTF-8")); @@ -1082,34 +1082,34 @@ fn operator_dashboard_run_activity_event_summarizes_active_runs() { assert_eq!(payload["type"], "runActivity"); assert_eq!(data["accountControl"]["mode"], "balanced"); - assert_eq!(data["activeRunsComplete"], true); - assert_eq!(data["activeRunScope"], "complete"); + assert_eq!(data["currentLanesComplete"], true); + assert_eq!(data["currentLaneScope"], "complete"); assert!(data["accounts"].is_array()); assert!(fingerprint.get("emittedAtUnixEpoch").is_none()); assert_eq!(fingerprint["accountControl"]["mode"], "balanced"); - assert_eq!(fingerprint["activeRunsComplete"], true); - assert_eq!(fingerprint["activeRunScope"], "complete"); + assert_eq!(fingerprint["currentLanesComplete"], true); + assert_eq!(fingerprint["currentLaneScope"], "complete"); assert!(fingerprint["accounts"].is_array()); - assert_eq!(fingerprint["activeRuns"][0]["run_id"], "run-1"); + assert_eq!(fingerprint["currentLanes"][0]["run_id"], "run-1"); assert_eq!( - fingerprint["activeRuns"][0]["project_display_name"], + fingerprint["currentLanes"][0]["project_display_name"], "hack-ink/pubfi-mono-v2" ); - assert_eq!(data["activeRuns"][0]["run_id"], "run-1"); - assert_eq!(data["activeRuns"][0]["project_id"], "pubfi"); - assert_eq!(data["activeRuns"][0]["project_display_name"], "hack-ink/pubfi-mono-v2"); - assert_eq!(data["activeRuns"][0]["protocol_activity"]["waiting_reason"], "model"); - assert_eq!(data["activeRuns"][0]["account"]["account_fingerprint"], "acct-1"); - assert_eq!(data["activeRuns"][0]["accounts"][0]["account_fingerprint"], "acct-1"); - assert!(data["activeRuns"][0].get("idle_for_seconds").is_some()); - assert!(data["activeRuns"][0].get("protocol_idle_for_seconds").is_some()); - assert!(fingerprint["activeRuns"][0].get("idle_for_seconds").is_none()); - assert!(fingerprint["activeRuns"][0].get("protocol_idle_for_seconds").is_none()); + assert_eq!(data["currentLanes"][0]["run_id"], "run-1"); + assert_eq!(data["currentLanes"][0]["project_id"], "pubfi"); + assert_eq!(data["currentLanes"][0]["project_display_name"], "hack-ink/pubfi-mono-v2"); + assert_eq!(data["currentLanes"][0]["protocol_activity"]["waiting_reason"], "model"); + assert_eq!(data["currentLanes"][0]["account"]["account_fingerprint"], "acct-1"); + assert_eq!(data["currentLanes"][0]["accounts"][0]["account_fingerprint"], "acct-1"); + assert!(data["currentLanes"][0].get("idle_for_seconds").is_some()); + assert!(data["currentLanes"][0].get("protocol_idle_for_seconds").is_some()); + assert!(fingerprint["currentLanes"][0].get("idle_for_seconds").is_none()); + assert!(fingerprint["currentLanes"][0].get("protocol_idle_for_seconds").is_none()); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] -fn operator_dashboard_run_activity_event_keeps_unleased_app_server_active_run() { +fn operator_dashboard_run_activity_event_keeps_unleased_app_server_current_lane() { let temp_dir = TempDir::new().expect("temp dir should exist"); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("temp path should be UTF-8")); @@ -1166,19 +1166,95 @@ fn operator_dashboard_run_activity_event_keeps_unleased_app_server_active_run() let (payload, _consumed) = websocket_text_payload(&message).expect("event should be a text frame"); let payload: Value = serde_json::from_slice(payload).expect("event data should be json"); let data = &payload["payload"]; - let active_runs = data["activeRuns"].as_array().expect("active runs should list"); + let current_lanes = data["currentLanes"].as_array().expect("current lanes should list"); assert_eq!(payload["type"], "runActivity"); - assert_eq!(data["activeRunsComplete"], true); - assert_eq!(active_runs.len(), 1); - assert_eq!(active_runs[0]["run_id"], "run-1"); - assert_eq!(active_runs[0]["project_id"], "pubfi"); - assert_eq!(active_runs[0]["project_display_name"], "hack-ink/pubfi-mono-v2"); - assert_eq!(active_runs[0]["active_lease"], false); - assert_eq!(active_runs[0]["execution_liveness"], "process_identity_mismatch"); - assert_eq!(active_runs[0]["process_alive"], false); - assert_eq!(active_runs[0]["process_liveness_reason"], "host_boot_id_mismatch"); - assert_eq!(active_runs[0]["thread_status"], "active"); + assert_eq!(data["currentLanesComplete"], true); + assert_eq!(current_lanes.len(), 1); + assert_eq!(current_lanes[0]["run_id"], "run-1"); + assert_eq!(current_lanes[0]["project_id"], "pubfi"); + assert_eq!(current_lanes[0]["project_display_name"], "hack-ink/pubfi-mono-v2"); + assert_eq!(current_lanes[0]["run_lease"], false); + assert_eq!(current_lanes[0]["execution_liveness"], "process_identity_mismatch"); + assert_eq!(current_lanes[0]["process_alive"], false); + assert_eq!(current_lanes[0]["process_liveness_reason"], "host_boot_id_mismatch"); + assert_eq!(current_lanes[0]["thread_status"], "active"); +} + +#[test] +fn operator_dashboard_run_activity_event_demotes_cleanup_complete_unleased_current_lane() { + let temp_dir = TempDir::new().expect("temp dir should exist"); + let _home_guard = + TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("temp path should be UTF-8")); + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let registration = ProjectRegistration::from_config( + config.service_id(), + &service_config_path(config.repo_root()), + &config, + true, + "test-fingerprint", + ); + let issue = sample_issue_with_sort_fields( + "issue-xy-952", + "XY-952", + "Done", + &[], + Some(3), + "2026-06-16T08:50:00Z", + ); + let worktree_path = config.worktree_root().join("XY-952"); + + git_status_success( + config.repo_root(), + &["remote", "add", "origin", "git@github.com:hack-ink/pubfi-mono-v2.git"], + ); + + state_store.upsert_project(®istration).expect("project should register"); + state_store + .record_run_attempt("xy-952-attempt-2-1781598614", &issue.id, 2, "running") + .expect("stale running attempt should record"); + state_store + .upsert_worktree( + config.service_id(), + &issue.id, + "y/elf-xy-952", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + state_store + .append_event( + "xy-952-attempt-2-1781598614", + 1, + "item/tool/call", + "{\"tool\":\"issue_progress_checkpoint\"}", + ) + .expect("protocol evidence should record"); + + seed_local_linear_execution_events( + &state_store, + &successful_linear_execution_history_comments_with_cleanup(&issue), + ); + + let event = + orchestrator::build_operator_run_activity_event(&state_store).expect("event should build"); + let message = orchestrator::dashboard_websocket_message( + event.event.event_type, + &event.event.payload, + ) + .expect("event should serialize"); + let (payload, _consumed) = websocket_text_payload(&message).expect("event should be a text frame"); + let payload: Value = serde_json::from_slice(payload).expect("event data should be json"); + let data = &payload["payload"]; + let fingerprint: Value = + serde_json::from_slice(&event.fingerprint).expect("fingerprint should be json"); + + assert_eq!(payload["type"], "runActivity"); + assert_eq!(data["currentLanesComplete"], true); + assert_eq!(data["currentLaneScope"], "complete"); + assert_eq!(data["currentLanes"].as_array().map(Vec::len), Some(0)); + assert_eq!(fingerprint["currentLanes"].as_array().map(Vec::len), Some(0)); + assert!(!data.to_string().contains("xy-952-attempt-2-1781598614")); } #[test] @@ -1189,7 +1265,7 @@ fn operator_dashboard_run_activity_fingerprint_ignores_volatile_timing_fields() "account_selector": null, }, "accounts": [], - "activeRuns": [ + "currentLanes": [ { "run_id": "run-1", "status": "running", @@ -1209,8 +1285,8 @@ fn operator_dashboard_run_activity_fingerprint_ignores_volatile_timing_fields() }, }, ], - "activeRunsComplete": true, - "activeRunScope": "complete", + "currentLanesComplete": true, + "currentLaneScope": "complete", }); let mut second = serde_json::json!({ "accountControl": { @@ -1218,7 +1294,7 @@ fn operator_dashboard_run_activity_fingerprint_ignores_volatile_timing_fields() "account_selector": null, }, "accounts": [], - "activeRuns": [ + "currentLanes": [ { "run_id": "run-1", "status": "running", @@ -1238,24 +1314,24 @@ fn operator_dashboard_run_activity_fingerprint_ignores_volatile_timing_fields() }, }, ], - "activeRunsComplete": true, - "activeRunScope": "complete", + "currentLanesComplete": true, + "currentLaneScope": "complete", }); orchestrator::strip_dashboard_run_activity_volatile_fields(&mut first); orchestrator::strip_dashboard_run_activity_volatile_fields(&mut second); assert_eq!(first, second); - assert_eq!(first["activeRuns"][0]["run_id"], "run-1"); - assert_eq!(first["activeRuns"][0]["child_agent_activity"]["buckets"][0]["event_count"], 7); - assert!(first["activeRuns"][0].get("idle_for_seconds").is_none()); + assert_eq!(first["currentLanes"][0]["run_id"], "run-1"); + assert_eq!(first["currentLanes"][0]["child_agent_activity"]["buckets"][0]["event_count"], 7); + assert!(first["currentLanes"][0].get("idle_for_seconds").is_none()); assert!( - first["activeRuns"][0]["child_agent_activity"] + first["currentLanes"][0]["child_agent_activity"] .get("current_elapsed_seconds") .is_none() ); assert!( - first["activeRuns"][0]["child_agent_activity"]["buckets"][0] + first["currentLanes"][0]["child_agent_activity"]["buckets"][0] .get("wall_seconds") .is_none() ); @@ -1309,7 +1385,7 @@ fn dashboard_event_hub_caches_and_filters_last_run_activity_event() { "account_selector": null, }, "accounts": [], - "activeRuns": [ + "currentLanes": [ { "project_id": "decodex", "issue_id": "issue-1", @@ -1321,8 +1397,8 @@ fn dashboard_event_hub_caches_and_filters_last_run_activity_event() { "run_id": "run-2", }, ], - "activeRunsComplete": true, - "activeRunScope": "complete", + "currentLanesComplete": true, + "currentLaneScope": "complete", }); let subscription = DashboardClientSubscription { project_id: Some(String::from("decodex")), @@ -1336,19 +1412,19 @@ fn dashboard_event_hub_caches_and_filters_last_run_activity_event() { let event = hub .cached_run_activity_event(&subscription) .expect("cached run activity should remain available after other event types"); - let active_runs = event.payload["activeRuns"] + let current_lanes = event.payload["currentLanes"] .as_array() - .expect("filtered active runs should be an array"); + .expect("filtered current lanes should be an array"); assert_eq!(event.event_type, "runActivity"); - assert_eq!(active_runs.len(), 1); - assert_eq!(active_runs[0]["issue_id"], "issue-1"); - assert_eq!(event.payload["activeRunsComplete"], false); - assert_eq!(event.payload["activeRunScope"], "filtered"); + assert_eq!(current_lanes.len(), 1); + assert_eq!(current_lanes[0]["issue_id"], "issue-1"); + assert_eq!(event.payload["currentLanesComplete"], false); + assert_eq!(event.payload["currentLaneScope"], "filtered"); } #[test] -fn operator_dashboard_run_activity_event_includes_disabled_project_active_runs() { +fn operator_dashboard_run_activity_event_includes_disabled_project_current_lanes() { let temp_dir = TempDir::new().expect("temp dir should exist"); let _home_guard = TestEnvVarGuard::set("HOME", temp_dir.path().to_str().expect("temp path should be UTF-8")); @@ -1374,10 +1450,10 @@ fn operator_dashboard_run_activity_event_includes_disabled_project_active_runs() observer_store.upsert_project(®istration).expect("project should register"); writer_store .record_run_attempt("run-disabled-active", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); writer_store .upsert_lease(config.service_id(), &issue.id, "run-disabled-active", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); writer_store .upsert_worktree( config.service_id(), @@ -1397,15 +1473,15 @@ fn operator_dashboard_run_activity_event_includes_disabled_project_active_runs() let (payload, _consumed) = websocket_text_payload(&message).expect("event should be a text frame"); let payload: Value = serde_json::from_slice(payload).expect("event data should be json"); let data = &payload["payload"]; - let active_runs = data["activeRuns"].as_array().expect("active runs should list"); + let current_lanes = data["currentLanes"].as_array().expect("current lanes should list"); assert_eq!(payload["type"], "runActivity"); - assert_eq!(active_runs.len(), 1); - assert_eq!(active_runs[0]["run_id"], "run-disabled-active"); - assert_eq!(active_runs[0]["project_id"], "pubfi"); - assert_eq!(active_runs[0]["project_display_name"], "hack-ink/pubfi-mono-v2"); - assert_eq!(data["activeRunsComplete"], true); - assert_eq!(data["activeRunScope"], "complete"); + assert_eq!(current_lanes.len(), 1); + assert_eq!(current_lanes[0]["run_id"], "run-disabled-active"); + assert_eq!(current_lanes[0]["project_id"], "pubfi"); + assert_eq!(current_lanes[0]["project_display_name"], "hack-ink/pubfi-mono-v2"); + assert_eq!(data["currentLanesComplete"], true); + assert_eq!(data["currentLaneScope"], "complete"); } #[test] @@ -1461,7 +1537,7 @@ fn operator_state_endpoint_overlays_live_account_control_on_published_snapshot() "account_selector": null, }, "accounts": [], - "active_runs": [], + "current_lanes": [], "recent_runs": [], "history_lanes": [], "queued_candidates": [], @@ -1533,7 +1609,7 @@ fn operator_state_endpoint_serves_large_app_snapshot_without_truncation() { "account_selector": null, }, "accounts": [], - "active_runs": [], + "current_lanes": [], "recent_runs": [], "history_lanes": [{ "issue_identifier": "PUB-100", @@ -1785,8 +1861,8 @@ fn operator_lane_inspect_api_returns_lane_identity() { assert_eq!(data["matchedRunCount"], 1); assert_eq!(data["runs"][0]["runId"], "pub-101-attempt-1"); assert_eq!(data["runs"][0]["attemptStatus"], "running"); - assert_eq!(data["runs"][0]["activeLease"], true); - assert_eq!(data["runs"][0]["ownershipState"], "owned_active"); + assert_eq!(data["runs"][0]["runLease"], true); + assert_eq!(data["runs"][0]["ownershipState"], "leased_run"); assert_eq!(data["runs"][0]["livenessState"], "unknown"); assert_eq!(data["runs"][0]["policyState"], "review_pending"); assert_eq!(data["runs"][0]["terminalizationState"], "none"); @@ -2044,10 +2120,10 @@ fn operator_lane_interrupt_api_force_does_not_hard_fallback_after_control_reject } #[cfg(unix)] -fn active_lease_missing_control_fixture( +fn run_lease_missing_control_fixture( config: &ServiceConfig, state_store: &StateStore, -) -> ActiveLeaseMissingControlFixture { +) -> RunLeaseMissingControlFixture { let registration = ProjectRegistration::from_config( config.service_id(), &service_config_path(config.repo_root()), @@ -2112,7 +2188,7 @@ fn active_lease_missing_control_fixture( .expect("control channel should publish"); state_store.clear_lease(&issue.id).expect("lease should clear"); - ActiveLeaseMissingControlFixture { issue, channel_path, child, child_process_id } + RunLeaseMissingControlFixture { issue, channel_path, child, child_process_id } } #[cfg(unix)] @@ -2126,7 +2202,7 @@ fn operator_json_response_body(response: &str, context: &str) -> Value { } #[cfg(unix)] -fn reap_active_lease_missing_child(fixture: &mut ActiveLeaseMissingControlFixture) { +fn reap_run_lease_missing_child(fixture: &mut RunLeaseMissingControlFixture) { if orchestrator::process_is_alive(fixture.child_process_id) { fixture .child @@ -2138,10 +2214,10 @@ fn reap_active_lease_missing_child(fixture: &mut ActiveLeaseMissingControlFixtur } #[cfg(unix)] -fn assert_active_lease_missing_control_audit( +fn assert_run_lease_missing_control_audit( config: &ServiceConfig, state_store: &StateStore, - fixture: &ActiveLeaseMissingControlFixture, + fixture: &RunLeaseMissingControlFixture, run_id: &str, ) { let events = state_store @@ -2152,17 +2228,17 @@ fn assert_active_lease_missing_control_audit( .find(|event| { event.event_type() == "control_action" && event.payload()["action"] == "steer" - && event.payload()["reason"] == "active_lease_missing" + && event.payload()["reason"] == "run_lease_missing" }) - .expect("active lease steer rejection should be audited"); + .expect("run lease steer rejection should be audited"); let missing_lease_interrupt_event = events .iter() .find(|event| { event.event_type() == "control_action" && event.payload()["action"] == "interrupt" - && event.payload()["reason"] == "active_lease_missing" + && event.payload()["reason"] == "run_lease_missing" }) - .expect("active lease interrupt rejection should be audited"); + .expect("run lease interrupt rejection should be audited"); let expected_channel_path = fixture.channel_path.display().to_string(); assert_eq!( @@ -2174,7 +2250,7 @@ fn assert_active_lease_missing_control_audit( Some(expected_channel_path.as_str()) ); assert_eq!( - missing_lease_interrupt_event.payload()["lane"]["active_lease"].as_bool(), + missing_lease_interrupt_event.payload()["lane"]["run_lease"].as_bool(), Some(false) ); assert_eq!( @@ -2199,7 +2275,7 @@ fn assert_active_lease_missing_control_audit( ); assert_eq!( missing_lease_interrupt_event.payload()["context"]["lane_control_conditions"][0].as_str(), - Some("active_lease_missing") + Some("run_lease_missing") ); assert_eq!( missing_lease_interrupt_event.payload()["context"]["control_capability"]["channel_path"] @@ -2219,10 +2295,10 @@ fn assert_active_lease_missing_control_audit( #[cfg(unix)] #[test] -fn operator_lane_interrupt_api_force_hard_fallbacks_after_active_lease_missing() { +fn operator_lane_interrupt_api_force_hard_fallbacks_after_run_lease_missing() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); - let mut fixture = active_lease_missing_control_fixture(&config, &state_store); + let mut fixture = run_lease_missing_control_fixture(&config, &state_store); let project_id = config.service_id().to_owned(); let issue_identifier = fixture.issue.identifier.clone(); let run_id = "pub-101-attempt-1"; @@ -2257,19 +2333,19 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_active_lease_missing() .expect("lane interrupt response should be utf-8"); let data = operator_json_response_body(&response, "lane interrupt"); - reap_active_lease_missing_child(&mut fixture); + reap_run_lease_missing_child(&mut fixture); assert!( steer_response.starts_with("HTTP/1.1 409 Conflict\r\n"), "{steer_response}" ); assert_eq!(steer_data["outcome"], "rejected"); - assert_eq!(steer_data["reason"], "active_lease_missing"); + assert_eq!(steer_data["reason"], "run_lease_missing"); assert_eq!(steer_data["failureClass"], "run_control_action_failed"); assert!(response.starts_with("HTTP/1.1 200 OK\r\n")); assert_eq!(data["classification"], "hard_interrupt_fallback"); assert_eq!(data["softInterrupt"]["status"], "rejected"); - assert_eq!(data["softInterrupt"]["errorClass"], "active_lease_missing"); + assert_eq!(data["softInterrupt"]["errorClass"], "run_lease_missing"); assert_eq!(data["hardInterrupt"]["classification"], "hard_interrupt_fallback"); assert_eq!(data["hardInterrupt"]["status"], "sent"); assert_eq!( @@ -2278,7 +2354,7 @@ fn operator_lane_interrupt_api_force_hard_fallbacks_after_active_lease_missing() ); assert_eq!(data["hardInterrupt"]["processAliveAfter"], false); - assert_active_lease_missing_control_audit(&config, &state_store, &fixture, run_id); + assert_run_lease_missing_control_audit(&config, &state_store, &fixture, run_id); } #[cfg(unix)] diff --git a/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs b/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs index c8a825441..dd3f297f8 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/publishing.rs @@ -406,7 +406,7 @@ fn operator_state_snapshot_publish_skips_terminal_run_metadata_refresh() { } #[test] -fn operator_state_snapshot_publish_still_refreshes_active_run_metadata() { +fn operator_state_snapshot_publish_still_refreshes_current_lane_metadata() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue_with_sort_fields( @@ -425,7 +425,7 @@ fn operator_state_snapshot_publish_still_refreshes_active_run_metadata() { .expect("running attempt should record"); state_store .upsert_lease(TEST_SERVICE_ID, &issue.id, "xy-355-attempt-1", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store .upsert_worktree( TEST_SERVICE_ID, @@ -444,19 +444,19 @@ fn operator_state_snapshot_publish_still_refreshes_active_run_metadata() { &[], &[], ) - .expect("active publish should build"); + .expect("current-lane publish should build"); let refresh_queries = tracker.refresh_queries.borrow(); assert!( refresh_queries .iter() .any(|query| query.len() == 1 && query.first() == Some(&issue.id)), - "active publish should still refresh the active run issue metadata" + "current-lane publish should still refresh the current lane issue metadata" ); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].issue_identifier.as_deref(), Some("XY-355")); - assert_eq!(snapshot.active_runs[0].title.as_deref(), Some("Implement orchestration")); - assert_eq!(snapshot.active_runs[0].author.as_deref(), Some("Yvette")); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].issue_identifier.as_deref(), Some("XY-355")); + assert_eq!(snapshot.current_lanes[0].title.as_deref(), Some("Implement orchestration")); + assert_eq!(snapshot.current_lanes[0].author.as_deref(), Some("Yvette")); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs index 06351d501..f9402f625 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs @@ -301,10 +301,10 @@ fn live_operator_status_snapshot_excludes_claimed_candidates_from_waiting_intake state_store .record_run_attempt("run-claimed", &claimed_issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(config.service_id(), &claimed_issue.id, "run-claimed", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); let snapshot = orchestrator::build_live_operator_status_snapshot( &tracker, @@ -319,9 +319,9 @@ fn live_operator_status_snapshot_excludes_claimed_candidates_from_waiting_intake snapshot.queued_candidates.first().expect("claimed queue echo should remain raw-visible"); let rendered = orchestrator::render_operator_status(&snapshot); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].run_id, "run-claimed"); - assert_eq!(project.active_run_count, 1); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].run_id, "run-claimed"); + assert_eq!(project.current_lane_count, 1); assert_eq!(candidate.issue_identifier, "PUB-103"); assert_eq!(candidate.classification, "claimed"); assert_eq!(candidate.reason, "shared_claim_present"); @@ -334,7 +334,7 @@ fn live_operator_status_snapshot_excludes_claimed_candidates_from_waiting_intake "claimed queue echoes must not inflate project waiting counts" ); assert!(rendered.contains("Backlog: 0")); - assert!(rendered.contains("Active queue echoes: 1")); + assert!(rendered.contains("Claimed queue echoes: 1")); } #[test] @@ -353,10 +353,10 @@ fn live_operator_status_snapshot_prioritizes_needs_attention_over_shared_claim() state_store .record_run_attempt("run-attention-claimed", &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(config.service_id(), &issue.id, "run-attention-claimed", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); let snapshot = orchestrator::build_live_operator_status_snapshot( &tracker, @@ -613,7 +613,7 @@ fn live_operator_status_snapshot_reports_capacity_waiting_separately_from_blocke state_store .upsert_lease(config.service_id(), "issue-running", "run-active", "In Progress") - .expect("active lease should consume the single global slot"); + .expect("run lease should consume the single global slot"); let snapshot = orchestrator::build_live_operator_status_snapshot( &tracker, @@ -1178,7 +1178,7 @@ fn live_operator_status_snapshot_surfaces_git_credential_failures() { .expect("needs-attention queued issue should exist"); let attention = candidate.attention.as_ref().expect("attention details should render"); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(candidate.classification, "blocked"); assert_eq!(candidate.reason, "issue_needs_attention"); assert_eq!(attention.current_operation.as_deref(), Some(state::RUN_OPERATION_GIT_CREDENTIALS)); @@ -1254,7 +1254,7 @@ fn live_operator_status_snapshot_recovers_shared_claims_for_fresh_status_store_i "shared_claim_present" ); assert!( - snapshot.active_runs.is_empty(), + snapshot.current_lanes.is_empty(), "fresh observer stores should not invent local running lanes while reconstructing the shared claim view" ); } @@ -1303,13 +1303,13 @@ fn live_operator_status_snapshot_reconstructs_same_shared_view_for_fresh_state_s .expect("snapshot should build"); serde_json::json!({ - "active_runs": snapshot.active_runs.iter().map(|run| { + "current_lanes": snapshot.current_lanes.iter().map(|run| { serde_json::json!({ "run_id": run.run_id, "issue_id": run.issue_id, "phase": run.phase, "current_operation": run.current_operation, - "active_lease": run.active_lease, + "run_lease": run.run_lease, "branch_name": run.branch_name, "worktree_path": run.worktree_path, }) diff --git a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs index 157d5f73d..69d0b7878 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs @@ -14,7 +14,7 @@ fn failure_comments_use_repo_relative_worktree_paths() { } #[test] -fn operator_status_snapshot_includes_active_runs_and_repo_relative_paths() { +fn operator_status_snapshot_includes_current_lanes_and_repo_relative_paths() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue("Todo", &[]); @@ -48,35 +48,108 @@ fn operator_status_snapshot_includes_active_runs_and_repo_relative_paths() { .expect("snapshot should build"); assert_eq!(snapshot.project_id, "pubfi"); - assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.current_lanes.len(), 1); assert_eq!(snapshot.recent_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].project_id, "pubfi"); - assert_eq!(snapshot.active_runs[0].project_display_name, "hack-ink/pubfi-mono-v2"); - assert_eq!(snapshot.active_runs[0].run_id, "run-1"); - assert_eq!(snapshot.active_runs[0].phase, "executing"); - assert_eq!(snapshot.active_runs[0].current_operation, state::RUN_OPERATION_AGENT_RUN); - assert_eq!(snapshot.active_runs[0].thread_id.as_deref(), Some("thread-1")); - assert_eq!(snapshot.active_runs[0].branch_name.as_deref(), Some("x/pubfi-pub-101")); - assert_eq!(snapshot.active_runs[0].worktree_path.as_deref(), Some(".worktrees/PUB-101")); - assert!(snapshot.active_runs[0].last_run_activity_at.is_some()); - assert!(snapshot.active_runs[0].last_progress_at.is_some()); - assert!(!snapshot.active_runs[0].suspected_stall); - assert_eq!(snapshot.active_runs[0].last_event_type.as_deref(), Some("turn/completed")); + assert_eq!(snapshot.current_lanes[0].project_id, "pubfi"); + assert_eq!(snapshot.current_lanes[0].project_display_name, "hack-ink/pubfi-mono-v2"); + assert_eq!(snapshot.current_lanes[0].run_id, "run-1"); + assert_eq!(snapshot.current_lanes[0].phase, "executing"); + assert_eq!(snapshot.current_lanes[0].current_operation, state::RUN_OPERATION_AGENT_RUN); + assert_eq!(snapshot.current_lanes[0].thread_id.as_deref(), Some("thread-1")); + assert_eq!(snapshot.current_lanes[0].branch_name.as_deref(), Some("x/pubfi-pub-101")); + assert_eq!(snapshot.current_lanes[0].worktree_path.as_deref(), Some(".worktrees/PUB-101")); + assert!(snapshot.current_lanes[0].last_run_activity_at.is_some()); + assert!(snapshot.current_lanes[0].last_progress_at.is_some()); + assert!(!snapshot.current_lanes[0].suspected_stall); + assert_eq!(snapshot.current_lanes[0].last_event_type.as_deref(), Some("turn/completed")); assert_eq!(snapshot.worktrees[0].worktree_path, ".worktrees/PUB-101"); - assert_eq!(snapshot.worktrees[0].ownership, "active_lane"); - assert!(snapshot.worktrees[0].ownership_reason.contains("Active lane `run-1`")); + assert_eq!(snapshot.worktrees[0].ownership, "current_lane"); + assert!(snapshot.worktrees[0].ownership_reason.contains("Current lane `run-1`")); let project = snapshot.projects.first().expect("project summary should exist"); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!( project.retained_worktree_count, 0, - "active running lane worktrees must not inflate project recovery counts" + "current lane worktrees must not inflate project recovery counts" ); assert_eq!(project.connector_state, "ok"); assert!(project.last_activity_at.is_some()); } +#[test] +fn operator_status_snapshot_surfaces_repeated_continuation_recovery_lineage() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); + let worktree_path = config.worktree_root().join("PUB-101"); + + for (run_id, attempt_number) in [("run-1", 1), ("run-2", 2)] { + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + run_id, + attempt_number, + PHASE_GOAL_RECOVERY_EVENT_TYPE, + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "signal": "phase_goal_recovered", + "payload": { + "nextPhase": "repair_validation_failures", + "sourceErrorClass": "app_server_preflight_timeout", + "sourceErrorMessage": "Timed out while waiting for app-server output.", + }, + }), + ) + .expect("phase goal recovery event should record"); + } + + state_store + .record_run_attempt("run-3", &issue.id, 3, "running") + .expect("current run attempt should record"); + state_store + .upsert_lease(TEST_SERVICE_ID, &issue.id, "run-3", "In Progress") + .expect("lease should record"); + state_store + .upsert_worktree( + TEST_SERVICE_ID, + &issue.id, + "x/pubfi-pub-101", + &worktree_path.display().to_string(), + ) + .expect("worktree should record"); + + let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) + .expect("snapshot should build"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); + let recovery = run + .continuation_recovery + .as_ref() + .expect("continuation recovery lineage should project onto current lane"); + let rendered = orchestrator::render_operator_status(&snapshot); + let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); + + assert_eq!(recovery.state, "continuation_scheduled"); + assert_eq!(recovery.source_phase, "implement_to_validation_ready"); + assert_eq!(recovery.next_phase, "repair_validation_failures"); + assert_eq!(recovery.source_error_class, "app_server_preflight_timeout"); + assert_eq!(recovery.recovery_count, 2); + assert_eq!(recovery.automatic_continuation_limit, 1); + assert!(recovery.budget_exceeded); + assert_eq!(run.policy_state, "continuation_recovery_churn_exceeded"); + assert!(run.lane_control_conditions.contains(&String::from( + "continuation_recovery_budget_exceeded" + ))); + assert!(rendered.contains("continuation_recovery: state=continuation_scheduled")); + assert!(rendered.contains("count=2/1 budget_exceeded=yes")); + assert_eq!( + snapshot_json["current_lanes"][0]["continuation_recovery"]["budget_exceeded"], + true + ); +} + #[test] fn operator_status_snapshot_surfaces_merged_dirty_ad_hoc_worktree() { let (_temp_dir, config, _workflow) = temp_project_layout(); @@ -235,7 +308,7 @@ fn operator_status_snapshot_updates_owned_merged_worktree_hygiene_without_global } #[test] -fn live_operator_status_snapshot_hydrates_active_run_issue_display_metadata() { +fn live_operator_status_snapshot_hydrates_current_lane_issue_display_metadata() { let (temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let run_id = "xy-392-attempt-1-1777551056"; @@ -260,10 +333,10 @@ fn live_operator_status_snapshot_hydrates_active_run_issue_display_metadata() { state_store .record_run_attempt(run_id, &issue.id, 1, "running") - .expect("active run should record"); + .expect("current lane should record"); state_store .upsert_lease(config.service_id(), &issue.id, run_id, "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store.update_run_thread(run_id, "thread-1").expect("thread should record"); state_store.update_run_turn(run_id, "turn-1").expect("turn should record"); @@ -281,44 +354,44 @@ fn live_operator_status_snapshot_hydrates_active_run_issue_display_metadata() { 10, ) .expect("snapshot should build"); - let active_run = snapshot.active_runs.first().expect("active run should exist"); + let current_lane = snapshot.current_lanes.first().expect("current lane should exist"); let recent_run = snapshot.recent_runs.first().expect("recent run should exist"); let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); - assert_eq!(active_run.project_id, config.service_id()); - assert_eq!(active_run.project_display_name, "hack-ink/pubfi-mono-v2"); - assert_eq!(active_run.issue_identifier.as_deref(), Some("XY-392")); - assert_eq!(active_run.title.as_deref(), Some("Hydrate issue display metadata on run rows")); - assert_eq!(active_run.author.as_deref(), Some("Yvette")); + assert_eq!(current_lane.project_id, config.service_id()); + assert_eq!(current_lane.project_display_name, "hack-ink/pubfi-mono-v2"); + assert_eq!(current_lane.issue_identifier.as_deref(), Some("XY-392")); + assert_eq!(current_lane.title.as_deref(), Some("Hydrate issue display metadata on run rows")); + assert_eq!(current_lane.author.as_deref(), Some("Yvette")); assert_eq!( - active_run.private_evidence.read_command, + current_lane.private_evidence.read_command, format!("decodex evidence XY-392 --run-id {run_id} --attempt 1 --json") ); assert_eq!(recent_run.issue_identifier.as_deref(), Some("XY-392")); assert_eq!(recent_run.title.as_deref(), Some("Hydrate issue display metadata on run rows")); assert_eq!(recent_run.author.as_deref(), Some("Yvette")); - assert_eq!(snapshot_json["active_runs"][0]["project_id"], "pubfi"); + assert_eq!(snapshot_json["current_lanes"][0]["project_id"], "pubfi"); assert_eq!( - snapshot_json["active_runs"][0]["project_display_name"], + snapshot_json["current_lanes"][0]["project_display_name"], "hack-ink/pubfi-mono-v2" ); - assert_eq!(snapshot_json["active_runs"][0]["issue_identifier"], "XY-392"); + assert_eq!(snapshot_json["current_lanes"][0]["issue_identifier"], "XY-392"); assert_eq!( - snapshot_json["active_runs"][0]["title"], + snapshot_json["current_lanes"][0]["title"], "Hydrate issue display metadata on run rows" ); - assert_eq!(snapshot_json["active_runs"][0]["author"], "Yvette"); + assert_eq!(snapshot_json["current_lanes"][0]["author"], "Yvette"); assert_eq!( - snapshot_json["active_runs"][0]["private_evidence"]["read_command"], + snapshot_json["current_lanes"][0]["private_evidence"]["read_command"], format!("decodex evidence XY-392 --run-id {run_id} --attempt 1 --json") ); - assert_eq!(snapshot_json["active_runs"][0]["control_capability"]["status"], "active"); + assert_eq!(snapshot_json["current_lanes"][0]["control_capability"]["status"], "active"); assert_eq!( - snapshot_json["active_runs"][0]["control_capability"]["thread_id"], + snapshot_json["current_lanes"][0]["control_capability"]["thread_id"], "thread-1" ); assert_eq!( - snapshot_json["active_runs"][0]["control_capability"]["turn_id"], + snapshot_json["current_lanes"][0]["control_capability"]["turn_id"], "turn-1" ); } @@ -342,7 +415,7 @@ fn idle_operator_status_snapshot_has_no_runtime_or_recovery_noise() { assert_eq!(snapshot.project_id, "pubfi"); assert_eq!(snapshot.run_limit, 10); assert!(snapshot.warnings.is_empty(), "idle snapshot warnings: {:?}", snapshot.warnings); - assert!(snapshot.active_runs.is_empty(), "idle snapshot should have no active runs"); + assert!(snapshot.current_lanes.is_empty(), "idle snapshot should have no current lanes"); assert!(snapshot.recent_runs.is_empty(), "idle snapshot should have no run history"); assert!(snapshot.history_lanes.is_empty(), "idle snapshot should have no run ledger lanes"); assert!( @@ -368,7 +441,7 @@ fn idle_operator_status_snapshot_has_no_runtime_or_recovery_noise() { for field in [ "warnings", "warning_details", - "active_runs", + "current_lanes", "recent_runs", "history_lanes", "queued_candidates", @@ -386,20 +459,20 @@ fn idle_operator_status_snapshot_has_no_runtime_or_recovery_noise() { assert!(rendered.contains("Running lanes: 0")); assert!(rendered.contains("Run ledger shown: 0 issue lanes from 0 history attempts")); assert!(rendered.contains("Backlog: 0")); - assert!(rendered.contains("Active queue echoes: 0")); + assert!(rendered.contains("Claimed queue echoes: 0")); assert!(rendered.contains("Stale closed queue labels: 0")); assert!(rendered.contains("Recovery worktrees: 0")); assert!(rendered.contains("Post-review lanes: 0")); - assert!(rendered.contains("\nRunning Lanes\n- none\n")); + assert!(rendered.contains("\nCurrent Lanes\n- none\n")); assert!(rendered.contains("\nRun Ledger\n- none\n")); assert!(rendered.contains("\nBacklog\n- none\n")); - assert!(rendered.contains("\nActive Queue Echoes\n- none\n")); + assert!(rendered.contains("\nClaimed Queue Echoes\n- none\n")); assert!(rendered.contains("\nStale Closed Queue Labels\n- none\n")); assert!(rendered.contains("\nRecovery Worktrees\n- none\n")); assert!(rendered.contains("\nPost-Review Lanes\n- none\n")); assert!(!rendered.contains("Warning details:")); assert!(!rendered.contains("run_id:")); - assert!(!rendered.contains("active_lease: true")); + assert!(!rendered.contains("run_lease: true")); assert!(!rendered.contains("role: post_review_lane")); assert!(!rendered.contains("role: cleanup_only")); } @@ -452,7 +525,7 @@ fn idle_operator_status_snapshot_includes_configured_codex_accounts() { let accounts = snapshot_json["accounts"].as_array().expect("snapshot should expose configured accounts"); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(snapshot_json["account_control"]["mode"], "balanced"); assert_eq!( snapshot_json["account_control"]["account_selector"], @@ -755,7 +828,7 @@ fn runtime_recovery_preserves_legacy_cleanup_only_provenance_without_recoverable .expect("legacy mapping should remain"); assert!( - recovered_state.active_issues.is_empty(), + recovered_state.recoverable_issues.is_empty(), "terminal cleanup-only worktree should not become a retry lane" ); assert_eq!(mapping.provenance().source(), "legacy_unknown"); @@ -798,8 +871,8 @@ fn runtime_recovery_records_recovered_provenance_for_fresh_active_worktree() { .expect("fresh active marker should recover the lease"); assert!( - recovered_state.active_issues.is_empty(), - "fresh marker should recover as the active lease instead of a retry queue item" + recovered_state.recoverable_issues.is_empty(), + "fresh marker should recover as the run lease instead of a retry queue item" ); assert_eq!(mapping.provenance().source(), "runtime_recovered"); assert_eq!(mapping.provenance().created_at_unix(), Some(observed_at_unix)); @@ -887,7 +960,7 @@ fn operator_status_snapshot_ignores_retry_schedule_on_running_attempt() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should exist"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); assert_eq!(run.phase, "executing"); assert_eq!(run.wait_reason, None); @@ -1051,7 +1124,7 @@ fn operator_status_snapshot_marks_soft_stalls_before_hard_timeout() { let marker_path = worktree_path.join(RUN_ACTIVITY_MARKER_FILE); let marker_body = fs::read_to_string(&marker_path).expect("marker body should load"); - let suspected_age = (ACTIVE_RUN_IDLE_TIMEOUT.as_secs() / 2).saturating_add(1) as i64; + let suspected_age = (RUN_LEASE_IDLE_TIMEOUT.as_secs() / 2).saturating_add(1) as i64; let stale_progress = OffsetDateTime::now_utc().unix_timestamp() - suspected_age; let rewritten = marker_body .lines() @@ -1070,7 +1143,7 @@ fn operator_status_snapshot_marks_soft_stalls_before_hard_timeout() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should exist"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); assert_eq!(run.current_operation, state::RUN_OPERATION_AGENT_RUN); assert!(run.last_progress_at.is_some()); @@ -1166,7 +1239,7 @@ fn operator_status_snapshot_diagnoses_protocol_only_model_execution() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should exist"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); let rendered = orchestrator::render_operator_status(&snapshot); assert_eq!(run.wait_reason.as_deref(), Some("model_execution")); @@ -1205,13 +1278,13 @@ fn operator_status_snapshot_counts_stopped_active_process_as_attention_not_runni let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); assert_eq!(run.phase, "executing"); assert_eq!(run.process_alive, Some(false)); assert_eq!(run.process_liveness_reason.as_deref(), Some("process_stopped")); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); } @@ -1272,13 +1345,13 @@ fn operator_status_snapshot_counts_zombie_active_process_as_attention_not_runnin assert!(observed_stopped, "exited child process must not count as alive"); let snapshot = snapshot.expect("snapshot should be captured while child is unreaped"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); assert_eq!(run.phase, "executing"); assert_eq!(run.process_alive, Some(false)); assert_eq!(run.process_liveness_reason.as_deref(), Some("process_stopped")); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); } @@ -1314,7 +1387,7 @@ fn operator_status_snapshot_counts_previous_boot_process_as_attention_not_runnin let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); assert_eq!(run.phase, "executing"); @@ -1325,14 +1398,14 @@ fn operator_status_snapshot_counts_previous_boot_process_as_attention_not_runnin assert!(!run.has_fresh_execution); assert!(!run.counts_as_running); assert!(run.needs_attention); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); } #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] -fn operator_status_snapshot_projects_unleased_app_server_active_run_as_retained_attention() { +fn operator_status_snapshot_projects_unleased_app_server_current_lane_as_retained_attention() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue("Todo", &[]); @@ -1365,14 +1438,14 @@ fn operator_status_snapshot_projects_unleased_app_server_active_run_as_retained_ let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); - assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.current_lanes.len(), 1); assert_eq!(run.run_id, "run-1"); assert_eq!(run.status, "running"); assert_eq!(run.phase, "executing"); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "process_identity_mismatch"); assert_eq!(run.process_alive, Some(false)); @@ -1388,7 +1461,7 @@ fn operator_status_snapshot_projects_unleased_app_server_active_run_as_retained_ assert!(run.has_fresh_execution); assert!(!run.counts_as_running); assert!(run.needs_attention); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); assert_eq!(snapshot.worktrees[0].ownership, "retained_attention"); @@ -1446,29 +1519,29 @@ fn operator_status_snapshot_does_not_shadow_post_review_lane_with_retained_atten mergeable: Some(String::from("UNKNOWN")), check_state: Some(String::from("SUCCESS")), unresolved_review_threads: Some(1), - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: None, readback_root_cause: Some(String::from("lineage_validation_failed")), loop_status: None, }]; - orchestrator::hydrate_post_review_lane_active_run_shadowing(&mut snapshot); + orchestrator::hydrate_post_review_lane_current_lane_shadowing(&mut snapshot); orchestrator::refresh_operator_project_summary(&mut snapshot, None); let project = snapshot.projects.first().expect("project summary should exist"); let lane = snapshot.post_review_lanes.first().expect("post-review lane should remain visible"); let rendered = orchestrator::render_operator_status(&snapshot); - assert!(snapshot.active_runs[0].has_fresh_execution); - assert!(!snapshot.active_runs[0].counts_as_running); - assert!(snapshot.active_runs[0].needs_attention); - assert_eq!(snapshot.active_runs[0].ownership_state, "retained_attention"); - assert!(!lane.shadowed_by_active_run); + assert!(snapshot.current_lanes[0].has_fresh_execution); + assert!(!snapshot.current_lanes[0].counts_as_running); + assert!(snapshot.current_lanes[0].needs_attention); + assert_eq!(snapshot.current_lanes[0].ownership_state, "retained_attention"); + assert!(!lane.shadowed_by_current_lane); assert_eq!(project.running_lane_count, 0); assert_eq!(project.post_review_lane_count, 1); assert_eq!(project.waiting_lane_count, 0); assert_eq!(project.attention_count, 1); - assert!(rendered.contains("shadowed_by_active_run: no")); + assert!(rendered.contains("shadowed_by_current_lane: no")); assert!(rendered.contains("readback_root_cause: lineage_validation_failed")); } @@ -1503,7 +1576,7 @@ fn operator_status_snapshot_counts_reused_pid_as_attention_not_running() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); assert_eq!(run.phase, "executing"); @@ -1514,7 +1587,7 @@ fn operator_status_snapshot_counts_reused_pid_as_attention_not_running() { run.process_liveness_reason.as_deref(), Some("process_start_identity_mismatch") ); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); } @@ -1544,14 +1617,14 @@ fn operator_status_snapshot_keeps_unleased_live_process_visible_but_not_running( let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); - assert_eq!(snapshot.active_runs.len(), 1); + assert_eq!(snapshot.current_lanes.len(), 1); assert_eq!(run.run_id, "run-1"); assert_eq!(run.status, "running"); assert_eq!(run.attempt_status, "running"); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "process_alive"); assert_eq!(run.process_alive, Some(true)); @@ -1565,11 +1638,11 @@ fn operator_status_snapshot_keeps_unleased_live_process_visible_but_not_running( assert!(run .lane_control_conditions .iter() - .any(|condition| condition == "active_lease_missing")); + .any(|condition| condition == "run_lease_missing")); assert!(run.has_fresh_execution); assert!(!run.counts_as_running); assert!(!run.needs_attention); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.retained_worktree_count, 1); assert_eq!(snapshot.worktrees[0].ownership, "orphaned_live_thread"); @@ -1607,13 +1680,13 @@ fn operator_status_snapshot_keeps_terminal_status_live_process_in_recent_orphan_ .expect("live terminal run should remain inspectable"); let project = snapshot.projects.first().expect("project summary should exist"); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(run.run_id, "run-1"); assert_eq!(run.status, "failed"); assert_eq!(run.attempt_status, "failed"); assert_eq!(run.status_projection_reason, None); assert_eq!(run.phase, "failed"); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "not_running"); assert_eq!(run.ownership_state, "orphaned_live_thread"); @@ -1625,7 +1698,7 @@ fn operator_status_snapshot_keeps_terminal_status_live_process_in_recent_orphan_ .any(|condition| condition == "terminal_attempt_has_live_evidence")); assert_eq!(run.process_alive, Some(true)); assert_eq!(run.process_liveness_reason.as_deref(), Some("process_alive")); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.running_lane_count, 0); assert_eq!(project.retained_worktree_count, 1); assert_eq!(snapshot.worktrees[0].ownership, "orphaned_live_thread"); @@ -1649,10 +1722,10 @@ fn operator_status_snapshot_excludes_terminal_thread_archive_from_running_lanes( let project = snapshot.projects.first().expect("project summary should exist"); assert!( - snapshot.active_runs.is_empty(), + snapshot.current_lanes.is_empty(), "terminal archive-only protocol events must not present as active execution" ); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.running_lane_count, 0); assert!( snapshot.recent_runs.iter().all(|run| run.run_id != "run-1"), @@ -1700,11 +1773,11 @@ fn operator_status_snapshot_projects_terminal_run_with_active_thread_as_retained .find(|run| run.run_id == "run-1") .expect("terminal run should remain inspectable"); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(run.status, "stalled"); assert_eq!(run.attempt_status, "stalled"); assert_eq!(run.status_projection_reason, None); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "not_running"); assert_eq!(run.ownership_state, "retained_attention"); @@ -1750,13 +1823,13 @@ fn operator_status_snapshot_keeps_succeeded_status_live_process_in_recent_orphan .expect("live succeeded run should remain inspectable"); let project = snapshot.projects.first().expect("project summary should exist"); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(run.run_id, "run-1"); assert_eq!(run.status, "succeeded"); assert_eq!(run.attempt_status, "succeeded"); assert_eq!(run.status_projection_reason, None); assert_eq!(run.phase, "completed"); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "not_running"); assert_eq!(run.ownership_state, "orphaned_live_thread"); @@ -1764,7 +1837,7 @@ fn operator_status_snapshot_keeps_succeeded_status_live_process_in_recent_orphan assert_eq!(run.terminalization_state, "barrier_started"); assert_eq!(run.process_alive, Some(true)); assert_eq!(run.process_liveness_reason.as_deref(), Some("process_alive")); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.running_lane_count, 0); assert_eq!(project.retained_worktree_count, 1); assert_eq!(snapshot.worktrees[0].ownership, "orphaned_live_thread"); @@ -1851,17 +1924,17 @@ fn assert_terminal_pending_status_projection(snapshot: &OperatorStatusSnapshot) .expect("terminal-pending run should remain inspectable in recent runs"); assert!( - snapshot.active_runs.is_empty(), + snapshot.current_lanes.is_empty(), "terminal-finalized runs must not keep presenting as active execution" ); - assert_eq!(project.active_run_count, 0); + assert_eq!(project.current_lane_count, 0); assert_eq!(project.running_lane_count, 0); assert_eq!(run.status, "review_handoff_pending"); assert_eq!(run.attempt_status, "running"); assert_eq!(run.phase, "terminal_pending"); assert_eq!(run.wait_reason.as_deref(), Some("review_handoff_writeback")); assert_eq!(run.current_operation, state::RUN_OPERATION_REVIEW_WRITEBACK); - assert!(!run.active_lease); + assert!(!run.run_lease); assert_eq!(run.queue_lease_state, "not_held"); assert_eq!(run.execution_liveness, "not_running"); assert!(!run.suspected_stall); @@ -1890,7 +1963,7 @@ fn assert_terminal_pending_lane_inspect(state_store: &StateStore) { assert_eq!(data["runs"][0]["phase"], "terminal_pending"); assert_eq!(data["runs"][0]["waitReason"], "review_handoff_writeback"); assert_eq!(data["runs"][0]["currentOperation"], state::RUN_OPERATION_REVIEW_WRITEBACK); - assert_eq!(data["runs"][0]["activeLease"], false); + assert_eq!(data["runs"][0]["runLease"], false); assert_eq!(data["runs"][0]["executionLiveness"], "not_running"); assert_eq!(data["runs"][0]["softInterruptAvailable"], false); assert_eq!(data["runs"][0]["hardInterruptAvailable"], false); @@ -1991,7 +2064,7 @@ fn operator_status_snapshot_promotes_starting_after_app_server_activity() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let rendered = orchestrator::render_operator_status(&snapshot); assert_eq!(run.status, "running"); @@ -2012,7 +2085,7 @@ fn operator_status_snapshot_counts_stale_starting_run_as_attention_not_running() let issue = sample_issue("Todo", &[]); let worktree_path = config.worktree_root().join("PUB-101"); let stale_activity = - OffsetDateTime::now_utc().unix_timestamp() - ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 - 30; + OffsetDateTime::now_utc().unix_timestamp() - RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 - 30; state_store .record_run_attempt("run-1", &issue.id, 1, "starting") @@ -2040,22 +2113,22 @@ fn operator_status_snapshot_counts_stale_starting_run_as_attention_not_running() let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should remain visible"); + let run = snapshot.current_lanes.first().expect("current lane should remain visible"); let project = snapshot.projects.first().expect("project summary should exist"); assert_eq!(run.status, "starting"); assert_eq!(run.phase, "executing"); assert_eq!(run.process_alive, None); assert!(run.protocol_idle_for_seconds.is_some_and(|idle| { - u64::try_from(idle).is_ok_and(|idle| idle >= ACTIVE_RUN_IDLE_TIMEOUT.as_secs()) + u64::try_from(idle).is_ok_and(|idle| idle >= RUN_LEASE_IDLE_TIMEOUT.as_secs()) })); - assert_eq!(project.active_run_count, 1); + assert_eq!(project.current_lane_count, 1); assert_eq!(project.running_lane_count, 0); assert_eq!(project.attention_count, 1); } #[test] -fn operator_status_snapshot_excludes_completed_lingering_lease_from_active_runs() { +fn operator_status_snapshot_excludes_completed_lingering_lease_from_current_lanes() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let completed_issue = sample_issue_with_sort_fields( @@ -2075,14 +2148,14 @@ fn operator_status_snapshot_excludes_completed_lingering_lease_from_active_runs( "2026-04-29T17:01:33.133Z", ); let completed_run_id = "xy-379-attempt-1-1777482033"; - let active_run_id = "xy-378-attempt-1-1777482000"; + let current_lane_run_id = "xy-378-attempt-1-1777482000"; state_store .record_run_attempt(completed_run_id, &completed_issue.id, 1, "running") .expect("completed run should record"); state_store .upsert_lease("pubfi", &completed_issue.id, completed_run_id, "In Progress") - .expect("stale active lease should remain in runtime db"); + .expect("stale run lease should remain in runtime db"); state_store .append_event(completed_run_id, 1, "turn/completed", "{\"turn\":\"1\"}") .expect("terminal protocol evidence should record"); @@ -2090,11 +2163,11 @@ fn operator_status_snapshot_excludes_completed_lingering_lease_from_active_runs( .update_run_status(completed_run_id, "succeeded") .expect("terminal status should update"); state_store - .record_run_attempt(active_run_id, &active_issue.id, 1, "running") - .expect("active run should record"); + .record_run_attempt(current_lane_run_id, &active_issue.id, 1, "running") + .expect("current lane should record"); state_store - .upsert_lease("pubfi", &active_issue.id, active_run_id, "In Progress") - .expect("active lease should record"); + .upsert_lease("pubfi", &active_issue.id, current_lane_run_id, "In Progress") + .expect("run lease should record"); let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 25) .expect("snapshot should build"); @@ -2105,14 +2178,14 @@ fn operator_status_snapshot_excludes_completed_lingering_lease_from_active_runs( .find(|run| run.run_id == completed_run_id) .expect("completed stale-lease run should remain in history"); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].run_id, active_run_id); - assert_eq!(snapshot.active_runs[0].phase, "executing"); - assert_eq!(project.active_run_count, 1); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].run_id, current_lane_run_id); + assert_eq!(snapshot.current_lanes[0].phase, "executing"); + assert_eq!(project.current_lane_count, 1); assert_eq!(snapshot.recent_runs.len(), 2); assert_eq!(completed_run.phase, "completed"); assert!( - completed_run.active_lease, + completed_run.run_lease, "regression setup should keep the stale lease visible in history" ); } @@ -2171,7 +2244,7 @@ fn operator_status_snapshot_rolls_current_child_bucket_elapsed_time_into_bucket( let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should exist"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); let activity = run.child_agent_activity.as_ref().expect("activity should render"); let protocol_activity = run.protocol_activity.as_ref().expect("protocol fallback should render"); @@ -2248,7 +2321,7 @@ fn operator_status_snapshot_uses_structured_protocol_activity_summary() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); - let run = snapshot.active_runs.first().expect("active run should exist"); + let run = snapshot.current_lanes.first().expect("current lane should exist"); let rendered = orchestrator::render_operator_status(&snapshot); assert_eq!(run.wait_reason.as_deref(), Some("approval_or_user_input")); @@ -2301,7 +2374,7 @@ fn operator_status_snapshot_ignores_marker_from_newer_attempt_for_stored_run() { } #[test] -fn operator_status_snapshot_keeps_all_active_runs_when_recent_runs_are_limited() { +fn operator_status_snapshot_keeps_all_current_lanes_when_recent_runs_are_limited() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let first_issue = sample_issue_with_sort_fields( @@ -2345,8 +2418,8 @@ fn operator_status_snapshot_keeps_all_active_runs_when_recent_runs_are_limited() assert_eq!(snapshot.run_limit, 1); assert_eq!(snapshot.recent_runs.len(), 2); - assert_eq!(snapshot.active_runs.len(), 2); - assert!(snapshot.active_runs.iter().all(|run| run.active_lease)); + assert_eq!(snapshot.current_lanes.len(), 2); + assert!(snapshot.current_lanes.iter().all(|run| run.run_lease)); } #[test] @@ -2365,7 +2438,7 @@ fn operator_status_snapshot_keeps_terminal_run_after_lane_cleanup() { state_store.record_run_attempt("run-done", &issue.id, 1, "running").expect("run should record"); state_store .upsert_lease("pubfi", &issue.id, "run-done", "In Progress") - .expect("active lease should record"); + .expect("run lease should record"); state_store .upsert_worktree( "pubfi", @@ -2375,18 +2448,18 @@ fn operator_status_snapshot_keeps_terminal_run_after_lane_cleanup() { ) .expect("worktree should record"); state_store.update_run_status("run-done", "succeeded").expect("terminal status should update"); - state_store.clear_lease(&issue.id).expect("terminal cleanup should clear active lease"); + state_store.clear_lease(&issue.id).expect("terminal cleanup should clear run lease"); state_store.clear_worktree(&issue.id).expect("terminal cleanup should clear worktree"); let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 25) .expect("snapshot should build"); let rendered = orchestrator::render_operator_status(&snapshot); - assert!(snapshot.active_runs.is_empty()); + assert!(snapshot.current_lanes.is_empty()); assert_eq!(snapshot.recent_runs.len(), 1); assert_eq!(snapshot.recent_runs[0].run_id, "run-done"); assert_eq!(snapshot.recent_runs[0].phase, "completed"); - assert!(!snapshot.recent_runs[0].active_lease); + assert!(!snapshot.recent_runs[0].run_lease); assert_eq!(snapshot.recent_runs[0].branch_name, None); assert_eq!(snapshot.recent_runs[0].worktree_path, None); assert_eq!(snapshot.history_lanes.len(), 1); @@ -2396,7 +2469,7 @@ fn operator_status_snapshot_keeps_terminal_run_after_lane_cleanup() { } #[test] -fn status_hydration_does_not_fabricate_active_leases_for_recovered_candidates() { +fn status_hydration_does_not_fabricate_run_leases_for_recovered_candidates() { let (_temp_dir, config, _workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue("In Progress", &[]); @@ -2414,7 +2487,7 @@ fn status_hydration_does_not_fabricate_active_leases_for_recovered_candidates() orchestrator::hydrate_status_snapshot_state( &config, &state_store, - RecoveredRuntimeState { active_issues: vec![issue.clone()] }, + RecoveredRuntimeState { recoverable_issues: vec![issue.clone()] }, ) .expect("status hydration should succeed"); @@ -2422,8 +2495,8 @@ fn status_hydration_does_not_fabricate_active_leases_for_recovered_candidates() .expect("snapshot should build"); assert!( - snapshot.active_runs.is_empty(), - "recovered retry candidates should not appear as active leased runs" + snapshot.current_lanes.is_empty(), + "recovered retry candidates should not appear as run leased runs" ); assert!( snapshot.recent_runs.is_empty(), @@ -2432,7 +2505,7 @@ fn status_hydration_does_not_fabricate_active_leases_for_recovered_candidates() } #[test] -fn live_operator_status_snapshot_hydrates_active_run_thread_and_event_metadata_from_marker() { +fn live_operator_status_snapshot_hydrates_current_lane_thread_and_event_metadata_from_marker() { let (_temp_dir, config, workflow) = temp_project_layout(); let active_label = tracker::automation_active_label(TEST_SERVICE_ID); let issue = sample_issue("In Progress", &[active_label.as_str()]); @@ -2508,20 +2581,20 @@ fn live_operator_status_snapshot_hydrates_active_run_thread_and_event_metadata_f ) .expect("snapshot should build"); - assert_eq!(snapshot.active_runs.len(), 1); - assert_eq!(snapshot.active_runs[0].thread_id.as_deref(), Some("thread-1")); - assert_eq!(snapshot.active_runs[0].turn_id.as_deref(), Some("turn-1")); - assert_eq!(snapshot.active_runs[0].thread_status.as_deref(), Some("active")); + assert_eq!(snapshot.current_lanes.len(), 1); + assert_eq!(snapshot.current_lanes[0].thread_id.as_deref(), Some("thread-1")); + assert_eq!(snapshot.current_lanes[0].turn_id.as_deref(), Some("turn-1")); + assert_eq!(snapshot.current_lanes[0].thread_status.as_deref(), Some("active")); assert_eq!( - snapshot.active_runs[0].thread_active_flags, + snapshot.current_lanes[0].thread_active_flags, vec![String::from("waitingOnApproval")] ); - assert!(snapshot.active_runs[0].interactive_requested); - assert_eq!(snapshot.active_runs[0].event_count, 2); - assert_eq!(snapshot.active_runs[0].last_event_type.as_deref(), Some("turn/completed")); - assert_eq!(snapshot.active_runs[0].effective_model.as_deref(), Some("gpt-5.4")); - assert_eq!(snapshot.active_runs[0].effective_model_provider.as_deref(), Some("openai")); - assert_eq!(snapshot.active_runs[0].effective_approval_policy.as_deref(), Some("never")); - assert_eq!(snapshot.active_runs[0].effective_sandbox_mode.as_deref(), Some("workspaceWrite")); - assert!(snapshot.active_runs[0].last_event_at.is_some()); + assert!(snapshot.current_lanes[0].interactive_requested); + assert_eq!(snapshot.current_lanes[0].event_count, 2); + assert_eq!(snapshot.current_lanes[0].last_event_type.as_deref(), Some("turn/completed")); + assert_eq!(snapshot.current_lanes[0].effective_model.as_deref(), Some("gpt-5.4")); + assert_eq!(snapshot.current_lanes[0].effective_model_provider.as_deref(), Some("openai")); + assert_eq!(snapshot.current_lanes[0].effective_approval_policy.as_deref(), Some("never")); + assert_eq!(snapshot.current_lanes[0].effective_sandbox_mode.as_deref(), Some("workspaceWrite")); + assert!(snapshot.current_lanes[0].last_event_at.is_some()); } diff --git a/apps/decodex/src/orchestrator/tests/operator/status/text.rs b/apps/decodex/src/orchestrator/tests/operator/status/text.rs index e3a1d8c74..0a8197144 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/text.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/text.rs @@ -29,7 +29,7 @@ fn operator_status_text_surfaces_github_cli_authority() { "No action needed; Decodex will use the configured GitHub CLI path.", ), }, - active_run_count: 0, + current_lane_count: 0, running_lane_count: 0, queued_candidate_count: 0, post_review_lane_count: 0, @@ -47,7 +47,7 @@ fn operator_status_text_surfaces_github_cli_authority() { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), queued_candidates: Vec::new(), recent_runs: Vec::new(), history_lanes: Vec::new(), @@ -64,7 +64,7 @@ fn operator_status_text_surfaces_github_cli_authority() { #[test] fn operator_status_text_renders_human_readable_sections() { - let active_run = operator_status_text_active_run(); + let current_lane = operator_status_text_current_lane(); let snapshot = OperatorStatusSnapshot { project_id: String::from("pubfi"), run_limit: 10, @@ -79,9 +79,9 @@ fn operator_status_text_renders_human_readable_sections() { account_selector: None, }, accounts: Vec::new(), - active_runs: vec![active_run.clone()], + current_lanes: vec![current_lane.clone()], queued_candidates: operator_status_text_queued_candidates(), - recent_runs: vec![active_run], + recent_runs: vec![current_lane], history_lanes: Vec::new(), execution_programs: Vec::new(), worktrees: operator_status_text_worktrees(), @@ -91,12 +91,12 @@ fn operator_status_text_renders_human_readable_sections() { assert!(rendered.contains("Project: pubfi")); assert!(rendered.contains("Warnings: 0")); - assert!(rendered.contains("Running Lanes")); + assert!(rendered.contains("Current Lanes")); assert!(rendered.contains( - "Run ledger shown: 0 issue lanes from 0 history attempts (running lanes inline)" + "Run ledger shown: 0 issue lanes from 0 history attempts (current lanes inline)" )); assert!(rendered.contains("Run Ledger")); - assert!(rendered.contains("- none (running lanes are shown above)")); + assert!(rendered.contains("- none (current lanes are shown above)")); assert!(rendered.contains("run_id: run-1")); assert_eq!(rendered.matches("run_id: run-1").count(), 1); assert!(rendered.contains("attempt_status: running")); @@ -144,12 +144,12 @@ fn operator_status_text_renders_human_readable_sections() { assert!(rendered.contains("last_progress_at: 2026-03-14 10:00:01Z")); assert!(rendered.contains("protocol_event: turn/completed @ 2026-03-14 10:00:01")); assert!(rendered.contains("Backlog: 1")); - assert!(rendered.contains("Active queue echoes: 1")); + assert!(rendered.contains("Claimed queue echoes: 1")); assert!(rendered.contains("Stale closed queue labels: 1")); assert!(rendered.contains("Backlog")); assert!(rendered.contains("issue: PUB-102")); assert!(rendered.contains("classification: ready")); - assert!(rendered.contains("Active Queue Echoes")); + assert!(rendered.contains("Claimed Queue Echoes")); assert!(rendered.contains("issue: PUB-101")); assert!(rendered.contains("running_owner_run: run-1")); assert!(rendered.contains("Stale Closed Queue Labels")); @@ -182,7 +182,7 @@ fn operator_status_text_surfaces_execution_program_summary() { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), queued_candidates: Vec::new(), recent_runs: Vec::new(), history_lanes: Vec::new(), @@ -251,7 +251,7 @@ fn operator_status_json_uses_direct_dispatch_program_fields() { "account_selector": null, }, "accounts": [], - "active_runs": [], + "current_lanes": [], "recent_runs": [], "history_lanes": [], "execution_programs": [{ @@ -650,10 +650,10 @@ fn operator_status_json_and_text_surface_loop_review_and_recovery_state() { let snapshot = orchestrator::build_operator_status_snapshot(&config, &state_store, 10) .expect("snapshot should build"); let snapshot_json = serde_json::to_value(&snapshot).expect("snapshot should serialize"); - let pending = active_run_json(&snapshot_json, "run-pending"); - let clean = active_run_json(&snapshot_json, "run-clean"); - let findings = active_run_json(&snapshot_json, "run-findings"); - let blocked = active_run_json(&snapshot_json, "run-blocked"); + let pending = current_lane_json(&snapshot_json, "run-pending"); + let clean = current_lane_json(&snapshot_json, "run-clean"); + let findings = current_lane_json(&snapshot_json, "run-findings"); + let blocked = current_lane_json(&snapshot_json, "run-blocked"); assert_eq!(pending["loop_status"]["review_level"], "strict"); assert_eq!(pending["loop_status"]["review"]["status"], "pending"); @@ -824,13 +824,13 @@ fn seed_loop_status_private_events(state_store: &StateStore, config: &ServiceCon .expect("decision request should record"); } -fn active_run_json<'a>(snapshot_json: &'a Value, run_id: &str) -> &'a Value { - snapshot_json["active_runs"] +fn current_lane_json<'a>(snapshot_json: &'a Value, run_id: &str) -> &'a Value { + snapshot_json["current_lanes"] .as_array() - .expect("active runs should be an array") + .expect("current lanes should be an array") .iter() .find(|run| run["run_id"] == run_id) - .unwrap_or_else(|| panic!("active run `{run_id}` should exist")) + .unwrap_or_else(|| panic!("current lane `{run_id}` should exist")) } #[test] @@ -890,7 +890,7 @@ fn operator_status_text_explains_empty_backlog_checks() { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), queued_candidates: Vec::new(), recent_runs: Vec::new(), history_lanes: Vec::new(), @@ -928,7 +928,7 @@ fn operator_status_text_surfaces_cleanup_blocker_pr_url() { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), queued_candidates: Vec::new(), recent_runs: Vec::new(), history_lanes: Vec::new(), @@ -969,7 +969,7 @@ fn operator_status_text_surfaces_cleanup_blocker_pr_url() { mergeable: Some(String::from("MERGEABLE")), check_state: Some(String::from("SUCCESS")), unresolved_review_threads: Some(0), - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: None, readback_root_cause: Some(String::from("lineage_validation_failed")), loop_status: None, @@ -986,11 +986,11 @@ fn operator_status_text_surfaces_cleanup_blocker_pr_url() { #[test] fn operator_status_text_terminal_run_freshness_uses_terminal_update() { - let mut terminal_run = operator_status_text_active_run(); + let mut terminal_run = operator_status_text_current_lane(); terminal_run.status = String::from("succeeded"); terminal_run.phase = String::from("completed"); - terminal_run.active_lease = true; + terminal_run.run_lease = true; terminal_run.updated_at = String::from("2026-03-14 10:05:00"); terminal_run.last_run_activity_at = Some(String::from("2026-03-14 10:10:00Z")); @@ -1009,7 +1009,7 @@ fn operator_status_text_terminal_run_freshness_uses_terminal_update() { account_selector: None, }, accounts: Vec::new(), - active_runs: Vec::new(), + current_lanes: Vec::new(), queued_candidates: Vec::new(), recent_runs: vec![terminal_run], history_lanes, @@ -1021,20 +1021,20 @@ fn operator_status_text_terminal_run_freshness_uses_terminal_update() { assert!(rendered.contains("run_id: run-1")); assert!(rendered.contains("phase: completed")); - assert!(rendered.contains("active_lease: yes")); + assert!(rendered.contains("run_lease: yes")); assert!(rendered.contains("freshness_at: 2026-03-14 10:05:00")); assert!(rendered.contains("freshness_source: updated_at")); assert!(rendered.contains("last_run_activity_at: 2026-03-14 10:10:00Z")); } #[test] -fn operator_status_text_active_run_without_live_activity_does_not_promote_updated_at() { - let mut active_run = operator_status_text_active_run(); +fn operator_status_text_current_lane_without_live_activity_does_not_promote_updated_at() { + let mut current_lane = operator_status_text_current_lane(); - active_run.updated_at = String::from("2026-03-14 09:00:00"); - active_run.last_run_activity_at = None; - active_run.last_protocol_activity_at = None; - active_run.last_progress_at = None; + current_lane.updated_at = String::from("2026-03-14 09:00:00"); + current_lane.last_run_activity_at = None; + current_lane.last_protocol_activity_at = None; + current_lane.last_progress_at = None; let snapshot = OperatorStatusSnapshot { project_id: String::from("pubfi"), @@ -1050,9 +1050,9 @@ fn operator_status_text_active_run_without_live_activity_does_not_promote_update account_selector: None, }, accounts: Vec::new(), - active_runs: vec![active_run.clone()], + current_lanes: vec![current_lane.clone()], queued_candidates: Vec::new(), - recent_runs: vec![active_run], + recent_runs: vec![current_lane], history_lanes: Vec::new(), execution_programs: Vec::new(), worktrees: Vec::new(), @@ -1067,12 +1067,12 @@ fn operator_status_text_active_run_without_live_activity_does_not_promote_update #[test] fn operator_status_text_explains_unleased_live_running_lane() { - let mut active_run = operator_status_text_active_run(); + let mut current_lane = operator_status_text_current_lane(); - active_run.active_lease = false; - active_run.queue_lease_state = String::from("not_held"); - active_run.attempt_status = String::from("stalled"); - active_run.status_projection_reason = + current_lane.run_lease = false; + current_lane.queue_lease_state = String::from("not_held"); + current_lane.attempt_status = String::from("stalled"); + current_lane.status_projection_reason = Some(String::from("terminal_attempt_promoted_by_process_alive")); let snapshot = OperatorStatusSnapshot { @@ -1089,9 +1089,9 @@ fn operator_status_text_explains_unleased_live_running_lane() { account_selector: None, }, accounts: Vec::new(), - active_runs: vec![active_run.clone()], + current_lanes: vec![current_lane.clone()], queued_candidates: Vec::new(), - recent_runs: vec![active_run], + recent_runs: vec![current_lane], history_lanes: Vec::new(), execution_programs: Vec::new(), worktrees: Vec::new(), @@ -1099,7 +1099,7 @@ fn operator_status_text_explains_unleased_live_running_lane() { }; let rendered = orchestrator::render_operator_status(&snapshot); - assert!(rendered.contains("active_lease: no")); + assert!(rendered.contains("run_lease: no")); assert!(rendered.contains("queue_lease_state: not_held")); assert!(rendered.contains("queue_lease: not_held (process_alive keeps lane visible)")); assert!( diff --git a/apps/decodex/src/orchestrator/tests/operator/status_support.rs b/apps/decodex/src/orchestrator/tests/operator/status_support.rs index ea3bc0587..c049447b5 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status_support.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status_support.rs @@ -328,7 +328,7 @@ fn operator_status_text_protocol_activity() -> ProtocolActivitySummary { } } -fn operator_status_text_active_run() -> OperatorRunStatus { +fn operator_status_text_current_lane() -> OperatorRunStatus { let account = operator_status_text_codex_account(); let backup_account = operator_status_text_backup_codex_account(); @@ -340,11 +340,14 @@ fn operator_status_text_active_run() -> OperatorRunStatus { issue_identifier: Some(String::from("PUB-101")), title: Some(String::from("Implement orchestration")), author: Some(String::from("Yvette")), + issue_state: None, + active_label_present: None, + needs_attention_label_present: None, attempt_number: 1, status: String::from("running"), attempt_status: String::from("running"), status_projection_reason: None, - ownership_state: String::from("owned_active"), + ownership_state: String::from("leased_run"), liveness_state: String::from("process_alive"), policy_state: String::from("allowed"), terminalization_state: String::from("none"), @@ -359,7 +362,8 @@ fn operator_status_text_active_run() -> OperatorRunStatus { thread_active_flags: vec![String::from("waitingOnApproval")], interactive_requested: true, continuation_pending: false, - active_lease: true, + continuation_recovery: None, + run_lease: true, queue_lease_state: String::from("held"), execution_liveness: String::from("process_alive"), has_fresh_execution: true, @@ -465,7 +469,7 @@ fn operator_status_text_worktrees() -> Vec { worktree_path: String::from(".worktrees/PUB-104"), ownership: String::from("cleanup_only"), ownership_reason: String::from( - "No active lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", + "No current lane, queued recovery, or post-review lane owns this worktree; local cleanup only.", ), provenance: test_worktree_provenance("runtime_recorded"), recovery_next_action: None, @@ -478,8 +482,8 @@ fn operator_status_text_worktrees() -> Vec { issue_state: Some(String::from("In Progress")), branch_name: String::from("x/pubfi-pub-101"), worktree_path: String::from(".worktrees/PUB-101"), - ownership: String::from("active_lane"), - ownership_reason: String::from("Active lane `run-1` owns this worktree."), + ownership: String::from("current_lane"), + ownership_reason: String::from("Current lane `run-1` owns this worktree."), provenance: test_worktree_provenance("runtime_recorded"), recovery_next_action: None, hygiene: None, @@ -528,7 +532,7 @@ fn operator_status_text_post_review_lanes() -> Vec mergeable: Some(String::from("MERGEABLE")), check_state: Some(String::from("SUCCESS")), unresolved_review_threads: Some(0), - shadowed_by_active_run: false, + shadowed_by_current_lane: false, readback_warning: None, readback_root_cause: None, loop_status: None, diff --git a/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs b/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs index a0f8743ee..e73e5895a 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/closeout/dispatch.rs @@ -89,7 +89,7 @@ fn closeout_dispatch_completes_merged_lane_without_agent_turn() { ); assert!( state_store.lease_for_issue(&issue.id).expect("lease lookup should succeed").is_none(), - "deterministic closeout should not leave an active lease" + "deterministic closeout should not leave an run lease" ); assert_eq!( tracker.label_removals.borrow().len(), diff --git a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs index 5b7e1df02..072d97902 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs @@ -17,7 +17,7 @@ fn reconciliation_sample_service_owned_issue(state_name: &str) -> TrackerIssue { } #[test] -fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { +fn run_lease_reconciliation_detects_terminal_not_dispatchable_and_stalled_runs() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let terminal_issue = sample_issue_with_sort_fields( @@ -28,8 +28,8 @@ fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { Some(3), "2026-03-13T04:16:17.133Z", ); - let nonactive_issue = sample_issue_with_sort_fields( - "issue-nonactive", + let not_dispatchable_issue = sample_issue_with_sort_fields( + "issue-not-dispatchable", "PUB-202", "Blocked", &[], @@ -46,11 +46,11 @@ fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { ); let tracker = FakeTracker::new(vec![ terminal_issue.clone(), - nonactive_issue.clone(), + not_dispatchable_issue.clone(), stalled_issue.clone(), ]); - for issue in [&terminal_issue, &nonactive_issue, &stalled_issue] { + for issue in [&terminal_issue, ¬_dispatchable_issue, &stalled_issue] { state_store .record_run_attempt(&format!("run-{}", issue.identifier), &issue.id, 1, "running") .expect("run attempt should record"); @@ -69,8 +69,8 @@ fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { .expect("stalled issue protocol event should record"); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -78,28 +78,28 @@ fn active_run_reconciliation_detects_terminal_nonactive_and_stalled_runs() { None, now, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!(actions.iter().any(|action| { action.issue.id == terminal_issue.id - && matches!(action.disposition, orchestrator::ActiveRunDisposition::Terminal) + && matches!(action.disposition, orchestrator::RunLeaseDisposition::Terminal) })); assert!(actions.iter().any(|action| { - action.issue.id == nonactive_issue.id - && matches!(action.disposition, orchestrator::ActiveRunDisposition::NonActive) + action.issue.id == not_dispatchable_issue.id + && matches!(action.disposition, orchestrator::RunLeaseDisposition::NotDispatchable) })); assert!(actions.iter().any(|action| { action.issue.id == stalled_issue.id && matches!( action.disposition, - ActiveRunDisposition::Stalled{ idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::Stalled{ idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT ) })); } #[test] -fn active_run_reconciliation_detects_stalled_run_without_protocol_events() { +fn run_lease_reconciliation_detects_stalled_run_without_protocol_events() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let stalled_issue = sample_issue_with_sort_fields( @@ -130,8 +130,8 @@ fn active_run_reconciliation_detects_stalled_run_without_protocol_events() { .expect("lease should record"); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -139,20 +139,20 @@ fn active_run_reconciliation_detects_stalled_run_without_protocol_events() { None, now, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!(actions.iter().any(|action| { action.issue.id == stalled_issue.id && matches!( action.disposition, - ActiveRunDisposition::Stalled{ idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::Stalled{ idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT ) })); } #[test] -fn active_run_reconciliation_supersedes_stale_lease_for_newer_attempt() { +fn run_lease_reconciliation_supersedes_stale_lease_for_newer_attempt() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue_with_sort_fields( @@ -179,7 +179,7 @@ fn active_run_reconciliation_supersedes_stale_lease_for_newer_attempt() { .upsert_lease(config.service_id(), &issue.id, stale_run_id, "In Progress") .expect("stale lease should record"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -187,18 +187,18 @@ fn active_run_reconciliation_supersedes_stale_lease_for_newer_attempt() { None, OffsetDateTime::now_utc().unix_timestamp(), ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert_eq!(actions.len(), 1); assert!(matches!( &actions[0].disposition, - ActiveRunDisposition::Superseded { + RunLeaseDisposition::Superseded { newer_run_id: observed_run_id, newer_attempt_number: 2, } if observed_run_id == newer_run_id )); - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -223,7 +223,7 @@ fn active_run_reconciliation_supersedes_stale_lease_for_newer_attempt() { } #[test] -fn active_run_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() { +fn run_lease_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() { let (_temp_dir, base_config, workflow) = temp_project_layout(); let config = service_config_with_github_token_env_var( &base_config, @@ -264,7 +264,7 @@ fn active_run_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() state::write_run_activity_marker(&worktree.path, run_id, 1) .expect("fresh activity marker should write"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -272,11 +272,11 @@ fn active_run_reconciliation_keeps_completed_closeout_lane_with_fresh_activity() None, OffsetDateTime::now_utc().unix_timestamp(), ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!( actions.is_empty(), - "completed retained closeout lanes with fresh activity must not be reconciled as terminal or non-active" + "completed retained closeout lanes with fresh activity must not be reconciled as terminal or not-dispatchable" ); } @@ -322,32 +322,32 @@ fn active_daemon_child_reconciliation_keeps_completed_closeout_lane_with_fresh_a state::write_run_activity_marker(&worktree.path, run_id, 1) .expect("fresh activity marker should write"); - let actions = orchestrator::inspect_active_daemon_child_reconciliation( + let actions = orchestrator::inspect_current_daemon_child_reconciliation( &tracker, &config, &workflow, &state_store, - ActiveChildRunContext { + CurrentChildRunContext { child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, workflow: &workflow, dispatch_mode: IssueDispatchMode::Closeout, }, ) - .expect("active daemon-child inspection should succeed"); + .expect("current daemon-child inspection should succeed"); assert!( actions.is_empty(), - "completed retained closeout daemon children with fresh activity must not be reconciled as terminal or non-active" + "completed retained closeout daemon children with fresh activity must not be reconciled as terminal or not-dispatchable" ); } #[test] -fn active_daemon_child_reconciliation_keeps_review_repair_lane_in_review() { +fn current_daemon_child_reconciliation_keeps_review_repair_lane_in_review() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = reconciliation_sample_service_owned_issue("In Review"); let tracker = FakeTracker::new(vec![issue.clone()]); let state_store = StateStore::open_in_memory().expect("state store should open"); - let run_id = "run-review-repair-active"; + let run_id = "run-review-repair-current"; let worktree_manager = WorktreeManager::new(config.service_id(), config.repo_root(), config.worktree_root()); let worktree = worktree_manager @@ -369,27 +369,27 @@ fn active_daemon_child_reconciliation_keeps_review_repair_lane_in_review() { ) .expect("worktree mapping should record"); - let actions = orchestrator::inspect_active_daemon_child_reconciliation( + let actions = orchestrator::inspect_current_daemon_child_reconciliation( &tracker, &config, &workflow, &state_store, - ActiveChildRunContext { + CurrentChildRunContext { child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, workflow: &workflow, dispatch_mode: IssueDispatchMode::ReviewRepair, }, ) - .expect("active review-repair daemon-child inspection should succeed"); + .expect("current review-repair daemon-child inspection should succeed"); assert!( actions.is_empty(), - "active review-repair lanes in In Review must stay active instead of being interrupted as non-active" + "review-repair lanes in In Review must stay current instead of being interrupted as not-dispatchable" ); } #[test] -fn active_daemon_child_reconciliation_keeps_closeout_child_after_tracker_completion() { +fn current_daemon_child_reconciliation_keeps_closeout_child_after_tracker_completion() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue("Done", &[]); let tracker = FakeTracker::new(vec![issue.clone()]); @@ -403,18 +403,18 @@ fn active_daemon_child_reconciliation_keeps_closeout_child_after_tracker_complet .upsert_lease(config.service_id(), &issue.id, run_id, "Done") .expect("lease should record"); - let actions = orchestrator::inspect_active_daemon_child_reconciliation( + let actions = orchestrator::inspect_current_daemon_child_reconciliation( &tracker, &config, &workflow, &state_store, - ActiveChildRunContext { + CurrentChildRunContext { child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, workflow: &workflow, dispatch_mode: IssueDispatchMode::Closeout, }, ) - .expect("active closeout daemon-child inspection should succeed"); + .expect("current closeout daemon-child inspection should succeed"); assert!( actions.is_empty(), @@ -423,7 +423,7 @@ fn active_daemon_child_reconciliation_keeps_closeout_child_after_tracker_complet } #[test] -fn active_run_reconciliation_treats_completed_retained_handoff_as_success() { +fn run_lease_reconciliation_treats_completed_retained_handoff_as_success() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue_with_sort_fields( "issue-handoff-complete", @@ -470,8 +470,8 @@ fn active_run_reconciliation_treats_completed_retained_handoff_as_success() { ); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -479,12 +479,12 @@ fn active_run_reconciliation_treats_completed_retained_handoff_as_success() { None, now, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert_eq!(actions.len(), 1); - assert!(matches!(actions[0].disposition, ActiveRunDisposition::RetainedReviewComplete)); + assert!(matches!(actions[0].disposition, RunLeaseDisposition::RetainedReviewComplete)); - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -516,7 +516,7 @@ fn active_run_reconciliation_treats_completed_retained_handoff_as_success() { } #[test] -fn active_run_reconciliation_ignores_stale_retained_handoff_marker() { +fn run_lease_reconciliation_ignores_stale_retained_handoff_marker() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue_with_sort_fields( "issue-stale-handoff", @@ -570,8 +570,8 @@ fn active_run_reconciliation_ignores_stale_retained_handoff_marker() { ); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -579,13 +579,13 @@ fn active_run_reconciliation_ignores_stale_retained_handoff_marker() { None, now, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert_eq!(actions.len(), 1); assert!(matches!( actions[0].disposition, - ActiveRunDisposition::Stalled { idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::Stalled { idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT )); } @@ -637,23 +637,23 @@ fn active_daemon_child_reconciliation_treats_completed_retained_handoff_as_succe ); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_daemon_child_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_current_daemon_child_reconciliation_at( &tracker, &config, &workflow, &state_store, - ActiveChildRunContext { + CurrentChildRunContext { child: ChildRunRef { issue_id: &issue.id, run_id, attempt_number: 1 }, workflow: &workflow, dispatch_mode: IssueDispatchMode::Normal, }, now, ) - .expect("active daemon-child inspection should succeed"); + .expect("current daemon-child inspection should succeed"); assert_eq!(actions.len(), 1); - assert!(matches!(actions[0].disposition, ActiveRunDisposition::RetainedReviewComplete)); + assert!(matches!(actions[0].disposition, RunLeaseDisposition::RetainedReviewComplete)); } #[test] @@ -687,7 +687,7 @@ fn stalled_idle_duration_ignores_future_last_activity() { } #[test] -fn active_run_reconciliation_uses_worktree_activity_marker_from_child_process() { +fn run_lease_reconciliation_uses_worktree_activity_marker_from_child_process() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue("In Progress", &[]); let tracker = FakeTracker::new(vec![issue.clone()]); @@ -722,20 +722,20 @@ fn active_run_reconciliation_uses_worktree_activity_marker_from_child_process() &marker_path, format!( "run_id={run_id}\nattempt_number=1\nlast_activity_unix_epoch={}\n", - last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + last_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 ), ) .expect("activity marker should write"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, &state_store, None, - last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) - .expect("active run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!( actions.is_empty(), @@ -744,7 +744,7 @@ fn active_run_reconciliation_uses_worktree_activity_marker_from_child_process() } #[test] -fn active_run_reconciliation_allows_running_model_execution_until_model_timeout() { +fn run_lease_reconciliation_allows_running_model_execution_until_model_timeout() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue("In Progress", &[]); let tracker = FakeTracker::new(vec![issue.clone()]); @@ -784,22 +784,22 @@ fn active_run_reconciliation_allows_running_model_execution_until_model_timeout( ) .expect("activity marker should write"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, &state_store, None, - last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) - .expect("active run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!( actions.is_empty(), "running model execution should not stall on the generic active idle timeout" ); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -807,20 +807,20 @@ fn active_run_reconciliation_allows_running_model_execution_until_model_timeout( None, last_activity + MODEL_EXECUTION_IDLE_TIMEOUT.as_secs() as i64 + 1, ) - .expect("active run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!(actions.iter().any(|action| { action.issue.id == issue.id && matches!( action.disposition, - ActiveRunDisposition::Stalled{ idle_for } + RunLeaseDisposition::Stalled{ idle_for } if idle_for >= MODEL_EXECUTION_IDLE_TIMEOUT ) })); } #[test] -fn active_run_reconciliation_defers_live_repo_gate_even_with_dirty_worktree() { +fn run_lease_reconciliation_defers_live_repo_gate_even_with_dirty_worktree() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = sample_issue("In Progress", &[]); let tracker = FakeTracker::new(vec![issue.clone()]); @@ -871,15 +871,15 @@ fn active_run_reconciliation_defers_live_repo_gate_even_with_dirty_worktree() { .expect("marker should load") .expect("marker should exist"); let last_activity = marker.last_activity_unix_epoch().expect("marker should record activity"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, &state_store, None, - last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) - .expect("active run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!( actions.is_empty(), @@ -899,22 +899,22 @@ fn active_run_reconciliation_defers_live_repo_gate_even_with_dirty_worktree() { .expect("marker should reload") .expect("marker should exist"); let last_activity = marker.last_activity_unix_epoch().expect("marker should record activity"); - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, &state_store, None, - last_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) - .expect("active run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!(actions.iter().any(|action| { action.issue.id == issue.id && matches!( action.disposition, - ActiveRunDisposition::StalledRetainedPartialProgress{ idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::StalledRetainedPartialProgress{ idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT ) })); } @@ -970,7 +970,7 @@ fn exited_child_reconciliation_defers_dirty_worktree_with_retry_marker() { &state_store, &issue.id, run_id, - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) .expect("exited child reconciliation should evaluate"); @@ -1076,8 +1076,8 @@ fn apply_stalled_phase_goal_reconciliation( .expect("stalled dirty issue protocol event should record"); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -1090,10 +1090,10 @@ fn apply_stalled_phase_goal_reconciliation( assert_eq!(actions.len(), 1); assert!(matches!( actions[0].disposition, - ActiveRunDisposition::StalledRetainedPartialProgress { .. } + RunLeaseDisposition::StalledRetainedPartialProgress { .. } )); - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1190,7 +1190,7 @@ fn stalled_protocol_idle_duration_ignores_future_protocol_activity() { } #[test] -fn active_run_reconciliation_ignores_startable_preclaim_states() { +fn run_lease_reconciliation_ignores_startable_preclaim_states() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); let issue = sample_issue_with_sort_fields( @@ -1211,7 +1211,7 @@ fn active_run_reconciliation_ignores_startable_preclaim_states() { .expect("lease should record"); let now = OffsetDateTime::now_utc().unix_timestamp() + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -1219,13 +1219,13 @@ fn active_run_reconciliation_ignores_startable_preclaim_states() { None, now, ) - .expect("active-run inspection should succeed"); + .expect("run lease inspection should succeed"); assert!(actions.is_empty(), "startable pre-claim states should not be interrupted"); } #[test] -fn active_run_reconciliation_clears_terminal_lane_labels() { +fn run_lease_reconciliation_clears_terminal_lane_labels() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = reconciliation_sample_service_owned_issue("Done"); let tracker = FakeTracker::new(vec![issue.clone()]); @@ -1250,7 +1250,7 @@ fn active_run_reconciliation_clears_terminal_lane_labels() { ) .expect("worktree mapping should record"); - let action = ActiveRunReconciliation { + let action = RunLeaseReconciliation { issue: issue.clone(), run_attempt: state_store .run_attempt(run_id) @@ -1259,11 +1259,11 @@ fn active_run_reconciliation_clears_terminal_lane_labels() { worktree_mapping: state_store .worktree_for_issue(&issue.id) .expect("worktree query should succeed"), - disposition: ActiveRunDisposition::Terminal, + disposition: RunLeaseDisposition::Terminal, workflow: workflow.clone(), }; - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1283,14 +1283,14 @@ fn active_run_reconciliation_clears_terminal_lane_labels() { } #[test] -fn active_run_reconciliation_keeps_nonterminal_nonactive_worktrees() { +fn run_lease_reconciliation_keeps_nonterminal_not_dispatchable_worktrees() { let (_temp_dir, config, workflow) = temp_project_layout(); let tracker = FakeTracker::new(vec![]); let state_store = StateStore::open_in_memory().expect("state store should open"); let worktree_manager = WorktreeManager::new("pubfi", config.repo_root(), config.worktree_root()); let issue = sample_issue("Todo", &[]); - let run_id = "run-nonactive"; + let run_id = "run-not-dispatchable"; let worktree_path = config.worktree_root().join("PUB-101"); state_store @@ -1308,7 +1308,7 @@ fn active_run_reconciliation_keeps_nonterminal_nonactive_worktrees() { ) .expect("worktree mapping should record"); - let action = ActiveRunReconciliation { + let action = RunLeaseReconciliation { issue: issue.clone(), run_attempt: state_store .run_attempt(run_id) @@ -1317,11 +1317,11 @@ fn active_run_reconciliation_keeps_nonterminal_nonactive_worktrees() { worktree_mapping: state_store .worktree_for_issue(&issue.id) .expect("worktree query should succeed"), - disposition: ActiveRunDisposition::NonActive, + disposition: RunLeaseDisposition::NotDispatchable, workflow: workflow.clone(), }; - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1375,7 +1375,7 @@ fn stalled_run_reconciliation_schedules_retry_before_attention_budget_exhaustion ) .expect("worktree mapping should record"); - let action = ActiveRunReconciliation { + let action = RunLeaseReconciliation { issue: issue.clone(), run_attempt: state_store .run_attempt(run_id) @@ -1384,13 +1384,13 @@ fn stalled_run_reconciliation_schedules_retry_before_attention_budget_exhaustion worktree_mapping: state_store .worktree_for_issue(&issue.id) .expect("worktree query should succeed"), - disposition: ActiveRunDisposition::Stalled { - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + disposition: RunLeaseDisposition::Stalled { + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }, workflow: workflow.clone(), }; - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1476,7 +1476,7 @@ fn stalled_run_reconciliation_routes_to_needs_attention_after_retry_budget_exhau ) .expect("worktree mapping should record"); - let action = ActiveRunReconciliation { + let action = RunLeaseReconciliation { issue: issue.clone(), run_attempt: state_store .run_attempt(run_id) @@ -1485,13 +1485,13 @@ fn stalled_run_reconciliation_routes_to_needs_attention_after_retry_budget_exhau worktree_mapping: state_store .worktree_for_issue(&issue.id) .expect("worktree query should succeed"), - disposition: ActiveRunDisposition::Stalled { - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + disposition: RunLeaseDisposition::Stalled { + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }, workflow: workflow.clone(), }; - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1563,8 +1563,8 @@ fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktr .expect("stalled dirty issue protocol event should record"); let now = - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1; - let actions = orchestrator::inspect_active_run_reconciliation_at( + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1; + let actions = orchestrator::inspect_run_lease_reconciliation_at( &tracker, &config, &workflow, @@ -1577,11 +1577,11 @@ fn stalled_run_reconciliation_reports_retained_partial_progress_for_dirty_worktr assert_eq!(actions.len(), 1); assert!(matches!( actions[0].disposition, - ActiveRunDisposition::StalledRetainedPartialProgress { idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::StalledRetainedPartialProgress { idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT )); - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -1816,7 +1816,7 @@ fn stalled_run_reconciliation_preserves_retry_budget_marker_from_retained_worktr ) .expect("worktree mapping should record"); - let action = ActiveRunReconciliation { + let action = RunLeaseReconciliation { issue: issue.clone(), run_attempt: state_store .run_attempt(run_id) @@ -1825,13 +1825,13 @@ fn stalled_run_reconciliation_preserves_retry_budget_marker_from_retained_worktr worktree_mapping: state_store .worktree_for_issue(&issue.id) .expect("worktree query should succeed"), - disposition: ActiveRunDisposition::Stalled { - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + disposition: RunLeaseDisposition::Stalled { + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }, workflow: workflow.clone(), }; - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, diff --git a/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs b/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs index d4ca19bbc..6d948a214 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs @@ -59,7 +59,7 @@ fn exited_child_reconciliation_detects_stalled_failed_runs_from_protocol_idle() &state_store, &issue.id, run_id, - last_protocol_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_protocol_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) .expect("exited child inspection should succeed"); @@ -67,8 +67,8 @@ fn exited_child_reconciliation_detects_stalled_failed_runs_from_protocol_idle() action.issue.id == issue.id && matches!( action.disposition, - ActiveRunDisposition::Stalled{ idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::Stalled{ idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT ) })); } @@ -134,15 +134,15 @@ fn exited_child_reconciliation_detects_retained_partial_progress_from_dirty_work &state_store, &issue.id, run_id, - last_protocol_activity + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + last_protocol_activity + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) .expect("exited child inspection should succeed"); assert_eq!(actions.len(), 1); assert!(matches!( actions[0].disposition, - ActiveRunDisposition::StalledRetainedPartialProgress { idle_for } - if idle_for >= ACTIVE_RUN_IDLE_TIMEOUT + RunLeaseDisposition::StalledRetainedPartialProgress { idle_for } + if idle_for >= RUN_LEASE_IDLE_TIMEOUT )); } @@ -207,20 +207,20 @@ fn exited_child_reconciliation_ignores_superseded_failed_run() { &state_store, &issue.id, stale_run_id, - OffsetDateTime::now_utc().unix_timestamp() + ACTIVE_RUN_IDLE_TIMEOUT.as_secs() as i64 + 1, + OffsetDateTime::now_utc().unix_timestamp() + RUN_LEASE_IDLE_TIMEOUT.as_secs() as i64 + 1, ) .expect("exited child inspection should succeed"); assert_eq!(actions.len(), 1); assert!(matches!( &actions[0].disposition, - ActiveRunDisposition::Superseded { + RunLeaseDisposition::Superseded { newer_run_id: observed_run_id, newer_attempt_number: 2, } if observed_run_id == newer_run_id )); - orchestrator::apply_active_run_reconciliation( + orchestrator::apply_run_lease_reconciliation( &tracker, &config, &state_store, @@ -294,7 +294,7 @@ fn recover_runtime_state_recovers_fresh_review_repair_activity_marker() { .expect("runtime recovery should succeed"); assert!( - recovered.active_issues.is_empty(), + recovered.recoverable_issues.is_empty(), "fresh review-repair activity should rebuild the lease instead of requeueing the lane" ); @@ -378,7 +378,7 @@ fn run_project_once_recovers_ready_post_review_lane_before_landing() { assert!( summary.is_none(), - "ready retained post-review landing should not dispatch a new active run" + "ready retained post-review landing should not dispatch a new current lane" ); let marker = persisted_review_orchestration_marker_for_path( @@ -606,7 +606,7 @@ fn run_project_once_skips_recovered_worktree_with_fresh_activity_marker() { assert!( summary.is_none(), - "fresh child activity should recover as an active lane instead of redispatching" + "fresh child activity should recover as a current lane instead of redispatching" ); assert_eq!( state_store @@ -1227,8 +1227,8 @@ fn recovery_skip_cache_suppresses_repeated_unowned_worktree_lookup() { .expect("cached recovery probe should succeed"); let identifier_queries = tracker.identifier_queries.borrow(); - assert!(first.active_issues.is_empty()); - assert!(second.active_issues.is_empty()); + assert!(first.recoverable_issues.is_empty()); + assert!(second.recoverable_issues.is_empty()); assert_eq!(identifier_queries.len(), 1); assert_eq!(identifier_queries[0], issue.identifier); assert!( diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index 3744a87cf..e02182a72 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -1194,7 +1194,7 @@ fn retryable_orchestrator_failures_do_not_write_attention_before_budget_exhausti Report::new(StalledRunNeedsAttention { issue_identifier: String::from("PUB-103"), run_id: String::from("pub-103-attempt-1-123"), - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }), "stalled_run_detected", ); diff --git a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs index f164b7738..e0a8af510 100644 --- a/apps/decodex/src/orchestrator/tests/retry/scheduling.rs +++ b/apps/decodex/src/orchestrator/tests/retry/scheduling.rs @@ -820,7 +820,7 @@ fn schedule_retry_after_child_exit_records_continuation_retry_for_clean_exit() { } #[test] -fn schedule_retry_after_child_exit_terminalizes_active_phase_goal_tracked_rewrites() { +fn schedule_retry_after_child_exit_terminalizes_open_phase_goal_tracked_rewrites() { let (_temp_dir, config, workflow) = temp_project_layout_with_workflow_markdown( &sample_workflow_markdown( "pubfi", @@ -894,7 +894,7 @@ fn schedule_retry_after_child_exit_terminalizes_active_phase_goal_tracked_rewrit IssueDispatchMode::Retry, exit_status, ) - .expect("active phase goal tracked rewrites should terminalize cleanly"); + .expect("open phase goal tracked rewrites should terminalize cleanly"); let run_attempt = state_store .run_attempt(run_id) @@ -1464,7 +1464,7 @@ fn spawn_sleeping_daemon_child( DaemonRunChild { child, issue_id: active_issue.id.clone(), - run_id: String::from("active-run"), + run_id: String::from("leased-run"), attempt_number: 1, initial_issue_state: active_issue.state.name.clone(), retry_project_slug: active_issue @@ -1528,8 +1528,8 @@ fn daemon_tick_reconciles_ready_retained_review_lane_before_dry_run_planning() { ) .expect("retained worktree should record"); state_store - .record_run_attempt("active-run", &active_issue.id, 1, "running") - .expect("active run should record"); + .record_run_attempt("leased-run", &active_issue.id, 1, "running") + .expect("current lane should record"); seed_review_handoff_marker_value( &state_store, diff --git a/apps/decodex/src/orchestrator/tests/retry/selection.rs b/apps/decodex/src/orchestrator/tests/retry/selection.rs index 1132dddb8..ae34ec522 100644 --- a/apps/decodex/src/orchestrator/tests/retry/selection.rs +++ b/apps/decodex/src/orchestrator/tests/retry/selection.rs @@ -279,7 +279,7 @@ fn future_retry_claim_stays_blocked_when_issue_moves_to_another_project_before_d } #[test] -fn future_retry_claim_stays_blocked_when_issue_becomes_non_active_before_due_time() { +fn future_retry_claim_stays_blocked_when_issue_becomes_not_dispatchable_before_due_time() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = selection_sample_service_owned_issue("In Review"); let tracker = @@ -326,7 +326,7 @@ fn future_retry_claim_stays_blocked_when_issue_becomes_non_active_before_due_tim } #[test] -fn due_retry_claim_releases_when_issue_becomes_non_active() { +fn due_retry_claim_releases_when_issue_becomes_not_dispatchable() { let (_temp_dir, config, workflow) = temp_project_layout(); let issue = selection_sample_service_owned_issue("In Review"); let tracker = @@ -357,7 +357,7 @@ fn due_retry_claim_releases_when_issue_becomes_non_active() { .expect("retry planning should succeed"); assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); - assert!(retry_queue.is_empty(), "due non-active issue should release the queued claim"); + assert!(retry_queue.is_empty(), "due not-dispatchable issue should release the queued claim"); } #[test] @@ -411,7 +411,7 @@ fn due_retry_claim_release_clears_persisted_retry_marker() { .expect("retry planning should succeed"); assert!(matches!(decision, orchestrator::RetryDispatchDecision::Continue)); - assert!(retry_queue.is_empty(), "non-active issue should release the queued claim when due"); + assert!(retry_queue.is_empty(), "not-dispatchable issue should release the queued claim when due"); let marker = state::read_run_activity_marker_snapshot(&worktree_path) .expect("marker should load") diff --git a/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs b/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs index eade8c68b..0a6dbea41 100644 --- a/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs +++ b/apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs @@ -1289,7 +1289,7 @@ fn reconcile_post_review_orchestration_fails_closed_when_pull_request_is_closed( } #[test] -fn reconcile_post_review_orchestration_skips_issue_with_active_lease() { +fn reconcile_post_review_orchestration_skips_issue_with_run_lease() { let (_temp_dir, config, workflow) = temp_project_layout(); let repo_root = config.repo_root().to_path_buf(); let issue = post_review_sample_service_owned_issue("In Review"); @@ -1335,7 +1335,7 @@ fn reconcile_post_review_orchestration_skips_issue_with_active_lease() { "active-repair-run", workflow.frontmatter().tracker().in_progress_state(), ) - .expect("active lease should acquire") + .expect("run lease should acquire") ); orchestrator::reconcile_post_review_orchestration_with_inspector( diff --git a/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs b/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs index 3eac886aa..b1de2662d 100644 --- a/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs +++ b/apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs @@ -1025,7 +1025,7 @@ fn build_post_review_lane_statuses_blocks_review_handoff_lineage_rewrite() { } #[test] -fn build_post_review_lane_statuses_blocks_nonactive_labeled_post_review_issues() { +fn build_post_review_lane_statuses_blocks_not_dispatchable_labeled_post_review_issues() { let (_temp_dir, config, workflow) = temp_project_layout(); let state_store = StateStore::open_in_memory().expect("state store should open"); diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs index 4cfb5cf0f..993925626 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/failure.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -182,11 +182,11 @@ fn failure_writeback_disposition_marks_retryable_recovery_classes() { RunFailureWritebackDisposition::RetryableStructuredRecovery, ), ( - "stalled active run", + "stalled current lane", Report::new(StalledRunNeedsAttention { issue_identifier: String::from("PUB-101"), run_id: String::from("pub-101-attempt-1-123"), - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }), RunFailureWritebackDisposition::RetryableStructuredRecovery, ), @@ -646,7 +646,7 @@ fn stalled_run_retry_comments_preserve_specific_error_class() { let error = Report::new(StalledRunNeedsAttention { issue_identifier: String::from("PUB-101"), run_id: String::from("pub-101-attempt-1-123"), - idle_for: ACTIVE_RUN_IDLE_TIMEOUT + Duration::from_secs(1), + idle_for: RUN_LEASE_IDLE_TIMEOUT + Duration::from_secs(1), }); let (error_class, next_action) = orchestrator::retry_comment_details(&error); diff --git a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs index fa8f1cd58..6608f7836 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs @@ -204,7 +204,7 @@ fn phase_goal_completion_runs_repo_gate_and_persists_handoff_phase() { } #[test] -fn active_phase_goal_tracked_rewrites_stop_instead_of_repair_continuation() { +fn open_phase_goal_tracked_rewrites_stop_instead_of_repair_continuation() { let (_temp_dir, config, workflow) = temp_project_layout_with_workflow_markdown( &sample_workflow_markdown( "pubfi", @@ -259,7 +259,7 @@ fn active_phase_goal_tracked_rewrites_stop_instead_of_repair_continuation() { ) .expect("phase goal event should record"); - let error = orchestrator::maybe_continue_after_active_phase_goal_recovery( + let error = orchestrator::maybe_continue_after_phase_goal_recovery( &config, &workflow, &state_store, @@ -288,6 +288,122 @@ fn active_phase_goal_tracked_rewrites_stop_instead_of_repair_continuation() { assert!(events.iter().all(|event| event.event_type() != "phase_goal_recovery")); } +#[test] +fn repeated_phase_goal_recovery_blocks_second_automatic_continuation() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let repo_root = config.repo_root(); + let issue = sample_issue("In Progress", &[tracker::automation_active_label(TEST_SERVICE_ID).as_str()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let app_server_timeout = Report::new(AppServerCapabilityPreflightFailure::method_timed_out_for_test( + "thread/goal/get", + String::from("Timed out while waiting for app-server output."), + )); + + commit_worktree_change(repo_root, "ready.txt", "before\n", "add ready file"); + + fs::write(repo_root.join("ready.txt"), "after\n").expect("tracked diff should write"); + + for (run_id, attempt_number) in [("pub-101-attempt-1", 1), ("pub-101-attempt-2", 2)] { + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + run_id, + attempt_number, + "phase_goal_set", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "implement_to_validation_ready", + "payload": { + "phase": "implement_to_validation_ready", + "status": "active", + }, + }), + ) + .expect("phase goal event should record"); + } + + let first_issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: repo_root.to_path_buf(), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1"), + retry_budget_base: 0, + }; + let second_issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: repo_root.to_path_buf(), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 2, + run_id: String::from("pub-101-attempt-2"), + retry_budget_base: 0, + }; + let first = orchestrator::maybe_continue_after_phase_goal_recovery( + &config, + &workflow, + &state_store, + &first_issue_run, + &app_server_timeout, + ) + .expect("first recovery should evaluate") + .expect("first recovery should schedule continuation"); + let second = orchestrator::maybe_continue_after_phase_goal_recovery( + &config, + &workflow, + &state_store, + &second_issue_run, + &app_server_timeout, + ) + .expect("second recovery should evaluate"); + let events = state_store + .list_private_execution_events_for_issue(TEST_SERVICE_ID, &issue.id) + .expect("private phase goal events should load"); + let scheduled_events = events + .iter() + .filter(|event| event.event_type() == PHASE_GOAL_RECOVERY_EVENT_TYPE) + .collect::>(); + let blocked_event = events + .iter() + .find(|event| { + event.event_type() == PHASE_GOAL_RECOVERY_BLOCKED_EVENT_TYPE + }) + .expect("second recovery should record blocked event"); + + assert!(first.continuation_pending); + assert!(second.is_none()); + assert_eq!(scheduled_events.len(), 1); + assert_eq!(blocked_event.payload()["signal"], "continuation_budget_exhausted"); + assert_eq!(blocked_event.payload()["payload"]["priorRecoveryCount"], 1); + assert_eq!( + blocked_event.payload()["payload"]["automaticContinuationLimit"], + orchestrator::PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT + ); + assert!(blocked_event.payload()["payload"]["sourceErrorMessage"] + .as_str() + .is_some_and(|message| message.contains("Timed out while waiting for app-server output."))); + assert_eq!( + blocked_event.payload()["payload"]["sourceErrorClass"], + "app_server_preflight_timeout" + ); +} + #[test] fn implementation_phase_goal_contract_requires_explicit_goal_completion() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index ca713b573..d32cb0e5d 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -16,6 +16,10 @@ pub(crate) const ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE: &str = "architecture_recovery_started"; pub(crate) const ARCHITECTURE_RECOVERY_TERMINAL_EVENT_TYPE: &str = "architecture_recovery_terminal"; +pub(crate) const PHASE_GOAL_RECOVERY_EVENT_TYPE: &str = "phase_goal_recovery"; +pub(crate) const PHASE_GOAL_RECOVERY_BLOCKED_EVENT_TYPE: &str = + "phase_goal_recovery_blocked"; +pub(crate) const PHASE_GOAL_RECOVERY_AUTOMATIC_CONTINUATION_LIMIT: i64 = 1; #[allow(dead_code)] const AUTHORITY_BOUNDARY_CHECK_SCHEMA: &str = "decodex.authority_boundary_check/1"; @@ -151,14 +155,14 @@ pub(crate) enum RetryDispatchDecision { } #[derive(Clone, Debug)] -pub(crate) enum ActiveRunDisposition { +pub(crate) enum RunLeaseDisposition { RetainedReviewComplete, Superseded { newer_run_id: String, newer_attempt_number: i64, }, Terminal, - NonActive, + NotDispatchable, Stalled { idle_for: Duration }, StalledRetainedPartialProgress { idle_for: Duration }, StalledAlreadyNeedsAttention { idle_for: Duration }, @@ -423,7 +427,7 @@ pub(crate) struct EvidenceRequest<'a> { pub(crate) include_payload: bool, } -/// Active lane steer request. +/// Current lane steer request. pub(crate) struct LaneSteerRequest<'a> { pub(crate) config_path: Option<&'a Path>, pub(crate) project_id: Option<&'a str>, @@ -435,7 +439,7 @@ pub(crate) struct LaneSteerRequest<'a> { pub(crate) wait_timeout: Duration, } -/// Active lane steer result without raw operator message content. +/// Current lane steer result without raw operator message content. #[derive(Clone, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct LaneSteerReport { @@ -525,7 +529,7 @@ struct IssueRunPlan { #[derive(Default)] struct RecoveredRuntimeState { - active_issues: Vec, + recoverable_issues: Vec, } #[derive(Clone, Copy)] @@ -952,7 +956,7 @@ struct ChildRunRef<'a> { } #[derive(Clone, Copy)] -struct ActiveChildRunContext<'a> { +struct CurrentChildRunContext<'a> { child: ChildRunRef<'a>, workflow: &'a WorkflowDocument, dispatch_mode: IssueDispatchMode, @@ -1225,11 +1229,11 @@ struct ActiveWorkflowOverride<'a> { } #[derive(Clone, Debug)] -struct ActiveRunReconciliation { +struct RunLeaseReconciliation { issue: TrackerIssue, run_attempt: RunAttempt, worktree_mapping: Option, - disposition: ActiveRunDisposition, + disposition: RunLeaseDisposition, workflow: WorkflowDocument, } @@ -1252,7 +1256,7 @@ struct OperatorStatusSnapshot { projects: Vec, account_control: OperatorCodexAccountControlStatus, accounts: Vec, - active_runs: Vec, + current_lanes: Vec, recent_runs: Vec, history_lanes: Vec, execution_programs: Vec, @@ -1291,7 +1295,7 @@ struct OperatorProjectStatus { repo_root: String, enabled: bool, github_cli_authority: OperatorGitHubCliAuthority, - active_run_count: usize, + current_lane_count: usize, running_lane_count: usize, queued_candidate_count: usize, post_review_lane_count: usize, @@ -1525,6 +1529,12 @@ struct OperatorRunStatus { issue_identifier: Option, title: Option, author: Option, + #[serde(skip_serializing_if = "Option::is_none")] + issue_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + active_label_present: Option, + #[serde(skip_serializing_if = "Option::is_none")] + needs_attention_label_present: Option, attempt_number: i64, status: String, attempt_status: String, @@ -1546,7 +1556,9 @@ struct OperatorRunStatus { thread_active_flags: Vec, interactive_requested: bool, continuation_pending: bool, - active_lease: bool, + #[serde(skip_serializing_if = "Option::is_none")] + continuation_recovery: Option, + run_lease: bool, queue_lease_state: String, execution_liveness: String, has_fresh_execution: bool, @@ -1589,6 +1601,23 @@ struct OperatorRunStatus { worktree_path: Option, } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +struct OperatorContinuationRecoveryStatus { + state: String, + source_phase: String, + next_phase: String, + source_error_class: String, + #[serde(skip_serializing_if = "Option::is_none")] + source_error_message: Option, + recorded_at: String, + run_id: String, + attempt_number: i64, + recovery_count: i64, + automatic_continuation_limit: i64, + budget_exceeded: bool, + next_action: String, +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] struct OperatorRunControlCapability { project_id: String, @@ -1759,7 +1788,7 @@ struct OperatorPostReviewLaneStatus { mergeable: Option, check_state: Option, unresolved_review_threads: Option, - shadowed_by_active_run: bool, + shadowed_by_current_lane: bool, readback_warning: Option, readback_root_cause: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1954,17 +1983,17 @@ struct TargetIssueRunContext<'a, T> { } struct ConcurrencySnapshot { - total_active: usize, + total_leased: usize, } impl ConcurrencySnapshot { fn new(project_id: &str, state_store: &StateStore) -> crate::prelude::Result { let leases = state_store.list_active_shared_leases(project_id)?; - Ok(Self { total_active: leases.len() }) + Ok(Self { total_leased: leases.len() }) } fn has_global_capacity(&self, execution: &WorkflowExecution) -> bool { - execution.max_concurrent_agents().has_capacity(self.total_active) + execution.max_concurrent_agents().has_capacity(self.total_leased) } } @@ -2417,8 +2446,8 @@ fn operator_execution_program_reason_code(reason: &str) -> &'static str { "dispatch_intent_not_ready" } else if reason == "node dispatch intent is paused" { "dispatch_intent_paused" - } else if reason == "node already has an active lane" { - "active_lane_present" + } else if reason == "node already has a current lane" { + "current_lane_present" } else if reason == "node dispatch intent is terminal" { "dispatch_intent_terminal" } else if reason == "node is ready for normal Linear issue execution" { @@ -2495,7 +2524,7 @@ fn operator_execution_program_node_next_action( crate::execution_program::ExecutionProgramNodeLifecycleState::Active ) { return String::from( - "Wait for the active lane or recover its retained state before dispatching this node.", + "Wait for the current lane or recover its retained state before dispatching this node.", ); } if node.dispatch_action().is_some() { diff --git a/apps/decodex/src/recovery.rs b/apps/decodex/src/recovery.rs index aa73f9d39..82664a6bd 100644 --- a/apps/decodex/src/recovery.rs +++ b/apps/decodex/src/recovery.rs @@ -1064,7 +1064,7 @@ fn rebind_state_transition_next_action(issue_identifier: &str, pr_url: &str) -> fn issue_state_mismatch_next_action(success_state: &str, in_progress_state: &str) -> String { format!( - "Move the issue to `{success_state}` or `{in_progress_state}` only after confirming the retained handoff lineage still belongs to the active lane." + "Move the issue to `{success_state}` or `{in_progress_state}` only after confirming the retained handoff lineage still belongs to the current lane." ) } diff --git a/apps/decodex/src/state/internal.rs b/apps/decodex/src/state/internal.rs index d2198f162..5c397a8de 100644 --- a/apps/decodex/src/state/internal.rs +++ b/apps/decodex/src/state/internal.rs @@ -191,7 +191,7 @@ impl StateData { attempt: &RunAttemptRecord, ) -> Option { let worktree = self.worktrees.get(&attempt.issue_id); - let active_lease = self + let run_lease = self .leases .get(&attempt.issue_id) .is_some_and(|lease| lease.project_id == project_id && lease.run_id == attempt.run_id); @@ -199,7 +199,7 @@ impl StateData { let in_project = remembered_project || worktree.is_some_and(|mapping| mapping.project_id == project_id) - || active_lease; + || run_lease; if !in_project { return None; @@ -228,7 +228,7 @@ impl StateData { updated_at_unix: attempt.updated_at_unix, branch_name: worktree.map(|mapping| mapping.branch_name.clone()), worktree_path: worktree.map(|mapping| mapping.worktree_path.clone()), - active_lease, + run_lease, event_count: event_summary.event_count, last_event_type: event_summary.last_event_type, last_event_at: event_summary.last_event_at, @@ -5380,8 +5380,8 @@ fn derived_program_issue_mapping_records( fn compare_project_run_status(left: &ProjectRunStatus, right: &ProjectRunStatus) -> cmp::Ordering { right - .active_lease - .cmp(&left.active_lease) + .run_lease + .cmp(&left.run_lease) .then_with(|| right.updated_at.cmp(&left.updated_at)) .then_with(|| right.attempt_number.cmp(&left.attempt_number)) .then_with(|| right.run_id.cmp(&left.run_id)) diff --git a/apps/decodex/src/state/models.rs b/apps/decodex/src/state/models.rs index 1ae5a5d85..a7e7f3fe8 100644 --- a/apps/decodex/src/state/models.rs +++ b/apps/decodex/src/state/models.rs @@ -76,7 +76,7 @@ impl RunAttempt { } } -/// Local control capability published by one active run attempt. +/// Local control capability published by one running run attempt. #[derive(Clone, Debug, Eq, PartialEq)] pub struct RunControlChannel { project_id: String, @@ -313,7 +313,7 @@ pub struct ProjectRunStatus { updated_at_unix: i64, branch_name: Option, worktree_path: Option, - active_lease: bool, + run_lease: bool, event_count: i64, last_event_type: Option, last_event_at: Option, @@ -369,8 +369,8 @@ impl ProjectRunStatus { } /// Whether this run still holds the active local lease. - pub fn active_lease(&self) -> bool { - self.active_lease + pub fn run_lease(&self) -> bool { + self.run_lease } /// Number of recorded protocol events for the run. diff --git a/apps/decodex/src/state/store.rs b/apps/decodex/src/state/store.rs index 651438cee..b6f1336c8 100644 --- a/apps/decodex/src/state/store.rs +++ b/apps/decodex/src/state/store.rs @@ -76,6 +76,23 @@ impl ProjectLoopEvidenceSnapshot { .unwrap_or(&[]) } + pub(crate) fn private_events_for_issue(&self, issue_id: &str) -> Vec<&PrivateExecutionEvent> { + let mut events = self + .private_events + .iter() + .filter(|((event_issue_id, _, _), _)| event_issue_id == issue_id) + .flat_map(|(_, events)| events.iter()) + .collect::>(); + + events.sort_by(|left, right| { + left.recorded_at_unix() + .cmp(&right.recorded_at_unix()) + .then_with(|| left.record_id().cmp(&right.record_id())) + }); + + events + } + pub(crate) fn review_policy_checkpoint( &self, issue_id: &str, @@ -374,7 +391,7 @@ impl StateStore { self.retarget_issue_identity_locked(previous_issue_id, canonical_issue_id) } - /// Create or replace the active lease for one issue. + /// Create or replace the run lease for one issue. pub fn upsert_lease( &self, project_id: &str, @@ -516,14 +533,14 @@ impl StateStore { Ok(true) } - /// Read the active lease for one issue. + /// Read the run lease for one issue. pub fn lease_for_issue(&self, issue_id: &str) -> Result> { let state = self.lock()?; Ok(state.leases.get(issue_id).cloned()) } - /// List all active leases. + /// List all run leases. pub fn list_leases(&self, project_id: &str) -> Result> { let mut state = self.lock_without_refresh()?; @@ -669,7 +686,7 @@ impl StateStore { } } - /// Remove the active lease for one issue. + /// Remove the run lease for one issue. pub fn clear_lease(&self, issue_id: &str) -> Result<()> { let mut state = self.lock()?; let _coordinator = match ( @@ -953,7 +970,7 @@ impl StateStore { self.upsert_run_control_channel_locked(&channel) } - /// Resolve a local run-control request against active runtime ownership and audit it. + /// Resolve a local run-control request against run lease ownership and audit it. #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn resolve_run_control_action( &self, @@ -1025,7 +1042,7 @@ impl StateStore { attempt_status: None, branch_name: None, worktree_path: None, - active_lease: None, + run_lease: None, event_count: None, last_event_type: None, last_event_at: None, @@ -1066,7 +1083,7 @@ impl StateStore { attempt_status: None, branch_name: None, worktree_path: None, - active_lease: None, + run_lease: None, event_count: None, last_event_type: None, last_event_at: None, @@ -1116,7 +1133,7 @@ impl StateStore { }, "lane": { "attempt_status": target.attempt_status.as_deref(), - "active_lease": target.active_lease, + "run_lease": target.run_lease, "branch": target.branch_name.as_deref(), "worktree_path": target.worktree_path.as_ref().map(|path| path.display().to_string()), "event_count": target.event_count, @@ -1266,8 +1283,8 @@ impl StateStore { Ok(()) } - /// Mark all active run attempts for one issue as succeeded. - pub fn succeed_active_run_attempts_for_issue(&self, issue_id: &str) -> Result { + /// Mark all running run attempts for one issue as succeeded. + pub fn succeed_running_run_attempts_for_issue(&self, issue_id: &str) -> Result { let now = timestamp_parts(); let mut state = self.lock()?; let mut updated_count = 0; @@ -1276,7 +1293,7 @@ impl StateStore { .run_attempts .values_mut() .filter(|attempt| attempt.issue_id == issue_id) - .filter(|attempt| active_run_attempt_status(&attempt.status)) + .filter(|attempt| running_run_attempt_status(&attempt.status)) { attempt.status = "succeeded".to_owned(); attempt.updated_at = now.text.clone(); @@ -1444,18 +1461,18 @@ impl StateStore { runs.sort_by(compare_project_run_status); - let active_runs = runs + let leased_runs = runs .iter() - .filter(|status| status.active_lease()) + .filter(|status| status.run_lease()) .cloned() .collect::>(); - let recent_limit = base_recent_limit.saturating_add(active_runs.len()); + let recent_limit = base_recent_limit.saturating_add(leased_runs.len()); let recent_run_ids = runs .iter() .take(recent_limit) .map(|run| run.run_id().to_owned()) .collect::>(); - let mut summary_run_ids = active_runs + let mut summary_run_ids = leased_runs .iter() .map(|run| run.run_id().to_owned()) .collect::>(); @@ -1475,20 +1492,20 @@ impl StateStore { selected_runs.sort_by(compare_project_run_status); - let active_runs = selected_runs + let leased_runs = selected_runs .iter() - .filter(|status| status.active_lease()) + .filter(|status| status.run_lease()) .cloned() .collect::>(); let mut recent_runs = selected_runs; recent_runs.truncate(recent_limit); - Ok((active_runs, recent_runs)) + Ok((leased_runs, recent_runs)) } - /// List all active leased runs for one project without applying the recent-run limit. - pub fn list_active_runs(&self, project_id: &str) -> Result> { + /// List all leased run attempts for one project without applying the recent-run limit. + pub fn list_leased_runs(&self, project_id: &str) -> Result> { let mut state = self.lock_without_refresh()?; self.refresh_project_run_metadata_state_locked(&mut state, project_id)?; @@ -1499,7 +1516,7 @@ impl StateStore { .filter_map(|attempt| { let status = state.project_run_status(project_id, attempt)?; - status.active_lease.then_some(status) + status.run_lease.then_some(status) }) .collect::>(); let mut run_ids = runs.iter().map(|run| run.run_id().to_owned()).collect::>(); @@ -1515,7 +1532,7 @@ impl StateStore { .filter_map(|attempt| { let status = state.project_run_status(project_id, attempt)?; - status.active_lease.then_some(status) + status.run_lease.then_some(status) }) .collect::>(); @@ -3025,7 +3042,7 @@ struct RunControlAuditTarget { context: Option, branch_name: Option, worktree_path: Option, - active_lease: Option, + run_lease: Option, event_count: Option, last_event_type: Option, last_event_at: Option, @@ -3158,7 +3175,7 @@ fn retarget_loop_guardrail_issue( } } -fn active_run_attempt_status(status: &str) -> bool { +fn running_run_attempt_status(status: &str) -> bool { matches!(status, "starting" | "running") } @@ -3202,7 +3219,7 @@ fn resolve_run_control_action_locked( worktree_path: project_run_status .as_ref() .and_then(|status| status.worktree_path().map(Path::to_path_buf)), - active_lease: project_run_status.as_ref().map(ProjectRunStatus::active_lease), + run_lease: project_run_status.as_ref().map(ProjectRunStatus::run_lease), event_count: project_run_status.as_ref().map(ProjectRunStatus::event_count), last_event_type: project_run_status .as_ref() @@ -3229,16 +3246,16 @@ fn resolve_run_control_action_locked( } let Some(lease) = state.leases.get(request.issue_id) else { - return rejected_run_control_resolution(request, Some(audit_target), "active_lease_missing"); + return rejected_run_control_resolution(request, Some(audit_target), "run_lease_missing"); }; if lease.project_id != request.project_id { return rejected_run_control_resolution(request, Some(audit_target), "project_mismatch"); } if lease.run_id != request.run_id { - return rejected_run_control_resolution(request, Some(audit_target), "active_run_mismatch"); + return rejected_run_control_resolution(request, Some(audit_target), "run_lease_mismatch"); } - if !active_run_attempt_status(&attempt.status) { + if !running_run_attempt_status(&attempt.status) { return rejected_run_control_resolution(request, Some(audit_target), "run_not_active"); } @@ -3275,7 +3292,7 @@ fn resolve_run_control_action_locked( RunControlActionResolution { audit_target, outcome: RUN_CONTROL_ACTION_ACCEPTED.to_owned(), - reason: String::from("active_run_control_channel_resolved"), + reason: String::from("run_lease_control_channel_resolved"), channel: Some(channel), } } @@ -3304,7 +3321,7 @@ fn rejected_run_control_resolution( context: request.context.cloned(), branch_name: None, worktree_path: None, - active_lease: None, + run_lease: None, event_count: None, last_event_type: None, last_event_at: None, diff --git a/apps/decodex/src/state/tests.rs b/apps/decodex/src/state/tests.rs index b5bf03051..6130cdc24 100644 --- a/apps/decodex/src/state/tests.rs +++ b/apps/decodex/src/state/tests.rs @@ -751,7 +751,7 @@ fn persistent_project_run_listing_does_not_refresh_full_event_journal() { .record_linear_execution_event(&writer_record) .expect("writer ledger event should persist"); - let runs = observer.list_active_runs("pubfi").expect("active runs should load"); + let runs = observer.list_leased_runs("pubfi").expect("leased runs should load"); let recent_runs = observer.list_recent_runs("pubfi", 10).expect("recent runs should load"); let leases = observer.list_active_shared_leases("pubfi").expect("shared leases should load"); let worktrees = observer.list_worktrees("pubfi").expect("worktrees should load"); @@ -2695,7 +2695,7 @@ fn lists_recent_project_runs_with_protocol_summary() { .expect("older run attempt should be recorded"); store .record_run_attempt("run-1", "PUB-101", 1, "running") - .expect("active run attempt should be recorded"); + .expect("running run attempt should be recorded"); store.update_run_thread("run-1", "thread-1").expect("thread id should attach"); store .upsert_lease("pubfi", "PUB-101", "run-1", IN_PROGRESS_STATE) @@ -2717,14 +2717,14 @@ fn lists_recent_project_runs_with_protocol_summary() { assert_eq!(runs.len(), 2); assert_eq!(runs[0].run_id(), "run-1"); - assert!(runs[0].active_lease()); + assert!(runs[0].run_lease()); assert_eq!(runs[0].thread_id(), Some("thread-1")); assert_eq!(runs[0].event_count(), 2); assert_eq!(runs[0].last_event_type(), Some("turn/completed")); assert_eq!(runs[0].branch_name(), Some("x/pubfi-pub-101")); assert_eq!(runs[0].worktree_path(), Some(Path::new("/tmp/worktrees/pub-101"))); assert_eq!(runs[1].run_id(), "run-2"); - assert!(!runs[1].active_lease()); + assert!(!runs[1].run_lease()); assert_eq!(runs[1].event_count(), 0); } @@ -2742,7 +2742,7 @@ fn lists_recent_project_runs_after_terminal_lane_cleanup() { .upsert_worktree("pubfi", "PUB-101", "x/pubfi-pub-101", "/tmp/worktrees/pub-101") .expect("worktree should record project ownership"); store.update_run_status("run-1", "succeeded").expect("terminal status should update"); - store.clear_lease("PUB-101").expect("terminal cleanup should clear active lease"); + store.clear_lease("PUB-101").expect("terminal cleanup should clear run lease"); store.clear_worktree("PUB-101").expect("terminal cleanup should clear worktree mapping"); let runs = store.list_recent_runs("pubfi", 10).expect("recent project runs should load"); @@ -2750,7 +2750,7 @@ fn lists_recent_project_runs_after_terminal_lane_cleanup() { assert_eq!(runs.len(), 1); assert_eq!(runs[0].run_id(), "run-1"); assert_eq!(runs[0].status(), "succeeded"); - assert!(!runs[0].active_lease()); + assert!(!runs[0].run_lease()); assert_eq!(runs[0].branch_name(), None); assert_eq!(runs[0].worktree_path(), None); assert!( @@ -2778,11 +2778,11 @@ fn lists_active_project_runs_only() { .upsert_worktree("other", "PUB-102", "x/other-pub-102", "/tmp/worktrees/pub-102") .expect("second worktree should record"); - let runs = store.list_active_runs("pubfi").expect("active project runs should load"); + let runs = store.list_leased_runs("pubfi").expect("active project runs should load"); assert_eq!(runs.len(), 1); assert_eq!(runs[0].run_id(), "run-1"); - assert!(runs[0].active_lease()); + assert!(runs[0].run_lease()); } #[test] @@ -2835,7 +2835,7 @@ fn state_store_open_persists_runtime_history_across_instances() { assert_eq!(second.event_count("run-1").expect("event count should load"), 1); assert!( second.lease_for_issue("PUB-101").expect("lease lookup should succeed").is_some(), - "persistent store should recover active leases" + "persistent store should recover run leases" ); assert!( second.worktree_for_issue("PUB-101").expect("worktree lookup should succeed").is_some(), @@ -3526,7 +3526,7 @@ fn run_control_accepts_active_attempt_and_persists_audit() { .expect("control request should resolve"); assert_eq!(receipt.outcome(), "accepted"); - assert_eq!(receipt.reason(), "active_run_control_channel_resolved"); + assert_eq!(receipt.reason(), "run_lease_control_channel_resolved"); assert!(receipt.channel().is_some()); for (outcome, reason) in [ @@ -3673,7 +3673,7 @@ fn run_control_rejects_missing_channel_file() { } #[test] -fn run_control_requires_active_run_ownership() { +fn run_control_requires_run_lease_ownership() { let temp_dir = TempDir::new().expect("tempdir should create"); let channel_path = temp_dir.path().join("control.channel"); let worktree_path = temp_dir.path().join("PUB-101"); @@ -3733,12 +3733,12 @@ fn run_control_requires_active_run_ownership() { metadata: None, context: None, }) - .expect("wrong active run should be audited"); + .expect("wrong run lease should be audited"); assert_eq!(no_lease.outcome(), "rejected"); - assert_eq!(no_lease.reason(), "active_lease_missing"); + assert_eq!(no_lease.reason(), "run_lease_missing"); assert_eq!(wrong_run.outcome(), "rejected"); - assert_eq!(wrong_run.reason(), "active_run_mismatch"); + assert_eq!(wrong_run.reason(), "run_lease_mismatch"); let events = store .list_private_execution_events("pubfi", "issue-1", "run-1", 1) @@ -3750,7 +3750,7 @@ fn run_control_requires_active_run_ownership() { let expected_worktree_path = worktree_path.display().to_string(); let expected_channel_path = channel_path.display().to_string(); - assert_eq!(no_lease_event.payload()["lane"]["active_lease"].as_bool(), Some(false)); + assert_eq!(no_lease_event.payload()["lane"]["run_lease"].as_bool(), Some(false)); assert_eq!(no_lease_event.payload()["lane"]["attempt_status"].as_str(), Some("running")); assert_eq!(no_lease_event.payload()["lane"]["branch"].as_str(), Some("x/pubfi-issue-1")); assert_eq!( diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 02d45b8da..5ebf9a76a 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -121,7 +121,7 @@ curl -sS -X POST http://127.0.0.1:8192/api/lane/interrupt \ app-server child to deliver with `turn/interrupt`. Add `"force": true` only when the operator explicitly wants hard process-kill fallback after soft interrupt is unavailable, does not return in the local wait window, or is rejected with -`active_lease_missing` while the same lane still has recorded live process evidence. +`run_lease_missing` while the same lane still has recorded live process evidence. Hard fallback is reported as `hard_interrupt_fallback`, not as a graceful stop. If no signalable child process is recorded, the force response remains a recovery/inspection result and must not be read as a successful soft interrupt. @@ -207,7 +207,7 @@ pins all new runs to an exhausted fixed account. | Surface | Owns | Does Not Own | | --- | --- | --- | -| Runtime SQLite DB | active leases, attempts, run-control channels, protocol events, private execution events, Decision Contracts, Program Intake Plans, internal Execution Programs, dispatch readiness, worktree mappings, retry state, retained PR state, review-policy checkpoints with structured independent-review detail, loop-guardrail checkpoints, phase-goal signals, phase timing, connector backoff, project registry | human backlog grooming or durable team-visible issue history | +| Runtime SQLite DB | run leases, attempts, run-control channels, protocol events, private execution events, Decision Contracts, Program Intake Plans, internal Execution Programs, dispatch readiness, worktree mappings, retry state, retained PR state, review-policy checkpoints with structured independent-review detail, loop-guardrail checkpoints, phase-goal signals, phase timing, connector backoff, project registry | human backlog grooming or durable team-visible issue history | | Central project config | `service_id`, repo root, worktree root, tracker/GitHub credential env-var names, enabled project registration | per-run state or issue ownership | | Project `WORKFLOW.md` | repo policy, validation gate, state names, retry/review policy | runtime ownership, queue labels, credentials, model overrides | | Linear | team-visible issue state, queue/active/manual-attention labels, coarse execution ledger comments, progress/failure/handoff/closeout summaries | high-frequency runtime truth, heartbeat, token pressure, raw attempts, private execution evidence, connector retry budgets | @@ -228,7 +228,7 @@ snapshot so an operator can catch stale tabs or an old listener port quickly. Because the runtime SQLite DB is authoritative, dashboard sections describe current runtime ownership before local directory presence. An existing `.worktrees/XY-*` directory does not, by itself, mean an active lane is still running; the owning -section says whether the path belongs to an active lease, retained review/landing +section says whether the path belongs to a run lease, retained review/landing lane, queued attention state, or cleanup/recovery inbox. `Projects` is its own dashboard section. It renders a single fleet table for this local @@ -250,7 +250,7 @@ service queue label before treating it as a connector problem. The browser dashboard and Decodex App read the complete published operator state from the local `GET /dashboard/control` WebSocket. That socket is the dashboard/App -authority for published snapshots, active-lane activity updates, and local dashboard +authority for published snapshots, current-lane activity updates, and local dashboard control acknowledgements. The App still uses the same base URL's `/api/accounts` HTTP surface for account-pool rows and actions. `GET /api/operator-snapshot` is a status/cache read path over the same runtime database, not a browser-dashboard or App @@ -276,31 +276,31 @@ diagnostics use `dependency_not_terminal` with a next action to complete the dependency issue or refresh the Execution Program dependency plan when a stale dependency program is the real blocker. -For the lane-control rollout, active-lane UI posture is observe-only. The dashboard -renders active-lane state, protocol activity, liveness, private-evidence references, +For the lane-control rollout, current-lane UI posture is observe-only. The dashboard +renders current-lane state, protocol activity, liveness, private-evidence references, local run-control capability metadata, and local acknowledgement/account controls, but it is not the supported place to author steer, retry, task replacement, or lifecycle mutations. CLI/API is the first operator-control surface for lane control, governed by [`../spec/lane-control.md`](../spec/lane-control.md). The browser UI does not show or -accept active-lane stop/interrupt controls, project pause/resume controls, manual retry -controls, or active-lane steer controls; use `decodex lane inspect`, `decodex lane +accept current-lane stop/interrupt controls, project pause/resume controls, manual retry +controls, or current-lane steer controls; use `decodex lane inspect`, `decodex lane interrupt`, or the local `/api/lane/*` endpoints instead. Account-pool selection remains available -because it changes the global Codex account selector, not an active lane. +because it changes the global Codex account selector, not a current lane. Active-lane steer is available through `decodex lane steer --run-id --expected-turn-id --message `, canonical `POST /api/lane/steer`, and legacy alias `POST /api/lane-steer`. These surfaces require the expected active turn id, audit accepted or rejected state locally, and keep raw steer text out of public tracker projections. -`runActivity.activeRunsComplete` -marks whether a payload is the complete active-run list; subscription-filtered +`runActivity.currentLanesComplete` +marks whether a payload is the complete current-lane list; subscription-filtered payloads set it to `false`, so consumers must not treat a missing run in that payload as ended. -Active run rows may include `control_capability` with the active attempt's project, +Current lane rows may include `control_capability` with the active attempt's project, issue, run id, attempt, current thread/turn ids, local transport, channel path, status, and timestamps. It is local routing metadata for CLI/API controls, not a dashboard command surface. -After a steer request is handled, active run protocol activity may show a compact +After a steer request is handled, current lane protocol activity may show a compact `turn/steer` entry with outcome, failure class, and response turn id. It does not include the operator message. Snapshot `warnings` remain stable machine-readable tokens. When a warning needs @@ -387,7 +387,7 @@ runtime evidence unless explicitly read through the local evidence command. Worktree visibility follows the owning dashboard section: -- `Running Lanes` means the runtime DB still has an active lease, active attempt, or +- `Running Lanes` means the runtime DB still has a run lease, active attempt, or child process/thread/protocol relationship for the path. Process liveness requires an alive PID plus matching `.decodex-run-activity` `host_boot_id` and `process_start_identity`; a previous-boot marker, same-boot PID reuse, missing @@ -396,15 +396,15 @@ Worktree visibility follows the owning dashboard section: process_identity_mismatch` is the stable summary for previous-boot or PID-reuse evidence, while `process_liveness_reason` explains the exact failed identity check when `process_alive` is false. - `active_lease` is queue lease ownership only; `execution_liveness` explains why + `run_lease` is queue lease ownership only; `execution_liveness` explains why the lane is still visible when the queue lease is not held. -- Lane steer and interrupt rejections such as `active_lease_missing` are private +- Lane steer and interrupt rejections such as `run_lease_missing` are private runtime evidence. They should preserve the queue lease state, branch, retained worktree path, current run id and attempt, active channel metadata, and observed process/protocol liveness so operators can decide between wait, force interrupt, retained resume, or manual attention without reconstructing state from local paths. -- In the JSON snapshot, `active_run_count` follows the same visibility boundary as - the top-level `active_runs` list. The `Projects` table's `running` work number uses +- In the JSON snapshot, `current_lane_count` follows the same visibility boundary as + the top-level `current_lanes` list. The `Projects` table's `running` work number uses `running_lane_count`, so stopped, stale, or attention lanes can stay visible as active work without being counted as currently running. - Running lanes derive CLI and dashboard text from the same `OperatorRunStatus` @@ -491,7 +491,7 @@ Worktree visibility follows the owning dashboard section: not clear it as a transient queue delay. - `linear_active_label_present` in `Intake Queue` means the issue still carries service active ownership while it is also queued, but local status could not prove a - matching active lease. Treat it as a recovery/attention row, not ready work. If its + matching run lease. Treat it as a recovery/attention row, not ready work. If its attention cause is `evidence_missing`, use the retained marker, worktree, and public Linear state as the available recovery evidence before retrying or cleaning labels. - `Recovery Worktrees` means the path is retained local state after the authoritative @@ -517,7 +517,7 @@ Worktree visibility follows the owning dashboard section: supersedes it. Every operator snapshot worktree row includes `ownership`, `ownership_reason`, -`provenance`, and optional `recovery_next_action` fields that distinguish active-lane +`provenance`, and optional `recovery_next_action` fields that distinguish current-lane ownership, post-review ownership, queued attention, retained attention, post-land cleanup, and cleanup-only local retention. Runtime-recorded mappings report `provenance.source = @@ -531,12 +531,12 @@ report `provenance.source = "legacy_unknown"` and may set A `Recovery Worktrees` row tells the operator to inspect the local path and either clean it up or recover local-only changes; it is not, by itself, evidence that the -SQLite runtime store lost an active lane. When the tracker issue is already `Done`, +SQLite runtime store lost a current lane. When the tracker issue is already `Done`, the row has runtime provenance, and no retained lane owns the worktree, the row is neutral cleanup-only state, not a blocking recovery error. When a retained worktree reports `role: cleanup_only`, treat it as local cleanup -hygiene rather than an active lane. It does not imply that an agent, child +hygiene rather than a current lane. It does not imply that an agent, child process, post-review repair, closeout, or queued recovery run is still executing, and it is not queue pressure or a hidden capacity claim. The row only says local disk still has a retained checkout after the runtime owner is gone; once the diff --git a/docs/reference/test-suite.md b/docs/reference/test-suite.md index e511a8e29..350cfedeb 100644 --- a/docs/reference/test-suite.md +++ b/docs/reference/test-suite.md @@ -82,7 +82,7 @@ large catch-all test file unless the behavior crosses several of these stages. | `apps/decodex/src/orchestrator/tests/recovery/runtime_reentry.rs` | 30 | Runtime reentry, recovered worktrees, liveness, and live-run recovery | | `apps/decodex/src/orchestrator/tests/operator/status_support.rs` | 0 | Shared operator status fixtures | | `apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs` | 10 | Registered project control-plane rows | -| `apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs` | 34 | Running lanes, stalled lanes, active-run hydration, and local worktrees | +| `apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs` | 34 | Running lanes, stalled lanes, current-lane hydration, and local worktrees | | `apps/decodex/src/orchestrator/tests/operator/status/history.rs` | 7 | Run ledger and Linear history hydration | | `apps/decodex/src/orchestrator/tests/operator/status/text.rs` | 10 | Human-readable operator status text | | `apps/decodex/src/orchestrator/tests/operator/status/publishing.rs` | 11 | Snapshot publishing, degraded observers, and tracker backoff | diff --git a/docs/reference/workspace-layout.md b/docs/reference/workspace-layout.md index f515fa9de..5a1048cfc 100644 --- a/docs/reference/workspace-layout.md +++ b/docs/reference/workspace-layout.md @@ -119,7 +119,7 @@ Runtime state that belongs to the local operator, not to this repository, lives `~/.codex/decodex/`: - `runtime.sqlite3` is the single-machine control-plane database for all registered - projects. It owns active leases, attempts, private execution events, tracker/PR + projects. It owns run leases, attempts, private execution events, tracker/PR caches, retained PR state, retry state, and project registration. - `agent-evidence//` stores local agent-readable diagnosis artifacts, including `handoff-index.json`, `events.jsonl`, `blockers/*.json`, and diff --git a/docs/runbook/lane-control-recovery.md b/docs/runbook/lane-control-recovery.md index 7ecbc6e72..03c7f1e8b 100644 --- a/docs/runbook/lane-control-recovery.md +++ b/docs/runbook/lane-control-recovery.md @@ -65,7 +65,7 @@ Before mutating anything, confirm: - run id, attempt, thread id, current turn id, and process/protocol liveness - control outcome such as accepted, rejected, timed out, failed, or `hard_interrupt_fallback` -- `active_lease_missing` rejections together with process, protocol, channel, branch, +- `run_lease_missing` rejections together with process, protocol, channel, branch, and retained worktree evidence when the lane still appears live - private evidence and public lifecycle signal - latest `authority_boundary_check` private event when guardrail pressure, broad @@ -82,8 +82,8 @@ or clean labels. | --- | --- | --- | | Active lane still matches the issue, branch, run id, attempt, and turn. | Let the runtime continue or wait for the control result. | No label change. Use the next CLI/API control only when the operator explicitly asks. | | Soft interrupt was accepted and the runtime is still resolving the attempt. | Wait for status, protocol activity, or evidence to settle. | Re-inspect; do not requeue or force-kill. | -| Soft control was rejected with `active_lease_missing`, but inspect/status still shows the same run id, attempt, branch, active channel, and live child process or protocol activity. | Treat the lane as degraded active execution, not cleanup-only state. | Re-inspect with `decodex lane inspect` or use `decodex lane interrupt --run-id --force` only when the operator explicitly wants hard process fallback. | -| Forced interrupt after `active_lease_missing` reports no signalable process. | Treat force as non-mutating for the child process. | Inspect retained worktree and private evidence; do not claim the interrupt succeeded or clear attention labels. | +| Soft control was rejected with `run_lease_missing`, but inspect/status still shows the same run id, attempt, branch, active channel, and live child process or protocol activity. | Treat the lane as degraded active execution, not cleanup-only state. | Re-inspect with `decodex lane inspect` or use `decodex lane interrupt --run-id --force` only when the operator explicitly wants hard process fallback. | +| Forced interrupt after `run_lease_missing` reports no signalable process. | Treat force as non-mutating for the child process. | Inspect retained worktree and private evidence; do not claim the interrupt succeeded or clear attention labels. | | Hard fallback reports `hard_interrupt_fallback`. | Treat it as an interrupted runtime event, not a graceful completion. | Inspect retained worktree and evidence; resume only if lineage is exact. | | Retained worktree has useful local changes and lineage matches issue, branch, runtime evidence, and PR when present. | Resume or repair the same lane. | Use `decodex run ` when the registered workflow makes it eligible, or use the specific retained recovery runbook. | | Review handoff marker is missing or stale but the retained PR lane appears recoverable. | Diagnose before rebind. | Run `decodex recover review-handoff diagnose ` and follow [`recover-review-handoff.md`](./recover-review-handoff.md). | @@ -179,7 +179,7 @@ not kill the child process from the side. Example: `decodex lane steer XY-123 --run-id run-abc --expected-turn-id turn-1 ...` or `decodex lane interrupt XY-123 --run-id run-abc` reports -`active_lease_missing` while `decodex lane inspect` still shows the same branch, +`run_lease_missing` while `decodex lane inspect` still shows the same branch, attempt, active channel, and live process/protocol state. Do not treat the lane as cleanup-only and do not clear `decodex:needs-attention`. If the operator needs to stop the child immediately, retry interrupt with `--force`; otherwise preserve the retained diff --git a/docs/runbook/self-dogfood-pilot.md b/docs/runbook/self-dogfood-pilot.md index a5aceb216..59a1e8caa 100644 --- a/docs/runbook/self-dogfood-pilot.md +++ b/docs/runbook/self-dogfood-pilot.md @@ -92,7 +92,7 @@ project directory: - `decodex status --json` or the operator UI should show no project backed by a flat `*.toml` config path inside a checkout or lane worktree. -Runtime state now lives in the Decodex-owned SQLite database at `~/.codex/decodex/runtime.sqlite3`, and logs live under `~/.codex/decodex/logs/`. On restart, `decodex` reloads retained worktree knowledge and active-lane recovery intent from that database, then refreshes low-frequency Linear and GitHub state as connector budgets allow. +Runtime state now lives in the Decodex-owned SQLite database at `~/.codex/decodex/runtime.sqlite3`, and logs live under `~/.codex/decodex/logs/`. On restart, `decodex` reloads retained worktree knowledge and current-lane recovery intent from that database, then refreshes low-frequency Linear and GitHub state as connector budgets allow. That recovery is still scoped by configured `service_id`, so reconciliation and cleanup stay within the single service instance represented by the registered project config. @@ -579,7 +579,7 @@ Decodex stores private runtime evidence in one SQLite database owned by the loca Decodex installation: - registered projects and config fingerprints -- active leases, dispatch slots, run attempts, retry schedules, protocol events, and +- run leases, dispatch slots, run attempts, retry schedules, protocol events, and local `Run Ledger` attempt rows - private execution events for full checkpoint payloads, verification notes, local head evidence, and recovery details scoped by project, issue, run, and attempt @@ -652,7 +652,7 @@ ordinary `decodex serve` and leaves scheduler cadence to CLI-owned defaults. Dev not a scheduler and must not be used for this runbook's automation, queue intake, project registration, or retained-lane recovery steps. -The listener serves the operator console from the canonical `GET /` and `GET /dashboard` routes, the same JSON operator snapshot used by `cargo run -p decodex --bin decodex -- status --json` through the `/dashboard/control` WebSocket, and the minimal `GET /livez` liveness probe on the same listener. The single console keeps `Projects`, `Running Lanes`, `Intake Queue`, `Review & Landing`, `Recovery Worktrees`, and `Run Ledger` visible together. Intake candidates that are already claimed by a running lane are shown as active queue echoes, capacity-bound candidates are shown as waiting rather than blocked, running lane worktrees stay with their owning lane, and retained/recovery worktrees remain folded until diagnostics are needed: +The listener serves the operator console from the canonical `GET /` and `GET /dashboard` routes, the same JSON operator snapshot used by `cargo run -p decodex --bin decodex -- status --json` through the `/dashboard/control` WebSocket, and the minimal `GET /livez` liveness probe on the same listener. The single console keeps `Projects`, `Running Lanes`, `Intake Queue`, `Review & Landing`, `Recovery Worktrees`, and `Run Ledger` visible together. Intake candidates that are already claimed by a running lane are shown as claimed queue echoes, capacity-bound candidates are shown as waiting rather than blocked, running lane worktrees stay with their owning lane, and retained/recovery worktrees remain folded until diagnostics are needed: - `GET /` or `GET /dashboard`: the same single-page operator console - `GET /dashboard/control`: WebSocket transport for snapshots, live run activity, and local dashboard control acknowledgements diff --git a/docs/spec/agent-evidence.md b/docs/spec/agent-evidence.md index beb4fd2da..a07d46b27 100644 --- a/docs/spec/agent-evidence.md +++ b/docs/spec/agent-evidence.md @@ -86,7 +86,7 @@ Required fields: - `source`: `diagnose_command` or `serve_tick` - `evidence_root`, `handoff_index_path`, `blockers_dir`, `runs_dir`, `events_path`: absolute local paths -- `summary`: counts for projects, active runs, recent runs, history lanes, queued +- `summary`: counts for projects, current lanes, recent runs, history lanes, queued candidates, post-review lanes, recovery worktrees, blockers, run capsules, connector backoffs, and warnings - `warnings`: typed operator snapshot or diagnose warning strings diff --git a/docs/spec/lane-control-state.md b/docs/spec/lane-control-state.md index f38d9b77e..628ee48f3 100644 --- a/docs/spec/lane-control-state.md +++ b/docs/spec/lane-control-state.md @@ -3,7 +3,7 @@ Purpose: Define the authoritative Decodex lane-control state model used by scheduler decisions, policy guards, terminal cleanup, and operator projections. Status: normative -Read this when: You are implementing or validating lane scheduling, active-run +Read this when: You are implementing or validating lane scheduling, current-lane projection, review-policy stops, retained recovery, closeout, or dashboard status. Not this document: The operator command sequence for steering or interrupting a lane. Use [`lane-control.md`](./lane-control.md) for CLI/API controls and @@ -31,7 +31,7 @@ from protocol activity. `ownership_state` values: - `pending`: eligible or waiting before an owned attempt starts. -- `owned_active`: Decodex owns the active lease and the active attempt may mutate the +- `leased_run`: Decodex owns the run lease and the active attempt may mutate the lane. - `terminalizing`: Decodex is retiring run control, finishing writeback, archiving the app-server thread, or cleaning up an owned attempt. @@ -74,10 +74,10 @@ from protocol activity. ## Invariants -- A lane counts as a running lane only when `ownership_state` is `owned_active`. +- A lane counts as a running lane only when `ownership_state` is `leased_run`. - Liveness evidence may update `liveness_state`, but it must not create or restore - `owned_active` ownership. -- `active_lease=false` is incompatible with `ownership_state=owned_active`. + `leased_run` ownership. +- `run_lease=false` is incompatible with `ownership_state=leased_run`. - Terminal attempt statuses such as `failed`, `interrupted`, `stalled`, or `succeeded` must not be promoted to `running` by live process, thread, or protocol evidence. - `policy_state=review_churn_exceeded` blocks further review-repair mutation for the @@ -119,8 +119,8 @@ projection renders it. ## Projection Rules Operator status and dashboard views must show the four state axes and `next_action`. -Legacy fields such as `status`, `phase`, and `execution_liveness` may remain for -compatibility, but they must not be used to infer ownership in new scheduler code. +Raw observation fields such as `status`, `phase`, and `execution_liveness` are +diagnostics; scheduler code must not infer ownership from them. Examples: @@ -141,7 +141,7 @@ next_action: inspect_recovery_evidence ``` ```text -ownership_state: owned_active +ownership_state: leased_run liveness_state: process_alive policy_state: review_findings terminalization_state: none diff --git a/docs/spec/lane-control.md b/docs/spec/lane-control.md index db2ec35cd..f4547abe3 100644 --- a/docs/spec/lane-control.md +++ b/docs/spec/lane-control.md @@ -45,9 +45,9 @@ agent-facing skills must guide responsible use. | Project dispatch pause | Supported for future dispatch | `decodex project disable ` and the runtime project enabled flag | Pause prevents new dispatch for the project. It must not kill or rewrite already active lanes. | | Project dispatch resume | Supported for future dispatch | `decodex project enable ` and the runtime project enabled flag | Resume re-enables future dispatch after the operator has inspected blockers, capacity, and queue state. | | Linear scan request | Supported | `POST /api/linear-scan` with optional `projectId` | Queue a scan for the next control-plane tick while respecting tracker backoff. This is an intake/status refresh request, not an execution command. | -| Run-control channel foundation | Supported foundation | Active attempts publish a local `.decodex-run-control/*` channel record, runtime SQLite `run_control_channels`, operator status `control_capability`, and private `control_action` audit events | Route lane-control mutations through the active attempt's project, issue, run id, attempt, thread id, current turn id, active lease, and local channel metadata. Invalid or stale requests fail closed and remain local audit evidence. | -| Soft interrupt | Supported CLI/API control | `decodex lane interrupt --run-id ` and `POST /api/lane/interrupt` write a run-control request that the active app-server child delivers with `turn/interrupt` | Prefer soft interrupt before hard interruption when the active turn id is known and the app-server capability is present. Soft interrupt requests a graceful turn stop and records the protocol outcome when app-server returns one. If the run-control resolver rejects soft delivery with `active_lease_missing`, preserve the observed process, channel, branch, and retained worktree evidence instead of hiding the lane as inactive. | -| Hard interrupt fallback | Explicit fallback only | `decodex lane interrupt --run-id --force` and `POST /api/lane/interrupt` with `"force": true` classify process signaling as `hard_interrupt_fallback` | Use only when soft interrupt is unavailable, timed out, or impossible because the process, app-server boundary, or active lease cannot be reached. A forced interrupt may signal the recorded child process after `active_lease_missing` only when inspection still identifies the same issue, run id, attempt, channel, and live process. Preserve retained worktree evidence and runtime classification. | +| Run-control channel foundation | Supported foundation | Active attempts publish a local `.decodex-run-control/*` channel record, runtime SQLite `run_control_channels`, operator status `control_capability`, and private `control_action` audit events | Route lane-control mutations through the active attempt's project, issue, run id, attempt, thread id, current turn id, run lease, and local channel metadata. Invalid or stale requests fail closed and remain local audit evidence. | +| Soft interrupt | Supported CLI/API control | `decodex lane interrupt --run-id ` and `POST /api/lane/interrupt` write a run-control request that the active app-server child delivers with `turn/interrupt` | Prefer soft interrupt before hard interruption when the active turn id is known and the app-server capability is present. Soft interrupt requests a graceful turn stop and records the protocol outcome when app-server returns one. If the run-control resolver rejects soft delivery with `run_lease_missing`, preserve the observed process, channel, branch, and retained worktree evidence instead of hiding the lane as inactive. | +| Hard interrupt fallback | Explicit fallback only | `decodex lane interrupt --run-id --force` and `POST /api/lane/interrupt` with `"force": true` classify process signaling as `hard_interrupt_fallback` | Use only when soft interrupt is unavailable, timed out, or impossible because the process, app-server boundary, or run lease cannot be reached. A forced interrupt may signal the recorded child process after `run_lease_missing` only when inspection still identifies the same issue, run id, attempt, channel, and live process. Preserve retained worktree evidence and runtime classification. | | Steer active lane | Supported CLI/API control; bottom-layer method stays broad | `decodex lane steer --run-id --expected-turn-id --message `, canonical `POST /api/lane/steer`, legacy alias `POST /api/lane-steer`, local run-control steer request/response files, app-server `turn/steer`, private `control_action` audit events, and protocol activity `turn/steer` summaries | Pass operator-supplied steer text through CLI/API to the current active turn. Require `expectedTurnId`; stale turn ids fail closed. Do not narrow the protocol to a fixed set of task-content categories. Apply policy, audit, privacy, and lifecycle guardrails above the protocol. | | Retained resume/retry | Supported through runtime lifecycle | `decodex run `, retry scheduling, retained worktree recovery, and `thread/resume` for same-thread app-server continuation | Resume only when retained worktree, issue, branch, PR, and runtime evidence still prove the same lane. Treat ambiguous lineage as manual attention. | | Manual attention | Supported terminal control path | `issue_label_add` intent for `decodex:needs-attention`, `issue_comment(kind = "manual_attention")`, and `issue_terminal_finalize(path = "manual_attention")` | Stop automation when policy requires a human decision. Explain the blocker through structured public fields and keep private evidence local. | @@ -63,7 +63,7 @@ Before any lane-control mutation, the operator or agent must inspect: - issue identifier and tracker state - branch and retained worktree ownership - run id, attempt number, thread id, and current turn id when available -- active lease state, process liveness, and protocol activity +- run lease state, process liveness, and protocol activity - recent private evidence and any public Linear lifecycle signal - PR lineage when the lane already crossed into review handoff @@ -74,7 +74,7 @@ guessing. ## Run-Control Channel Foundation Every live app-server attempt publishes a per-attempt local control capability when -Decodex still owns the active lease for the run. The current mechanism is a local file +Decodex still owns the run lease for the run. The current mechanism is a local file channel under the run worktree's `.decodex-run-control/` directory plus a `run_control_channels` runtime SQLite row. This is foundation plumbing only: it proves where an active attempt can receive future control traffic without exposing steer, @@ -92,14 +92,14 @@ A control request is valid only when all of the following hold: - the requested run exists - requested project id, issue id, run id, and attempt number match the active attempt - requested thread id and turn id, when supplied, match the current attempt values -- the active lease for the issue is held by the same project and run id +- the run lease for the issue is held by the same project and run id - the attempt status is active - the persisted channel row is active and the local channel path still exists Any mismatch fails closed. Rejections are not converted into process signals, tracker state changes, or worktree mutations. -`active_lease_missing` is a soft-control rejection, not proof that execution is gone. +`run_lease_missing` is a soft-control rejection, not proof that execution is gone. The audit payload must retain the requested project, issue, run id, attempt, current thread and turn ids, active channel metadata when present, branch, retained worktree mapping, and operator-local process/protocol liveness context. If the operator used @@ -143,7 +143,7 @@ evidence, marks an active attempt as `interrupted` when a recorded child is sign clears or retains ownership according to the runtime contract, and avoids pretending the agent completed a terminal path. -When forced interrupt follows an `active_lease_missing` soft-control rejection, the +When forced interrupt follows an `run_lease_missing` soft-control rejection, the CLI/API response must show both facts: soft control was rejected, and hard fallback was attempted only because force was explicit and process evidence was still present. If no signalable child process is recorded, the hard-fallback report must say it was diff --git a/docs/spec/linear-execution-ledger.md b/docs/spec/linear-execution-ledger.md index aca3eabca..f56d01bb4 100644 --- a/docs/spec/linear-execution-ledger.md +++ b/docs/spec/linear-execution-ledger.md @@ -399,7 +399,7 @@ Operator status and dashboard consumers may aggregate ledger records for complet history lanes that are already present in the local runtime attempt window. That aggregation is display-only: it may show PR URL, landed or merge commit, closeout status, needs-attention reason, and elapsed lifecycle timing from Linear comments, but -it must not replay those comments as active leases, dispatch ownership, retry state, or +it must not replay those comments as run leases, dispatch ownership, retry state, or post-review orchestration authority. Successful closeout and cleanup results must remain successful in local runtime history. diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 2885d68a5..ab66271a5 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -44,7 +44,7 @@ state or this state machine. ## Source of truth boundaries -- The Decodex runtime SQLite database is the single-machine source of truth for active leases, attempts, run-control channels, protocol events, private execution events, harness-outcome telemetry, latent and promoted Decision Contracts, internal Execution Programs, worktree mappings, retained PR state, review-policy checkpoints, loop-guardrail checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. +- The Decodex runtime SQLite database is the single-machine source of truth for run leases, attempts, run-control channels, protocol events, private execution events, harness-outcome telemetry, latent and promoted Decision Contracts, internal Execution Programs, worktree mappings, retained PR state, review-policy checkpoints, loop-guardrail checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. - Linear remains the team-visible tracker surface for issue lifecycle, queue/active/manual-attention labels, and coarse lifecycle summaries such as start, PR-ready, blocked, failed, landed, and done. - Versioned Linear execution event comments use the schema in [`linear-execution-ledger.md`](./linear-execution-ledger.md), but fine-grained runtime truth must not be rebuilt from comments every tick. @@ -79,7 +79,7 @@ state or this state machine. `--apply` writes only local runtime Program Intake and Execution Program state. - Each scheduler pass evaluates persisted Execution Programs before ordinary queued issue selection. The Program scheduler refreshes mapped Linear issue state, - dependency observations, local active leases, retained review/landing worktrees, + dependency observations, local run leases, retained review/landing worktrees, needs-attention labels, and occupied conflict domains; then it directly selects dispatchable ready nodes with `program` dispatch mode. It must not apply or remove service queue labels, and Program readiness must not wait for the ordinary @@ -95,7 +95,7 @@ mirror: | Runtime SQLite `execution_programs` | Versioned `decodex.execution_program/1` payloads with embedded or linked `decodex.program_intake_plan/1` planning data. They hold internal node lifecycle/readiness, dependency, conflict-domain, dispatch intent, drift, and normal-issue mapping; Linear issue descriptions and ledger comments are only coarse projections. | | Runtime SQLite `program_intake_plans` | Queryable local projection of `decodex.program_intake_plan/1` metadata, including intake kind, source contract when present, authority fingerprint, and public-safe summary. | | Runtime SQLite `program_issue_mappings` | Queryable local projection of each internal program node's mapped Linear issue, tracker state, dispatch intent, active/manual/attention facts, and dispatch-briefing fact. | -| Runtime SQLite `run_control_channels` | Local control capability metadata for active run attempts. It records the project, issue, run id, attempt, transport, local channel path, channel status, and publish/update timestamps needed to route future control requests without bypassing active lease ownership. | +| Runtime SQLite `run_control_channels` | Local control capability metadata for active run attempts. It records the project, issue, run id, attempt, transport, local channel path, channel status, and publish/update timestamps needed to route future control requests without bypassing run lease ownership. | | Runtime SQLite `review_policy_checkpoints` | Latest bounded-review checkpoint state for one project, issue, run, attempt, and phase, including structured independent-review detail. This row is the authority for review handoff and retained repair gating. | | Runtime SQLite `loop_guardrail_checkpoints` | Latest convergence checkpoint for one project, issue, and guardrail reason. It stores the fingerprint, consecutive count, run id, attempt number, and structured detail used to stop non-converging loops without replaying Linear comments. | | Agent evidence under `~/.codex/decodex/agent-evidence//` | Derived local handoff view for repair agents. It may reference private evidence readback commands and compact run capsules, but it is not scheduling authority and is not a public mirror. | @@ -126,7 +126,7 @@ The following facts are local runtime truth and must not be rebuilt from Linear - phase timing and operator activity summaries - retained worktree mappings, retained PR handoff identity, post-review phase, and cleanup or repair ownership -Linear issue fields and Linear execution ledger comments are the team-visible tracker mirror for low-frequency lifecycle records. They may enrich completed run history when the connector is available, but they must not become the live source for active leases, dispatch ownership, retry/backoff state, phase timing, retained worktree ownership, or operator snapshot continuity. +Linear issue fields and Linear execution ledger comments are the team-visible tracker mirror for low-frequency lifecycle records. They may enrich completed run history when the connector is available, but they must not become the live source for run leases, dispatch ownership, retry/backoff state, phase timing, retained worktree ownership, or operator snapshot continuity. Operator snapshots must expose lightweight protocol event summaries, not materialized event journals. Count and latest-event metadata such as `event_count`, @@ -216,7 +216,7 @@ Current runtime note: - The runtime owns lane planning, creation, reuse, and cleanup for those linked worktrees. - The visible lane path lives under the configured worktree root, commonly `.worktrees/` inside the target repository, while `git_dir` resolves under the repository's shared `.git/worktrees/*` admin area and `git_common_dir` resolves to the repository's primary `.git`. - Before starting a live run, `decodex` must reject any prepared lane that is not a registered linked Git worktree for the configured repository. -- Worktree mappings and active leases must remain scoped to the registered project `service_id` so reconciliation does not cross project boundaries. +- Worktree mappings and run leases must remain scoped to the registered project `service_id` so reconciliation does not cross project boundaries. ## Runtime state machine @@ -290,7 +290,7 @@ the repo gate and select the next phase. An `issue_progress_checkpoint`, final c text, or "await next phase" statement is evidence only; it is not a phase exit and must not be treated as a substitute for goal completion. -If an app-server run fails, a supervised child exits unsuccessfully, or active-run +If an app-server run fails, a supervised child exits unsuccessfully, or current-lane reconciliation finds a stalled retained lane while the latest private phase-goal signal for that same run is still an `active` implementation or repair phase, Decodex must run the registered repo gate before converting retained tracked changes into human @@ -534,7 +534,7 @@ Before writing a retry comment, transitioning an issue, or applying `decodex:needs-attention`, Decodex must classify the failure through one writeback disposition: generic retryable failure, structured retryable recovery, or human-required terminal attention. Structured retryable recovery includes typed runtime -failures such as zero-evidence app-server startup failures, stalled active-run +failures such as zero-evidence app-server startup failures, stalled current-lane reconciliation without retained tracked changes, app-server capability preflight timeouts, startup transport disconnects, turn failures, dynamic-tool failures, and retryable repo-gate failures; @@ -665,7 +665,7 @@ disabled and public projections rely on the schema and deterministic guard only. The runtime database stores at least: - registered projects and config fingerprints -- active leases and dispatch ownership +- run leases and dispatch ownership - run attempts and attempt status - protocol event journals - private execution events scoped by project, issue, run, and attempt @@ -683,7 +683,7 @@ The runtime database stores at least: - tracker and PR cache rows needed to survive connector outages - typed connector health and external API backoff -For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees//.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `active_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. If a raw attempt is already terminal but the matching marker still proves live process, active thread, or active work-protocol execution, operator-facing status must also keep the lane visible as `running` while preserving the raw terminal value in `attempt_status`; terminal maintenance events such as `thread/archive` and completed-turn bookkeeping are not active execution evidence. Only terminal-finalize writeback projections may hide a live marker from active execution. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md). +For child supervision, the active lane may also carry a short-lived worktree heartbeat marker at `.worktrees//.decodex-run-activity`. That marker is advisory, keyed to the current `run_id` plus `attempt_number`, and exists so the control plane can observe child activity across process boundaries, surface active thread and protocol progress in operator status, and keep high-frequency telemetry out of Linear. When the marker records process liveness, it must pair `process_id` with both host boot identity (`host_boot_id`) and process start identity (`process_start_identity`). A marker from a previous boot, a marker missing either identity, a marker whose process start identity no longer matches the live PID, a marker whose PID has exited into an unreaped zombie state, or a marker observed while Decodex cannot read the current host or process identity must not be treated as a live process even if that PID currently exists. Operator snapshots expose `process_liveness_reason` so operators can distinguish stopped processes, previous-boot markers, and same-boot PID reuse from genuine live execution. The marker may also carry additive `child_agent_activity`, protocol, account, and legacy review-policy JSON or scalar fields for operator diagnostics. Legacy review-policy marker fields are breadcrumbs only: review-policy gating belongs to the runtime store and must not be overwritten from marker values. Operator snapshots must keep queue ownership separate from execution liveness: `run_lease` and `queue_lease_state` describe the local queue lease, while `execution_liveness` describes the observed process, app-server thread, or protocol marker that keeps an active lane visible. If a raw attempt is still `starting` after app-server thread, model, or protocol activity is observed, operator-facing `status` must report `running` and preserve the raw value in `attempt_status`. If a raw attempt is already terminal but the matching marker still proves live process, active thread, or active work-protocol execution, operator-facing status must also keep the lane visible as `running` while preserving the raw terminal value in `attempt_status`; terminal maintenance events such as `thread/archive` and completed-turn bookkeeping are not active execution evidence. Only terminal-finalize writeback projections may hide a live marker from active execution. High-frequency heartbeat, child-agent buckets, token counts, idle ages, and other transient liveness details stay local/operator-only under the boundary defined by [`linear-execution-ledger.md`](./linear-execution-ledger.md). If a persisted attempt has a terminal-looking status such as `failed`, `interrupted`, or `stalled` while current marker, active thread, or active work-protocol evidence still identifies the same `run_id` and `attempt_number` as live, operator status must @@ -700,7 +700,7 @@ After the hidden `_attempt` child adopts the inherited issue-claim and dispatch- - The child-owned dispatch-slot FD is the cross-process mutual-exclusion guard for the occupied slot. A competing `decodex` process must still observe that slot as unavailable while the child owns the descriptor. - The parent must release its process-local issue-claim and dispatch-slot guard handles after the child adopts them. Any parent-side record left for observation or cleanup is bookkeeping only and must not hold an additional dispatch-slot FD or reserve another slot. -- The runtime database lease remains visible while the child owns the run. Releasing parent-local guard handles must not delete, hide, or downgrade the DB-backed active lease that operator status and restart recovery use to identify the running lane. +- The runtime database lease remains visible while the child owns the run. Releasing parent-local guard handles must not delete, hide, or downgrade the DB-backed run lease that operator status and restart recovery use to identify the running lane. Restart recovery must use the runtime database plus retained worktrees and external caches instead of replaying Linear comments as the runtime ledger. @@ -721,7 +721,7 @@ coarse lifecycle records. The status surface should describe runtime DB-backed execution state, plus low-frequency connector refreshes and retained `.worktrees` lanes, for example: -- active leased runs +- run-leased runs - persisted run attempts with local status, thread id, and latest recorded protocol event - registered project summaries with enabled state, fleet health/capacity counts, connector state, last activity, and retained worktree counts that exclude actively running lane worktrees - queued tracker issues currently labeled for automatic dispatch, together with the current dispatch classification (`ready`, `claimed`, `blocked`, or `closed`) and any local policy reason that explains why they would or would not run next @@ -730,7 +730,7 @@ The status surface should describe runtime DB-backed execution state, plus low-f Retained worktree counts and recovery-worktree details must come from one consistent operator snapshot. If the summary count and detail list disagree, surface it as a snapshot consistency warning or bug, not as cleanup work for the operator. -After a process restart, recent-run history, active lease ownership, retained post-review state, and recovery worktree mappings must be reloaded from the runtime database before new work is scheduled. The control plane may refresh low-frequency tracker and PR cache rows, but it must continue publishing local operator state while Linear or GitHub is unavailable. +After a process restart, recent-run history, run lease ownership, retained post-review state, and recovery worktree mappings must be reloaded from the runtime database before new work is scheduled. The control plane may refresh low-frequency tracker and PR cache rows, but it must continue publishing local operator state while Linear or GitHub is unavailable. ## Retention and cleanup @@ -740,7 +740,7 @@ After a process restart, recent-run history, active lease ownership, retained po terminal runs may be compacted by `decodex maintenance prune --apply` or by the automatic `decodex serve` auto-safe maintenance path once the latest event is at least 14 days old, but only after Decodex writes the compact run summary and - confirms that no active lease, retained worktree, review handoff, review + confirms that no run lease, retained worktree, review handoff, review orchestration, human-attention ledger event, terminal-failure ledger event, or cleanup blocker still owns that run or issue. The first private execution event schema has no compaction path; add one only when runtime maintenance owns a @@ -823,7 +823,7 @@ After a process restart, recent-run history, active lease ownership, retained po and `terminal_path = "retained_partial_progress"`. - If stalled reconciliation finds no tracked changes in the retained worktree, it must classify the lane as structured retryable recovery with `error_class = "stalled_run_detected"` while retry budget remains. The retry must keep active ownership, write a failure retry schedule for the same worktree, and must not add `decodex:needs-attention` until retry budget exhaustion or another terminal boundary applies. - If the supervised child already exited before the next control-plane tick, stalled reconciliation must still inspect the just-finished lane using recorded protocol activity and retained worktree state rather than skipping directly to generic failure handling. -- Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, optional progress diagnostics, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. The snapshot producer owns the derived operator booleans `has_fresh_execution`, `counts_as_running`, and `needs_attention`; dashboard, App, and other UI consumers must use those fields when present instead of reinterpreting raw `process_alive`, thread, protocol, or idle fields independently. The snapshot producer also owns `shadowed_by_active_run` on retained post-review lanes, and current project review, waiting, attention, landing, and cleanup counts must exclude lanes shadowed by fresh active execution for the same issue. `process_alive = false` is only process-marker evidence and must not be displayed as stopped when `has_fresh_execution = true`. `last_progress_at` is meaningful-work progress only: tool calls, file or diff changes, plan/model output, repo validation, PR/review/terminal lifecycle, or other explicit work events may refresh it, but account, rate-limit, phase-goal, passive status, warning, token-usage, heartbeat, or similar non-work protocol traffic must only refresh protocol liveness. When a lane remains in `model_execution` with fresh protocol activity but stale or missing work progress and the recent protocol events are only non-work traffic, status should expose `progress_diagnostic = "protocol_only_activity"` while preserving process and protocol liveness separately. +- Operator status snapshots must expose structured liveness and wait-state fields derived from runtime records plus marker breadcrumbs, including current phase, optional wait reason, current operation, last run/protocol/progress times, idle age, a soft `suspected_stall` signal, optional progress diagnostics, and any queued retry kind plus due time, so operators can distinguish active execution from continuation waits, retry backoff, early stall suspicion, and genuine hard stalls without inferring progress from filesystem churn. The snapshot producer owns the derived operator booleans `has_fresh_execution`, `counts_as_running`, and `needs_attention`; dashboard, App, and other UI consumers must use those fields when present instead of reinterpreting raw `process_alive`, thread, protocol, or idle fields independently. The snapshot producer also owns `shadowed_by_current_lane` on retained post-review lanes, and current project review, waiting, attention, landing, and cleanup counts must exclude lanes shadowed by fresh active execution for the same issue. `process_alive = false` is only process-marker evidence and must not be displayed as stopped when `has_fresh_execution = true`. `last_progress_at` is meaningful-work progress only: tool calls, file or diff changes, plan/model output, repo validation, PR/review/terminal lifecycle, or other explicit work events may refresh it, but account, rate-limit, phase-goal, passive status, warning, token-usage, heartbeat, or similar non-work protocol traffic must only refresh protocol liveness. When a lane remains in `model_execution` with fresh protocol activity but stale or missing work progress and the recent protocol events are only non-work traffic, status should expose `progress_diagnostic = "protocol_only_activity"` while preserving process and protocol liveness separately. - Operator status snapshots may expose an additive `child_agent_activity` object when app-server protocol events have produced one for the current run. The object must stay machine-readable and dashboard/CLI shared, and should describe dynamic observed buckets rather than a fixed workflow: current child bucket and elapsed time, bucket wall/event/tool counts, current/max/cumulative input tokens, cumulative output tokens, largest tool output, and warnings for repeated large outputs. Missing `child_agent_activity` means no child breakdown was captured; existing JSON consumers must continue to work without it. - If the agent Git credential preflight fails, operator status must report the retained lane as a credential failure requiring operator recovery, not as a still-running lane. - If retry budget or needs-attention recovery finds tracked changes in the retained worktree after active phase-goal recovery has no applicable continuation path, operator status must report retained partial progress rather than only a generic retry-budget hold. Retained progress is the recovery disposition; later runtime, app-server, credential, transport, or repo-gate failure classes must be preserved as source evidence instead of overriding the retained-progress lifecycle path. The failure class may be `partial_progress_retained` when no more specific runtime error class is available. Operators should then inspect the patch, finish validation and PR handoff if it is useful, or reset the retained worktree explicitly. @@ -868,7 +868,7 @@ After a process restart, recent-run history, active lease ownership, retained po operator projection. Status must not also render the issue as an intake queue candidate, because the queue label is then a stale echo of the retained terminal lane rather than dispatchable backlog. -- If Linear still has `decodex:active:` on an issue that also remains queued, but the local runtime cannot prove a matching active lease, status must classify the queued row as blocked with reason `linear_active_label_present`; it must not treat the issue as ready intake. If the retained marker or private execution event rows for that run are missing, status must surface `evidence_missing` in the recovery details. If the retained worktree has tracked changes, that dirty worktree remains owned by queued recovery/attention instead of being hidden as cleanup-only state. +- If Linear still has `decodex:active:` on an issue that also remains queued, but the local runtime cannot prove a matching run lease, status must classify the queued row as blocked with reason `linear_active_label_present`; it must not treat the issue as ready intake. If the retained marker or private execution event rows for that run are missing, status must surface `evidence_missing` in the recovery details. If the retained worktree has tracked changes, that dirty worktree remains owned by queued recovery/attention instead of being hidden as cleanup-only state. - Operator status snapshots must expose worktree provenance in both JSON and human text output. A cleanup-only worktree with `provenance_source = "legacy_unknown"` must set `audit_required = true` and provide a `decodex recover legacy-closeout` next action; @@ -876,7 +876,7 @@ After a process restart, recent-run history, active lease ownership, retained po - During an active run, operator snapshots must expose `thread_id` as soon as the Codex thread exists, plus monotonically advancing `event_count`, `last_event_type`, and `last_event_at` once protocol events are recorded. These fields may be hydrated either from the current process journal or from the active lane's `.decodex-run-activity` marker when `status` is running in a separate process. - `thread_id = null` is expected only before the worker creates the Codex thread for the current run. `event_count = 0`, `last_event_type = null`, and `last_event_at = null` are expected only before the first protocol event for that same run. After the thread exists and protocol activity has started, those empty values indicate missing hydration rather than normal progress. - Operator snapshots may expose an additive `protocol_activity` object derived from app-server structured messages for the current run. The object stays local/operator-only and should summarize turn status, waiting reason, rate-limit status, and a compact recent event list for high-value app-server activity such as `turn/started`, `turn/completed`, plan updates, diff updates, item start/completion, command output deltas, server request responses, account updates, and rate-limit updates. Missing `protocol_activity` means no structured summary was captured yet; consumers must continue to rely on the older `event_count`, `last_event_type`, `last_event_at`, thread fields, and `child_agent_activity` fields when it is absent. Presence in `protocol_activity` is not by itself meaningful progress; non-work account, rate-limit, phase-goal, passive status, warning, model-routing, and token-usage events must remain distinguishable from work-progress events through `last_progress_at` and `progress_diagnostic`. -- The operator snapshot transport must stay local/operator-only. `decodex serve` exposes the human-facing operator console from the canonical HTTP `GET /` and `GET /dashboard` routes, serves only the necessary dashboard assets, `GET /livez` liveness probe, and local account-control API over HTTP, and delivers published snapshots, active-run activity, and dashboard control acknowledgements through the local `GET /dashboard/control` WebSocket upgrade. +- The operator snapshot transport must stay local/operator-only. `decodex serve` exposes the human-facing operator console from the canonical HTTP `GET /` and `GET /dashboard` routes, serves only the necessary dashboard assets, `GET /livez` liveness probe, and local account-control API over HTTP, and delivers published snapshots, current-lane activity, and dashboard control acknowledgements through the local `GET /dashboard/control` WebSocket upgrade. - `GET /livez` is only a process- and listener-level liveness probe. It must not claim control-plane tick freshness or forward progress by itself. - The dashboard must not depend on a separate HTTP snapshot or readiness endpoint; snapshot freshness belongs to the WebSocket-delivered snapshot payload and the browser connection state. - Reconciliation must mark locally active run attempts as `interrupted` when their