From 4506f317f60fa0b8f2fef7f2e7a73ebbb360c72d Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Wed, 10 Jun 2026 02:43:52 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Add loop guardrail policy stops","authority":"XY-856"} --- apps/decodex/src/agent/tracker_tool_bridge.rs | 7 + .../src/agent/tracker_tool_bridge/tools.rs | 1 + apps/decodex/src/orchestrator.rs | 3 +- apps/decodex/src/orchestrator/execution.rs | 398 ++++++++++++++++-- .../decodex/src/orchestrator/operator_http.rs | 10 +- apps/decodex/src/orchestrator/selection.rs | 15 + apps/decodex/src/orchestrator/status.rs | 70 ++- .../tests/operator/status/queue.rs | 84 ++++ .../tests/recovery/terminal_failures.rs | 3 + .../src/orchestrator/tests/runtime/failure.rs | 299 +++++++++++++ apps/decodex/src/orchestrator/types.rs | 98 +++++ apps/decodex/src/state/internal.rs | 188 +++++++++ apps/decodex/src/state/models.rs | 60 +++ apps/decodex/src/state/store.rs | 150 +++++++ apps/decodex/src/state/tests.rs | 99 ++++- docs/reference/operator-control-plane.md | 13 +- docs/runbook/lane-control-recovery.md | 22 + docs/spec/lane-control.md | 10 + docs/spec/linear-execution-ledger.md | 10 + docs/spec/loop-runtime.md | 13 +- docs/spec/post-review-lifecycle.md | 5 + docs/spec/review-orchestration.md | 2 +- docs/spec/runtime.md | 37 +- 23 files changed, 1527 insertions(+), 70 deletions(-) diff --git a/apps/decodex/src/agent/tracker_tool_bridge.rs b/apps/decodex/src/agent/tracker_tool_bridge.rs index ba09be492..e8020d7f4 100644 --- a/apps/decodex/src/agent/tracker_tool_bridge.rs +++ b/apps/decodex/src/agent/tracker_tool_bridge.rs @@ -136,6 +136,7 @@ pub(crate) struct TrackerToolBridge<'a> { local_opt_out_requested: RefCell, manual_attention_requested: RefCell, manual_attention_comment_recorded: RefCell, + manual_attention_error_class: RefCell>, continuation_blocking_tracker_write: RefCell>, pending_review_completion: RefCell>, finalized_completion_path: RefCell>, @@ -162,6 +163,7 @@ impl<'a> TrackerToolBridge<'a> { ), manual_attention_requested: RefCell::new(false), manual_attention_comment_recorded: RefCell::new(false), + manual_attention_error_class: RefCell::new(None), continuation_blocking_tracker_write: RefCell::new(None), pending_review_completion: RefCell::new(None), finalized_completion_path: RefCell::new(None), @@ -215,6 +217,7 @@ impl<'a> TrackerToolBridge<'a> { ), manual_attention_requested: RefCell::new(false), manual_attention_comment_recorded: RefCell::new(false), + manual_attention_error_class: RefCell::new(None), continuation_blocking_tracker_write: RefCell::new(None), pending_review_completion: RefCell::new(None), finalized_completion_path: RefCell::new(None), @@ -357,6 +360,10 @@ impl<'a> TrackerToolBridge<'a> { pub(crate) fn review_context(&self) -> Option<&ReviewHandoffContext> { self.review_context.as_ref() } + + pub(crate) fn manual_attention_error_class(&self) -> Option { + self.manual_attention_error_class.borrow().clone() + } } impl DynamicToolHandler for TrackerToolBridge<'_> { diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tools.rs b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs index 9f3969397..ede9897bf 100644 --- a/apps/decodex/src/agent/tracker_tool_bridge/tools.rs +++ b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs @@ -817,6 +817,7 @@ impl<'a> TrackerToolBridge<'a> { } self.manual_attention_comment_recorded.replace(true); + self.manual_attention_error_class.replace(Some(comment.error_class.clone())); let verb = if created { "added" } else { "already existed for" }; diff --git a/apps/decodex/src/orchestrator.rs b/apps/decodex/src/orchestrator.rs index 8db567fc8..48afa205e 100644 --- a/apps/decodex/src/orchestrator.rs +++ b/apps/decodex/src/orchestrator.rs @@ -30,11 +30,12 @@ use color_eyre::Report; use libc::pid_t; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use sha2::Sha256; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::{agent, default_branch_sync, git_credentials, maintenance, state}; #[rustfmt::skip] -use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{InternalReviewMode, ServiceConfig}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; +use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{InternalReviewMode, ServiceConfig}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, LoopGuardrailCheckpoint, LoopGuardrailCheckpointInput, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; include!("orchestrator/types.rs"); diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 2909aada9..b2a057661 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -3,9 +3,12 @@ use agent::CodexAccountPool; use agent::CodexAccountProvider; use agent::AppServerThreadArchiveRequest; use records::LinearExecutionEventPublicProjection; +use sha2::Digest; use crate::tracker::privacy_classifier::PublicProjectionPrivacyClassifier; +const LOOP_GUARDRAIL_CONVERGENCE_BUDGET: i64 = 3; + #[derive(Debug)] pub(crate) struct AppServerZeroEvidenceStartFailure { issue_identifier: String, @@ -163,6 +166,10 @@ impl RepoGatePhaseGoalController<'_> { &self.issue_run.worktree.path, ) { Ok(()) => { + self.state_store.clear_loop_guardrail_checkpoints_for_issue( + self.project.service_id(), + &self.issue_run.issue.id, + )?; self.record_phase_goal_transition( phase, "validation_pass", @@ -187,6 +194,15 @@ impl RepoGatePhaseGoalController<'_> { )?; if repo_gate_failure.disposition() == RepoGateFailureDisposition::ContinueRepair { + if let Some(loop_guardrail_stop) = retryable_failure_loop_guardrail_stop( + self.project, + self.state_store, + self.issue_run, + &error, + )? { + return Err(Report::new(loop_guardrail_stop).wrap_err(error)); + } + let detail = format!( "Repo gate failed with `{}`. Inspect the worktree, run the registered canonicalize and verify commands, and repair only the validation failure.", repo_gate_failure.error_class() @@ -324,6 +340,25 @@ struct ZeroEvidenceAppServerStartFailureDiagnostic { source_error_chain: Vec, } +struct LoopGuardrailWorktreeFingerprint { + head_sha: String, + tracked_status_hash: String, + tracked_diff_hash: String, +} + +struct FailureHandlingContext<'a, T> +where + T: IssueTracker, +{ + tracker: &'a T, + project: &'a ServiceConfig, + workflow: &'a WorkflowDocument, + state_store: &'a StateStore, + issue_run: &'a IssueRunPlan, + worktree_path: &'a str, + retry_budget_attempts: i64, +} + #[derive(Clone, Copy, Eq, PartialEq)] enum TerminalFailureEventRecordStatus { Recorded, @@ -368,6 +403,13 @@ where Ok(summary) => { persist_issue_run_outcome(state_store, &issue_run.run_id, &summary)?; + if !summary.continuation_pending { + state_store.clear_loop_guardrail_checkpoints_for_issue( + project.service_id(), + &issue_run.issue.id, + )?; + } + tracing::info!( project_id = project.service_id(), issue_id = issue_run.issue.id, @@ -1332,6 +1374,7 @@ where issue_identifier: issue_run.issue.identifier.clone(), label: workflow.frontmatter().tracker().needs_attention_label().to_owned(), run_id: issue_run.run_id.clone(), + error_class: tracker_tool_bridge.manual_attention_error_class(), })); }, RunCompletionDisposition::ReviewRepair => { @@ -1492,6 +1535,7 @@ fn preserve_manual_attention_request( issue_identifier: issue_run.issue.identifier.clone(), label: workflow.frontmatter().tracker().needs_attention_label().to_owned(), run_id: issue_run.run_id.clone(), + error_class: None, }) .wrap_err(error); } @@ -1690,8 +1734,183 @@ fn truncate_private_diagnostic_text(text: &str) -> String { truncated } +fn retryable_failure_loop_guardrail_stop( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + error: &Report, +) -> Result> { + let Some(worktree_fingerprint) = + loop_guardrail_worktree_fingerprint(&issue_run.worktree.path)? + else { + return Ok(None); + }; + let mut observations = Vec::new(); + + if let Some(repo_gate_failure) = error.downcast_ref::() + && repo_gate_failure.disposition() == RepoGateFailureDisposition::ContinueRepair + { + observations.push(( + LoopGuardrailReason::ValidationRepeat, + format!( + "{}:{}:{}", + repo_gate_failure.error_class(), + worktree_fingerprint.head_sha.as_str(), + loop_guardrail_text_hash(&error.to_string()) + ), + Some(repo_gate_failure.error_class()), + )); + observations.push(( + LoopGuardrailReason::RemainingDeltaUnchanged, + format!( + "{}:{}:{}:{}", + repo_gate_failure.error_class(), + worktree_fingerprint.head_sha.as_str(), + worktree_fingerprint.tracked_status_hash.as_str(), + worktree_fingerprint.tracked_diff_hash.as_str() + ), + Some(repo_gate_failure.error_class()), + )); + } + + observations.push(( + LoopGuardrailReason::NoEffectiveDiff, + format!( + "{}:{}:{}", + worktree_fingerprint.head_sha.as_str(), + worktree_fingerprint.tracked_status_hash.as_str(), + worktree_fingerprint.tracked_diff_hash.as_str() + ), + retained_progress_source_error_class(error), + )); + + for (reason, fingerprint, source_error_class) in observations { + let checkpoint = state_store.observe_loop_guardrail_checkpoint( + LoopGuardrailCheckpointInput { + project_id: project.service_id(), + issue_id: &issue_run.issue.id, + reason: reason.error_class(), + fingerprint: &fingerprint, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + details_json: &json!({ + "schema": "decodex.loop_guardrail_checkpoint/1", + "reason": reason.error_class(), + "source_error_class": source_error_class, + "head_sha": worktree_fingerprint.head_sha.as_str(), + "tracked_status_hash": worktree_fingerprint.tracked_status_hash.as_str(), + "tracked_diff_hash": worktree_fingerprint.tracked_diff_hash.as_str(), + "threshold": LOOP_GUARDRAIL_CONVERGENCE_BUDGET, + }) + .to_string(), + }, + )?; + + record_loop_guardrail_private_event( + project, + state_store, + issue_run, + &checkpoint, + source_error_class, + )?; + + if checkpoint.consecutive_count() >= LOOP_GUARDRAIL_CONVERGENCE_BUDGET { + return Ok(Some(LoopGuardrailStopRequested { + issue_identifier: issue_run.issue.identifier.clone(), + run_id: issue_run.run_id.clone(), + reason, + consecutive_count: checkpoint.consecutive_count(), + fingerprint, + source_error_class: source_error_class.map(ToOwned::to_owned), + })); + } + } + + Ok(None) +} + +fn loop_guardrail_worktree_fingerprint( + worktree_path: &Path, +) -> Result> { + let Some(head_sha) = worktree_head_oid(worktree_path)? else { + return Ok(None); + }; + let Some(tracked_status) = + git_guardrail_output(worktree_path, &["status", "--porcelain", "--untracked-files=no"])? + else { + return Ok(None); + }; + let Some(tracked_diff) = + git_guardrail_output(worktree_path, &["diff", "--binary", "--no-ext-diff", "HEAD", "--"])? + else { + return Ok(None); + }; + + Ok(Some(LoopGuardrailWorktreeFingerprint { + head_sha, + tracked_status_hash: loop_guardrail_text_hash(&tracked_status), + tracked_diff_hash: loop_guardrail_text_hash(&tracked_diff), + })) +} + +fn git_guardrail_output(worktree_path: &Path, args: &[&str]) -> Result> { + let output = Command::new("git") + .arg("-C") + .arg(worktree_path) + .args(args) + .output()?; + + if !output.status.success() { + return Ok(None); + } + + Ok(Some(String::from_utf8_lossy(&output.stdout).into_owned())) +} + +fn loop_guardrail_text_hash(text: &str) -> String { + let digest = ::digest(text.as_bytes()); + let mut hash = String::with_capacity(64); + + for byte in digest { + hash.push(char::from(b"0123456789abcdef"[(byte >> 4) as usize])); + hash.push(char::from(b"0123456789abcdef"[(byte & 0x0f) as usize])); + } + + hash +} + +fn record_loop_guardrail_private_event( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + checkpoint: &LoopGuardrailCheckpoint, + source_error_class: Option<&str>, +) -> Result<()> { + state_store + .append_private_execution_event( + project.service_id(), + &issue_run.issue.id, + &issue_run.run_id, + issue_run.attempt_number, + "loop_guardrail_checkpoint", + json!({ + "schema": "decodex.loop_guardrail_checkpoint/1", + "reason": checkpoint.reason(), + "fingerprint": checkpoint.fingerprint(), + "consecutive_count": checkpoint.consecutive_count(), + "threshold": LOOP_GUARDRAIL_CONVERGENCE_BUDGET, + "checkpoint_run_id": checkpoint.run_id(), + "checkpoint_attempt_number": checkpoint.attempt_number(), + "source_error_class": source_error_class, + "details": checkpoint.details_json(), + }), + ) + .map(|_| ()) +} + fn run_failure_requires_terminal_attention(error: &Report) -> bool { error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() @@ -1729,54 +1948,27 @@ where let worktree_path = relative_worktree_path(project, &issue_run.worktree); let retry_budget_attempts = retry_budget_attempts_for_current_failure(state_store, issue_run)?; + let failure_context = FailureHandlingContext { + tracker, + project, + workflow, + state_store, + issue_run, + worktree_path: &worktree_path, + retry_budget_attempts, + }; + let loop_guardrail_stop = if requires_terminal_attention { + None + } else { + retryable_failure_loop_guardrail_stop(project, state_store, issue_run, error)? + }; - if !requires_terminal_attention && retry_budget_attempts < max_attempts { - let (retry_error_class, retry_next_action) = retry_comment_details(error); - - write_retry_schedule_marker_for_runtime_retry( - error, - workflow, - issue_run, - retry_budget_attempts, - )?; - - tracing::warn!( - project_id = project.service_id(), - issue_id = issue_run.issue.id, - issue = issue_run.issue.identifier, - run_id = issue_run.run_id, - attempt = issue_run.attempt_number, - retry_budget_attempt = retry_budget_attempts, - max_attempts, - branch = issue_run.worktree.branch_name, - worktree_path = %worktree_path, - error_class = retry_error_class, - "Run failed and remains retryable." - ); - - tracker::create_public_comment( - tracker, - &issue_run.issue.id, - &format_retry_comment(RetryComment { - run_id: &issue_run.run_id, - attempt_number: issue_run.attempt_number, - retry_budget_attempt_number: retry_budget_attempts, - max_attempts, - worktree_path, - branch_name: &issue_run.worktree.branch_name, - error_class: retry_error_class, - next_action: &retry_next_action, - }), - )?; - - write_retry_budget_marker( - &issue_run.worktree.path, - &issue_run.run_id, - issue_run.attempt_number, - retry_budget_attempts, - )?; + if let Some(loop_guardrail_stop) = loop_guardrail_stop { + return apply_loop_guardrail_failure_writeback(&failure_context, loop_guardrail_stop); + } - return Ok(()); + if !requires_terminal_attention && retry_budget_attempts < max_attempts { + return apply_retryable_failure_writeback(&failure_context, error, max_attempts); } let retained_partial_progress = retained_partial_progress_error( @@ -1832,6 +2024,119 @@ where Ok(()) } +fn apply_retryable_failure_writeback( + context: &FailureHandlingContext<'_, T>, + error: &Report, + max_attempts: i64, +) -> Result<()> +where + T: IssueTracker, +{ + let (retry_error_class, retry_next_action) = retry_comment_details(error); + + write_retry_schedule_marker_for_runtime_retry( + error, + context.workflow, + context.issue_run, + context.retry_budget_attempts, + )?; + + tracing::warn!( + project_id = context.project.service_id(), + issue_id = context.issue_run.issue.id, + issue = context.issue_run.issue.identifier, + run_id = context.issue_run.run_id, + attempt = context.issue_run.attempt_number, + retry_budget_attempt = context.retry_budget_attempts, + max_attempts, + branch = context.issue_run.worktree.branch_name, + worktree_path = %context.worktree_path, + error_class = retry_error_class, + "Run failed and remains retryable." + ); + + tracker::create_public_comment( + context.tracker, + &context.issue_run.issue.id, + &format_retry_comment(RetryComment { + run_id: &context.issue_run.run_id, + attempt_number: context.issue_run.attempt_number, + retry_budget_attempt_number: context.retry_budget_attempts, + max_attempts, + worktree_path: context.worktree_path.to_owned(), + branch_name: &context.issue_run.worktree.branch_name, + error_class: retry_error_class, + next_action: &retry_next_action, + }), + )?; + + write_retry_budget_marker( + &context.issue_run.worktree.path, + &context.issue_run.run_id, + context.issue_run.attempt_number, + context.retry_budget_attempts, + )?; + + Ok(()) +} + +fn apply_loop_guardrail_failure_writeback( + context: &FailureHandlingContext<'_, T>, + loop_guardrail_stop: LoopGuardrailStopRequested, +) -> Result<()> +where + T: IssueTracker, +{ + let terminal_error = Report::new(loop_guardrail_stop); + let privacy_classifier = configured_public_projection_privacy_classifier(context.project)?; + let outcome = apply_terminal_failure_writeback( + context.tracker, + TerminalFailureWritebackRuntime { + service_id: context.project.service_id(), + state_store: Some(context.state_store), + privacy_classifier: &privacy_classifier, + }, + context.workflow, + context.issue_run, + context.worktree_path, + false, + &terminal_error, + )?; + + if outcome.retry_guarded_by_state { + write_terminal_guard_marker( + &context.issue_run.worktree.path, + &context.issue_run.run_id, + context.issue_run.attempt_number, + )?; + + context + .state_store + .update_run_status(&context.issue_run.run_id, TERMINAL_GUARDED_RUN_STATUS)?; + } + + write_retry_budget_marker( + &context.issue_run.worktree.path, + &context.issue_run.run_id, + context.issue_run.attempt_number, + context.retry_budget_attempts, + )?; + + tracing::warn!( + project_id = context.project.service_id(), + issue_id = context.issue_run.issue.id, + issue = context.issue_run.issue.identifier, + run_id = context.issue_run.run_id, + attempt = context.issue_run.attempt_number, + branch = context.issue_run.worktree.branch_name, + worktree_path = context.worktree_path, + error_class = outcome.error_class, + "Run stopped by loop guardrail." + ); + + Ok(()) +} + fn retry_budget_attempts_for_current_failure( state_store: &StateStore, issue_run: &IssueRunPlan, @@ -1874,6 +2179,7 @@ fn retained_partial_progress_error( fn retained_progress_should_defer_to_terminal_intent(error: &Report) -> bool { error.downcast_ref::().is_some() + || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() || error.downcast_ref::().is_some() diff --git a/apps/decodex/src/orchestrator/operator_http.rs b/apps/decodex/src/orchestrator/operator_http.rs index 7b065a19c..9b66025ac 100644 --- a/apps/decodex/src/orchestrator/operator_http.rs +++ b/apps/decodex/src/orchestrator/operator_http.rs @@ -1,5 +1,5 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; -use sha1::{Digest as _, Sha1}; +use sha1::Sha1; use crate::accounts::{self, AccountUseRequest}; @@ -1201,12 +1201,12 @@ fn websocket_upgrade_required_response() -> Vec { fn websocket_accept_key(key: &str) -> String { const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - let mut hasher = Sha1::new(); + let mut hasher = ::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); + sha1::Digest::update(&mut hasher, key.as_bytes()); + sha1::Digest::update(&mut hasher, WEBSOCKET_GUID.as_bytes()); - STANDARD.encode(hasher.finalize()) + STANDARD.encode(sha1::Digest::finalize(hasher)) } fn operator_http_header_value<'a>(request: &'a str, header_name: &str) -> Option<&'a str> { diff --git a/apps/decodex/src/orchestrator/selection.rs b/apps/decodex/src/orchestrator/selection.rs index 81b4c46ac..1bc487b91 100644 --- a/apps/decodex/src/orchestrator/selection.rs +++ b/apps/decodex/src/orchestrator/selection.rs @@ -197,7 +197,22 @@ fn terminal_failure_comment_details( retained_review_needs_attention.reason ), ) + } else if let Some(loop_guardrail_stop) = error.downcast_ref::() { + ( + loop_guardrail_stop.reason.error_class(), + loop_guardrail_stop.reason.terminal_next_action(recovery_gate), + ) } else if manual_attention_requested { + if let Some(manual_attention) = error.downcast_ref::() + && let Some(error_class) = manual_attention.error_class.as_deref() + && let Some(reason) = LoopGuardrailReason::from_error_class(error_class) + { + return ( + reason.error_class(), + reason.terminal_next_action(recovery_gate), + ); + } + ( "human_attention_required", format!( diff --git a/apps/decodex/src/orchestrator/status.rs b/apps/decodex/src/orchestrator/status.rs index b01249e3f..db667046b 100644 --- a/apps/decodex/src/orchestrator/status.rs +++ b/apps/decodex/src/orchestrator/status.rs @@ -2138,7 +2138,9 @@ fn queued_issue_blocker_identifiers( workflow: &WorkflowDocument, reason: &str, ) -> Vec { - if reason != "open_tracker_blockers" { + if reason != "open_tracker_blockers" + && reason != LoopGuardrailReason::DependencyProgramStale.error_class() + { return Vec::new(); } @@ -2150,6 +2152,51 @@ fn queued_issue_blocker_identifiers( .collect() } +fn observe_dependency_program_stale_guardrail( + project: &ServiceConfig, + workflow: &WorkflowDocument, + state_store: &StateStore, + issue: &TrackerIssue, +) -> crate::prelude::Result { + let blocker_fingerprint = dependency_blocker_fingerprint(issue, workflow); + let checkpoint = state_store.observe_loop_guardrail_checkpoint( + LoopGuardrailCheckpointInput { + project_id: project.service_id(), + issue_id: &issue.id, + reason: LoopGuardrailReason::DependencyProgramStale.error_class(), + fingerprint: &blocker_fingerprint, + run_id: "queued-dependency-blocker", + attempt_number: 0, + details_json: &json!({ + "schema": "decodex.loop_guardrail_checkpoint/1", + "reason": LoopGuardrailReason::DependencyProgramStale.error_class(), + "blockers": queued_issue_blocker_identifiers( + issue, + workflow, + "open_tracker_blockers", + ), + "threshold": LOOP_GUARDRAIL_CONVERGENCE_BUDGET, + }) + .to_string(), + }, + )?; + + Ok(checkpoint) +} + +fn dependency_blocker_fingerprint(issue: &TrackerIssue, workflow: &WorkflowDocument) -> String { + let mut blockers = issue + .blockers + .iter() + .filter(|blocker| !state_name_is_terminal(&blocker.state.name, workflow)) + .map(|blocker| format!("{}:{}", blocker.identifier, blocker.state.name)) + .collect::>(); + + blockers.sort(); + + loop_guardrail_text_hash(&blockers.join("|")) +} + fn classify_queued_issue( tracker: &T, project: &ServiceConfig, @@ -2186,8 +2233,27 @@ where return Ok(("blocked", "issue_opted_out")); } if !todo_blocker_rule_passes(issue, workflow) { - return Ok(("blocked", "open_tracker_blockers")); + let checkpoint = observe_dependency_program_stale_guardrail( + project, + workflow, + state_store, + issue, + )?; + let reason = if checkpoint.consecutive_count() >= LOOP_GUARDRAIL_CONVERGENCE_BUDGET { + LoopGuardrailReason::DependencyProgramStale.error_class() + } else { + "open_tracker_blockers" + }; + + return Ok(("blocked", reason)); } + + state_store.clear_loop_guardrail_checkpoint( + project.service_id(), + &issue.id, + LoopGuardrailReason::DependencyProgramStale.error_class(), + )?; + if !issue_has_generic_dispatch_briefing(issue) { return Ok(("blocked", "missing_dispatch_briefing")); } diff --git a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs index 44c9fdda2..9bbcc06f6 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/queue.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/queue.rs @@ -201,6 +201,90 @@ fn live_operator_status_snapshot_reports_only_open_tracker_blockers() { assert_eq!(candidate.blocker_identifiers, vec![String::from("PUB-105")]); } +#[test] +fn live_operator_status_snapshot_marks_repeated_open_blockers_as_stale_program() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let mut issue = sample_issue_with_sort_fields( + "issue-blocked", + "PUB-108", + "Todo", + &[], + Some(1), + "2026-03-13T04:16:17.133Z", + ); + + issue.blockers = vec![sample_blocker("issue-open", "PUB-105", "Todo")]; + + for expected_reason in [ + "open_tracker_blockers", + "open_tracker_blockers", + "dependency_program_stale", + ] { + let tracker = FakeTracker::new(vec![issue.clone()]); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("snapshot should build"); + let candidate = + snapshot.queued_candidates.first().expect("blocked queued issue should exist"); + + assert_eq!(candidate.reason, expected_reason); + assert_eq!(candidate.classification, "blocked"); + assert_eq!(candidate.blocker_identifiers, vec![String::from("PUB-105")]); + } + + let checkpoint = state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "dependency_program_stale") + .expect("dependency guardrail checkpoint should read") + .expect("dependency guardrail checkpoint should exist"); + + assert_eq!(checkpoint.consecutive_count(), 3); + + issue.blockers = vec![sample_blocker("issue-open", "PUB-105", "Done")]; + + let tracker = FakeTracker::new(vec![issue.clone()]); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("resolved blocker snapshot should build"); + let candidate = + snapshot.queued_candidates.first().expect("ready queued issue should exist"); + + assert_eq!(candidate.reason, "eligible_for_dispatch"); + assert!( + state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "dependency_program_stale") + .expect("cleared dependency checkpoint should read") + .is_none() + ); + + issue.blockers = vec![sample_blocker("issue-open", "PUB-105", "Todo")]; + + let tracker = FakeTracker::new(vec![issue.clone()]); + let snapshot = orchestrator::build_live_operator_status_snapshot( + &tracker, + &config, + &workflow, + &state_store, + 10, + ) + .expect("recurring blocker snapshot should build"); + let candidate = + snapshot.queued_candidates.first().expect("blocked queued issue should exist"); + + assert_eq!(candidate.reason, "open_tracker_blockers"); + assert_eq!(candidate.blocker_identifiers, vec![String::from("PUB-105")]); +} + #[test] fn live_operator_status_snapshot_excludes_claimed_candidates_from_waiting_intake_count() { let (_temp_dir, config, workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index da8167523..f70f2cfb3 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -43,6 +43,7 @@ fn terminal_failures_without_needs_attention_label_use_nonstartable_guard_state( issue_identifier: issue.identifier.clone(), label: String::from("decodex:needs-attention"), run_id: issue_run.run_id.clone(), + error_class: None, }); fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); @@ -124,6 +125,7 @@ fn terminal_failures_apply_incremental_label_mutations_when_issue_labels_paginat issue_identifier: issue.identifier.clone(), label: String::from("decodex:needs-attention"), run_id: issue_run.run_id.clone(), + error_class: None, }); fs::create_dir_all(&issue_run.worktree.path).expect("worktree path should exist"); @@ -729,6 +731,7 @@ fn explicit_manual_attention_keeps_manual_terminal_path_with_dirty_worktree() { issue_identifier: issue.identifier.clone(), label: String::from("decodex:needs-attention"), run_id: issue_run.run_id.clone(), + error_class: None, }); state_store diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs index f1bda98b5..761ca4c7d 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/failure.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -4,6 +4,8 @@ use orchestrator::{ AgentGitCredentialEnvironment, AgentGitCredentialsUnavailable, RepoGateFailureKind, }; use orchestrator::AppServerZeroEvidenceStartFailure; +use orchestrator::LoopGuardrailReason; +use orchestrator::LoopGuardrailStopRequested; fn git_config_value( repo_root: &Path, @@ -38,6 +40,32 @@ fn injected_git_config_keys(credentials: &AgentGitCredentialEnvironment) -> Vec< .collect() } +fn loop_guardrail_issue_run( + config: &ServiceConfig, + issue: &TrackerIssue, + attempt_number: i64, +) -> IssueRunPlan { + IssueRunPlan { + issue: issue.clone(), + issue_state: issue.state.name.clone(), + initial_issue_state: issue.state.name.clone(), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.repo_root().to_path_buf(), + reused_existing: true, + }, + retry_project_slug: issue + .project_slug + .clone() + .expect("sample issue should carry a project slug"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number, + run_id: format!("pub-101-attempt-{attempt_number}-123"), + retry_budget_base: 0, + } +} + #[test] fn terminal_failure_comments_surface_actionable_error_classes() { for (error_class, next_action, expected_snippets) in [ @@ -76,6 +104,112 @@ fn terminal_failure_comments_surface_actionable_error_classes() { } } +#[test] +fn loop_guardrail_terminal_failure_details_normalize_stop_classes() { + let recovery_gate = "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired"; + + for (reason, error_class, expected_snippet) in [ + ( + LoopGuardrailReason::ValidationRepeat, + "validation_repeat", + "repeated validation failure", + ), + ( + LoopGuardrailReason::NoEffectiveDiff, + "no_effective_diff", + "do not continue automatic repair", + ), + ( + LoopGuardrailReason::RemainingDeltaUnchanged, + "remaining_delta_unchanged", + "unchanged remaining delta", + ), + ( + LoopGuardrailReason::ReviewChurn, + "review_churn", + "repeated review findings", + ), + ( + LoopGuardrailReason::DependencyProgramStale, + "dependency_program_stale", + "Execution Program readiness", + ), + ( + LoopGuardrailReason::UncoveredDirection, + "uncovered_direction", + "research or decision contract", + ), + ( + LoopGuardrailReason::AmbiguousRetainedProgress, + "ambiguous_retained_progress", + "retained partial progress", + ), + ] { + let error = Report::new(LoopGuardrailStopRequested { + issue_identifier: String::from("PUB-101"), + run_id: String::from("pub-101-attempt-3-123"), + reason, + consecutive_count: 3, + fingerprint: String::from("fingerprint"), + source_error_class: Some(String::from("repo_gate_verify_failed")), + }); + let (actual_error_class, next_action) = + orchestrator::terminal_failure_comment_details(false, &error, recovery_gate); + + assert_eq!(actual_error_class, error_class); + assert!(next_action.contains(expected_snippet), "{error_class} missing expected action"); + assert!(next_action.contains("clear label `decodex:needs-attention`")); + } +} + +#[test] +fn manual_attention_loop_error_classes_preserve_runtime_attribution() { + let recovery_gate = "clear label `decodex:needs-attention`, then move the issue back to a startable state if another automated run is desired"; + + for (source_error_class, expected_error_class, expected_snippet) in [ + ("review_policy_exhausted", "review_churn", "repeated review findings"), + ( + "dependency_blocked", + "dependency_program_stale", + "Execution Program readiness", + ), + ( + "research_contract_required", + "uncovered_direction", + "research or decision contract", + ), + ( + "ownership_ambiguous", + "ambiguous_retained_progress", + "retained partial progress", + ), + ] { + let error = Report::new(ManualAttentionRequested { + issue_identifier: String::from("PUB-101"), + label: String::from("decodex:needs-attention"), + run_id: String::from("pub-101-attempt-1-123"), + error_class: Some(String::from(source_error_class)), + }); + let (actual_error_class, next_action) = + orchestrator::terminal_failure_comment_details(true, &error, recovery_gate); + + assert_eq!(actual_error_class, expected_error_class); + assert!(next_action.contains(expected_snippet), "{expected_error_class} action missing"); + } + + let generic = Report::new(ManualAttentionRequested { + issue_identifier: String::from("PUB-101"), + label: String::from("decodex:needs-attention"), + run_id: String::from("pub-101-attempt-1-123"), + error_class: Some(String::from("operator_requested_stop")), + }); + let (actual_error_class, next_action) = + orchestrator::terminal_failure_comment_details(true, &generic, recovery_gate); + + assert_eq!(actual_error_class, "human_attention_required"); + assert!(next_action.contains("inspect the issue comment and worktree")); +} + #[test] fn terminal_failure_comments_surface_review_handoff_pr_url_when_available() { let pr_url = "https://github.com/hack-ink/decodex/pull/101"; @@ -471,6 +605,171 @@ fn repo_gate_lock_contention_terminal_failures_preserve_specific_error_class_aft assert!(next_action.contains("active or stale `.git/index.lock` holder")); } +#[test] +fn loop_guardrail_stops_repeated_validation_failures_after_three_observations() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let error = || { + Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + String::from("Repo verify command `cargo make test` failed: same assertion failed"), + )) + }; + + for attempt_number in 1..=2 { + let issue_run = loop_guardrail_issue_run(&config, &issue, attempt_number); + + assert!( + orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error(), + ) + .expect("guardrail observation should persist") + .is_none(), + "guardrail should allow repair attempt {attempt_number}" + ); + } + + let issue_run = loop_guardrail_issue_run(&config, &issue, 3); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error(), + ) + .expect("third matching failure should evaluate") + .expect("third matching validation failure should stop"); + + assert_eq!(stop.reason, orchestrator::LoopGuardrailReason::ValidationRepeat); + assert_eq!(stop.consecutive_count, 3); + assert_eq!(stop.source_error_class.as_deref(), Some("repo_gate_verify_failed")); + + let checkpoint = state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "validation_repeat") + .expect("validation checkpoint should read") + .expect("validation checkpoint should exist"); + + assert_eq!(checkpoint.consecutive_count(), 3); + + let events = state_store + .list_private_execution_events(config.service_id(), &issue.id, &issue_run.run_id, 3) + .expect("private events should list"); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].event_type(), "loop_guardrail_checkpoint"); + assert_eq!(events[0].payload()["reason"], "validation_repeat"); +} + +#[test] +fn loop_guardrail_stops_unchanged_remaining_delta_when_validation_text_changes() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + + for attempt_number in 1..=2 { + let issue_run = loop_guardrail_issue_run(&config, &issue, attempt_number); + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + format!("Repo verify command failed with assertion variant {attempt_number}"), + )); + + assert!( + orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("guardrail observation should persist") + .is_none() + ); + } + + let issue_run = loop_guardrail_issue_run(&config, &issue, 3); + let error = Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + String::from("Repo verify command failed with assertion variant 3"), + )); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("third unchanged delta should evaluate") + .expect("unchanged remaining delta should stop"); + + assert_eq!(stop.reason, orchestrator::LoopGuardrailReason::RemainingDeltaUnchanged); + assert_eq!(stop.consecutive_count, 3); + assert_eq!(stop.source_error_class.as_deref(), Some("repo_gate_verify_failed")); + + let validation_checkpoint = state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "validation_repeat") + .expect("validation checkpoint should read") + .expect("validation checkpoint should exist"); + + assert_eq!( + validation_checkpoint.consecutive_count(), + 1, + "changing validation text should keep validation_repeat below threshold" + ); + + let delta_checkpoint = state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "remaining_delta_unchanged") + .expect("remaining-delta checkpoint should read") + .expect("remaining-delta checkpoint should exist"); + + assert_eq!(delta_checkpoint.consecutive_count(), 3); +} + +#[test] +fn loop_guardrail_stops_no_effective_diff_for_retryable_errors_without_delta() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + + for attempt_number in 1..=2 { + let issue_run = loop_guardrail_issue_run(&config, &issue, attempt_number); + let error = Report::msg("child exited without useful change"); + + assert!( + orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("guardrail observation should persist") + .is_none() + ); + } + + let issue_run = loop_guardrail_issue_run(&config, &issue, 3); + let error = Report::msg("child exited without useful change"); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("third no-diff observation should evaluate") + .expect("no effective diff should stop"); + + assert_eq!(stop.reason, orchestrator::LoopGuardrailReason::NoEffectiveDiff); + assert_eq!(stop.consecutive_count, 3); + assert_eq!(stop.source_error_class, None); + + let checkpoint = state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "no_effective_diff") + .expect("no-diff checkpoint should read") + .expect("no-diff checkpoint should exist"); + + assert_eq!(checkpoint.consecutive_count(), 3); +} + #[test] fn app_server_terminal_failures_preserve_specific_error_classes() { let cases = [ diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index b9a391fd7..d6cfc1e47 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -579,6 +579,7 @@ struct ManualAttentionRequested { issue_identifier: String, label: String, run_id: String, + error_class: Option, } impl Display for ManualAttentionRequested { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { @@ -645,6 +646,103 @@ impl Display for RetainedPartialProgress { impl Error for RetainedPartialProgress {} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LoopGuardrailReason { + ValidationRepeat, + NoEffectiveDiff, + RemainingDeltaUnchanged, + ReviewChurn, + DependencyProgramStale, + UncoveredDirection, + AmbiguousRetainedProgress, +} +impl LoopGuardrailReason { + fn error_class(self) -> &'static str { + match self { + Self::ValidationRepeat => "validation_repeat", + Self::NoEffectiveDiff => "no_effective_diff", + Self::RemainingDeltaUnchanged => "remaining_delta_unchanged", + Self::ReviewChurn => "review_churn", + Self::DependencyProgramStale => "dependency_program_stale", + Self::UncoveredDirection => "uncovered_direction", + Self::AmbiguousRetainedProgress => "ambiguous_retained_progress", + } + } + + fn from_error_class(error_class: &str) -> Option { + match error_class { + "validation_repeat" | "validation_failure_repeated" => Some(Self::ValidationRepeat), + "no_effective_diff" => Some(Self::NoEffectiveDiff), + "remaining_delta_unchanged" => Some(Self::RemainingDeltaUnchanged), + "review_churn" | "review_policy_exhausted" => Some(Self::ReviewChurn), + "dependency_program_stale" | "dependency_blocked" => { + Some(Self::DependencyProgramStale) + }, + "uncovered_direction" | "research_contract_required" => { + Some(Self::UncoveredDirection) + }, + "ambiguous_retained_progress" | "ownership_ambiguous" => { + Some(Self::AmbiguousRetainedProgress) + }, + _ => None, + } + } + + fn terminal_next_action(self, recovery_gate: &str) -> String { + match self { + Self::ValidationRepeat => format!( + "inspect the repeated validation failure, preserved worktree, and prior repair attempts; change repair strategy or route the issue to architecture/research review manually, {recovery_gate}" + ), + Self::NoEffectiveDiff => format!( + "inspect the retained worktree and retry evidence; do not continue automatic repair until a human identifies a concrete next diff or resets the lane, {recovery_gate}" + ), + Self::RemainingDeltaUnchanged => format!( + "inspect the unchanged remaining delta and validation evidence; decide the next bounded repair manually before requeueing, {recovery_gate}" + ), + Self::ReviewChurn => format!( + "inspect the repeated review findings and current head; decide the next repair or architecture review manually before requeueing, {recovery_gate}" + ), + Self::DependencyProgramStale => format!( + "inspect the dependency blocker and Execution Program readiness evidence; refresh dependencies or split/research the program before requeueing, {recovery_gate}" + ), + Self::UncoveredDirection => format!( + "capture the missing direction in a research or decision contract before continuing execution, {recovery_gate}" + ), + Self::AmbiguousRetainedProgress => format!( + "inspect retained partial progress and ownership evidence; choose resume, reset, or manual repair explicitly before clearing the guard, {recovery_gate}" + ), + } + } +} + +#[derive(Debug)] +struct LoopGuardrailStopRequested { + issue_identifier: String, + run_id: String, + reason: LoopGuardrailReason, + consecutive_count: i64, + fingerprint: String, + source_error_class: Option, +} +impl Display for LoopGuardrailStopRequested { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let source = self.source_error_class.as_deref().unwrap_or("none"); + + write!( + f, + "Run `{}` for issue `{}` hit loop guardrail `{}` after {} consecutive matching observations with source `{}` and fingerprint `{}`; stop automatic retries.", + self.run_id, + self.issue_identifier, + self.reason.error_class(), + self.consecutive_count, + source, + self.fingerprint + ) + } +} + +impl Error for LoopGuardrailStopRequested {} + #[derive(Debug)] struct AgentGitCredentialsUnavailable { run_id: String, diff --git a/apps/decodex/src/state/internal.rs b/apps/decodex/src/state/internal.rs index 1791c727c..e6be83a97 100644 --- a/apps/decodex/src/state/internal.rs +++ b/apps/decodex/src/state/internal.rs @@ -131,6 +131,7 @@ struct StateData { review_handoffs: HashMap, review_orchestrations: HashMap, review_policy_checkpoints: HashMap, + loop_guardrail_checkpoints: HashMap, connector_backoffs: HashMap<(String, String), ConnectorBackoff>, dispatch_slot_configs: HashMap, issue_claim_guards: HashMap, @@ -151,6 +152,7 @@ impl StateData { self.review_handoffs = loaded.review_handoffs; self.review_orchestrations = loaded.review_orchestrations; self.review_policy_checkpoints = loaded.review_policy_checkpoints; + self.loop_guardrail_checkpoints = loaded.loop_guardrail_checkpoints; self.connector_backoffs = loaded.connector_backoffs; } @@ -354,6 +356,7 @@ ON linear_execution_events (service_id, issue_id, event_unix, recorded_at_unix); self.bootstrap_connector_backoffs_schema()?; self.bootstrap_private_execution_events_schema()?; self.bootstrap_decision_contracts_schema()?; + self.bootstrap_loop_guardrail_schema()?; self.record_schema_version()?; Ok(()) @@ -459,6 +462,28 @@ ON run_control_channels (project_id, issue_id, attempt_number); Ok(()) } + fn bootstrap_loop_guardrail_schema(&self) -> Result<()> { + self.connection.execute_batch( + r#" +CREATE TABLE IF NOT EXISTS loop_guardrail_checkpoints ( + project_id TEXT NOT NULL, + issue_id TEXT NOT NULL, + reason TEXT NOT NULL, + fingerprint TEXT NOT NULL, + run_id TEXT NOT NULL, + attempt_number INTEGER NOT NULL, + consecutive_count INTEGER NOT NULL, + details_json TEXT NOT NULL, + updated_at TEXT NOT NULL, + updated_at_unix INTEGER NOT NULL, + PRIMARY KEY (project_id, issue_id, reason) +); +"#, + )?; + + Ok(()) + } + fn bootstrap_connector_backoffs_schema(&self) -> Result<()> { self.connection.execute_batch( r#" @@ -560,6 +585,7 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; self.load_review_handoffs(&mut state)?; self.load_review_orchestrations(&mut state)?; self.load_review_policy_checkpoints(&mut state)?; + self.load_loop_guardrail_checkpoints(&mut state)?; self.load_connector_backoffs(&mut state)?; Ok(state) @@ -600,6 +626,7 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; persist_review_handoffs(&transaction, state)?; persist_review_orchestrations(&transaction, state)?; persist_review_policy_checkpoints(&transaction, state)?; + persist_loop_guardrail_checkpoints(&transaction, state)?; persist_connector_backoffs(&transaction, state)?; transaction.commit()?; @@ -623,6 +650,10 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; "DELETE FROM decision_contracts WHERE project_id = ?1", params![service_id], )?; + transaction.execute( + "DELETE FROM loop_guardrail_checkpoints WHERE project_id = ?1", + params![service_id], + )?; transaction.commit()?; Ok(()) @@ -874,6 +905,20 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; "UPDATE decision_contracts SET source_issue_id = ?2 WHERE source_issue_id = ?1", params![previous_issue_id, canonical_issue_id], )?; + transaction.execute( + "INSERT OR IGNORE INTO loop_guardrail_checkpoints ( + project_id, issue_id, reason, fingerprint, run_id, attempt_number, + consecutive_count, details_json, updated_at, updated_at_unix + ) + SELECT project_id, ?2, reason, fingerprint, run_id, attempt_number, + consecutive_count, details_json, updated_at, updated_at_unix + FROM loop_guardrail_checkpoints WHERE issue_id = ?1", + params![previous_issue_id, canonical_issue_id], + )?; + transaction.execute( + "DELETE FROM loop_guardrail_checkpoints WHERE issue_id = ?1", + params![previous_issue_id], + )?; transaction.execute( "INSERT OR IGNORE INTO review_policy_checkpoints ( project_id, issue_id, run_id, attempt_number, phase, status, head_sha, @@ -938,6 +983,10 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; "DELETE FROM review_policy_checkpoints WHERE issue_id = ?1", params![issue_id], )?; + transaction.execute( + "DELETE FROM loop_guardrail_checkpoints WHERE issue_id = ?1", + params![issue_id], + )?; transaction.commit()?; Ok(()) @@ -974,6 +1023,34 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; Ok(()) } + fn delete_loop_guardrail_checkpoints_for_issue( + &mut self, + project_id: &str, + issue_id: &str, + ) -> Result<()> { + self.connection.execute( + "DELETE FROM loop_guardrail_checkpoints WHERE project_id = ?1 AND issue_id = ?2", + params![project_id, issue_id], + )?; + + Ok(()) + } + + fn delete_loop_guardrail_checkpoint( + &mut self, + project_id: &str, + issue_id: &str, + reason: &str, + ) -> Result<()> { + self.connection.execute( + "DELETE FROM loop_guardrail_checkpoints \ + WHERE project_id = ?1 AND issue_id = ?2 AND reason = ?3", + params![project_id, issue_id, reason], + )?; + + Ok(()) + } + fn delete_review_policy_checkpoints_for_run_attempt( &mut self, project_id: &str, @@ -1689,6 +1766,43 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value; Ok(()) } + fn load_loop_guardrail_checkpoints(&self, state: &mut StateData) -> Result<()> { + let mut statement = self.connection.prepare( + "SELECT project_id, issue_id, reason, fingerprint, run_id, attempt_number, \ + consecutive_count, details_json, updated_at, updated_at_unix \ + FROM loop_guardrail_checkpoints", + )?; + let rows = statement.query_map([], |row| { + let project_id: String = row.get(0)?; + let issue_id: String = row.get(1)?; + let reason: String = row.get(2)?; + + Ok(( + LoopGuardrailKey::new(&project_id, &issue_id, &reason), + LoopGuardrailRuntimeRecord { + project_id, + issue_id, + reason, + fingerprint: row.get(3)?, + run_id: row.get(4)?, + attempt_number: row.get(5)?, + consecutive_count: row.get(6)?, + details_json: row.get(7)?, + updated_at: row.get(8)?, + updated_at_unix: row.get(9)?, + }, + )) + })?; + + for row in rows { + let (key, record) = row?; + + state.loop_guardrail_checkpoints.insert(key, record); + } + + Ok(()) + } + fn load_connector_backoffs(&self, state: &mut StateData) -> Result<()> { let mut statement = self.connection.prepare( "SELECT project_id, connector, sync_phase, quota_class, reset_unix_epoch, \ @@ -2037,6 +2151,52 @@ impl ReviewPolicyRuntimeRecord { } } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct LoopGuardrailKey { + project_id: String, + issue_id: String, + reason: String, +} +impl LoopGuardrailKey { + fn new(project_id: &str, issue_id: &str, reason: &str) -> Self { + Self { + project_id: project_id.to_owned(), + issue_id: issue_id.to_owned(), + reason: reason.to_owned(), + } + } +} + +#[derive(Clone, Debug)] +struct LoopGuardrailRuntimeRecord { + project_id: String, + issue_id: String, + reason: String, + fingerprint: String, + run_id: String, + attempt_number: i64, + consecutive_count: i64, + details_json: String, + updated_at: String, + updated_at_unix: i64, +} +impl LoopGuardrailRuntimeRecord { + fn as_public(&self) -> LoopGuardrailCheckpoint { + LoopGuardrailCheckpoint { + project_id: self.project_id.clone(), + issue_id: self.issue_id.clone(), + reason: self.reason.clone(), + fingerprint: self.fingerprint.clone(), + run_id: self.run_id.clone(), + attempt_number: self.attempt_number, + consecutive_count: self.consecutive_count, + details_json: self.details_json.clone(), + updated_at: self.updated_at.clone(), + updated_at_unix: self.updated_at_unix, + } + } +} + #[derive(Clone, Default)] struct RunActivityMarkerRecord { run_id: Option, @@ -2997,6 +3157,34 @@ fn persist_review_policy_checkpoints( Ok(()) } +fn persist_loop_guardrail_checkpoints( + transaction: &Transaction<'_>, + state: &StateData, +) -> Result<()> { + for record in state.loop_guardrail_checkpoints.values() { + transaction.execute( + "INSERT OR REPLACE INTO loop_guardrail_checkpoints ( + project_id, issue_id, reason, fingerprint, run_id, attempt_number, + consecutive_count, details_json, updated_at, updated_at_unix + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + record.project_id, + record.issue_id, + record.reason, + record.fingerprint, + record.run_id, + record.attempt_number, + record.consecutive_count, + record.details_json, + record.updated_at, + record.updated_at_unix, + ], + )?; + } + + Ok(()) +} + fn persist_connector_backoffs(transaction: &Transaction<'_>, state: &StateData) -> Result<()> { for record in state.connector_backoffs.values() { transaction.execute( diff --git a/apps/decodex/src/state/models.rs b/apps/decodex/src/state/models.rs index ef52f6549..6d13beb1a 100644 --- a/apps/decodex/src/state/models.rs +++ b/apps/decodex/src/state/models.rs @@ -611,6 +611,66 @@ impl ReviewPolicyCheckpoint { } } +/// Latest loop-guardrail checkpoint for one issue and stop reason. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct LoopGuardrailCheckpoint { + project_id: String, + issue_id: String, + reason: String, + fingerprint: String, + run_id: String, + attempt_number: i64, + consecutive_count: i64, + details_json: String, + updated_at: String, + updated_at_unix: i64, +} +impl LoopGuardrailCheckpoint { + #[cfg(test)] + pub(crate) fn project_id(&self) -> &str { + &self.project_id + } + + #[cfg(test)] + pub(crate) fn issue_id(&self) -> &str { + &self.issue_id + } + + pub(crate) fn reason(&self) -> &str { + &self.reason + } + + pub(crate) fn fingerprint(&self) -> &str { + &self.fingerprint + } + + pub(crate) fn run_id(&self) -> &str { + &self.run_id + } + + pub(crate) fn attempt_number(&self) -> i64 { + self.attempt_number + } + + pub(crate) fn consecutive_count(&self) -> i64 { + self.consecutive_count + } + + pub(crate) fn details_json(&self) -> &str { + &self.details_json + } + + #[cfg(test)] + pub(crate) fn updated_at(&self) -> &str { + &self.updated_at + } + + #[cfg(test)] + pub(crate) fn updated_at_unix(&self) -> i64 { + self.updated_at_unix + } +} + /// Foundation request for resolving a local run-control action. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[cfg_attr(not(test), allow(dead_code))] diff --git a/apps/decodex/src/state/store.rs b/apps/decodex/src/state/store.rs index 2d3975325..bf87b84a4 100644 --- a/apps/decodex/src/state/store.rs +++ b/apps/decodex/src/state/store.rs @@ -24,6 +24,17 @@ pub(crate) struct ReviewPolicyCheckpointInput<'a> { pub(crate) details_json: &'a str, } +/// Input fields for recording the latest loop-guardrail checkpoint. +pub(crate) struct LoopGuardrailCheckpointInput<'a> { + pub(crate) project_id: &'a str, + pub(crate) issue_id: &'a str, + pub(crate) reason: &'a str, + pub(crate) fingerprint: &'a str, + pub(crate) run_id: &'a str, + pub(crate) attempt_number: i64, + pub(crate) details_json: &'a str, +} + /// Local runtime store for leases, attempts, worktrees, protocol events, and private evidence. #[derive(Default)] pub struct StateStore { @@ -227,6 +238,11 @@ impl StateStore { previous_issue_id, canonical_issue_id, ); + retarget_loop_guardrail_issue( + &mut state.loop_guardrail_checkpoints, + previous_issue_id, + canonical_issue_id, + ); if let Some(guard) = state.issue_claim_guards.remove(previous_issue_id) { state.issue_claim_guards.entry(canonical_issue_id.to_owned()).or_insert(guard); @@ -1895,6 +1911,85 @@ impl StateStore { ) } + /// Record one loop-guardrail observation and return its consecutive count. + pub(crate) fn observe_loop_guardrail_checkpoint( + &self, + input: LoopGuardrailCheckpointInput<'_>, + ) -> Result { + let now = timestamp_parts(); + let key = LoopGuardrailKey::new(input.project_id, input.issue_id, input.reason); + let mut state = self.lock()?; + let previous = state.loop_guardrail_checkpoints.get(&key); + let consecutive_count = previous.map_or(1, |record| { + if record.fingerprint == input.fingerprint { + record.consecutive_count.saturating_add(1) + } else { + 1 + } + }); + let record = LoopGuardrailRuntimeRecord { + project_id: input.project_id.to_owned(), + issue_id: input.issue_id.to_owned(), + reason: input.reason.to_owned(), + fingerprint: input.fingerprint.to_owned(), + run_id: input.run_id.to_owned(), + attempt_number: input.attempt_number, + consecutive_count, + details_json: input.details_json.to_owned(), + updated_at: now.text, + updated_at_unix: now.unix, + }; + + state.loop_guardrail_checkpoints.insert(key, record.clone()); + self.persist_runtime_state_locked(&state)?; + + Ok(record.as_public()) + } + + /// Read one loop-guardrail checkpoint by project, issue, and reason. + #[cfg(test)] + pub(crate) fn loop_guardrail_checkpoint( + &self, + project_id: &str, + issue_id: &str, + reason: &str, + ) -> Result> { + let state = self.lock()?; + let key = LoopGuardrailKey::new(project_id, issue_id, reason); + + Ok(state.loop_guardrail_checkpoints.get(&key).map(LoopGuardrailRuntimeRecord::as_public)) + } + + /// Clear loop-guardrail checkpoints for one issue. + pub(crate) fn clear_loop_guardrail_checkpoints_for_issue( + &self, + project_id: &str, + issue_id: &str, + ) -> Result<()> { + let mut state = self.lock()?; + + state.loop_guardrail_checkpoints.retain(|key, _record| { + key.project_id != project_id || key.issue_id != issue_id + }); + + self.delete_loop_guardrail_checkpoints_for_issue_locked(project_id, issue_id) + } + + /// Clear one loop-guardrail checkpoint reason for one issue. + pub(crate) fn clear_loop_guardrail_checkpoint( + &self, + project_id: &str, + issue_id: &str, + reason: &str, + ) -> Result<()> { + let key = LoopGuardrailKey::new(project_id, issue_id, reason); + let mut state = self.lock()?; + + state.loop_guardrail_checkpoints.remove(&key); + + self.delete_loop_guardrail_checkpoint_locked(project_id, issue_id, reason) + } + /// Remove the exact retained review markers created for one handoff identity. pub(crate) fn clear_review_markers_for_handoff( &self, @@ -2278,6 +2373,37 @@ impl StateStore { attempt_number, ) } + + fn delete_loop_guardrail_checkpoints_for_issue_locked( + &self, + project_id: &str, + issue_id: &str, + ) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_loop_guardrail_checkpoints_for_issue(project_id, issue_id) + } + + fn delete_loop_guardrail_checkpoint_locked( + &self, + project_id: &str, + issue_id: &str, + reason: &str, + ) -> Result<()> { + let Some(sqlite) = self.sqlite.as_ref() else { + return Ok(()); + }; + let mut sqlite = sqlite + .lock() + .map_err(|_| eyre::eyre!("StateStore SQLite mutex is poisoned."))?; + + sqlite.delete_loop_guardrail_checkpoint(project_id, issue_id, reason) + } } #[cfg_attr(not(test), allow(dead_code))] @@ -2416,6 +2542,30 @@ fn retarget_review_policy_issue( } } +fn retarget_loop_guardrail_issue( + records: &mut HashMap, + previous_issue_id: &str, + canonical_issue_id: &str, +) { + let previous_keys = records + .keys() + .filter(|key| key.issue_id == previous_issue_id) + .cloned() + .collect::>(); + + for key in previous_keys { + let Some(mut record) = records.remove(&key) else { + continue; + }; + + record.issue_id = canonical_issue_id.to_owned(); + + records + .entry(LoopGuardrailKey::new(&key.project_id, canonical_issue_id, &key.reason)) + .or_insert(record); + } +} + fn active_run_attempt_status(status: &str) -> bool { matches!(status, "starting" | "running") } diff --git a/apps/decodex/src/state/tests.rs b/apps/decodex/src/state/tests.rs index 5c65f1b88..531362cd1 100644 --- a/apps/decodex/src/state/tests.rs +++ b/apps/decodex/src/state/tests.rs @@ -18,12 +18,13 @@ use crate::{ }, state::{ self, ChildAgentActivitySummary, CodexAccountActivitySummary, CodexAccountMarker, - ConnectorBackoffInput, DispatchSlotLimit, EffectiveRuntimeMarker, PreacquiredLeaseGuards, - ProjectRegistration, ProtocolActivityMarker, ProtocolActivitySummary, - RUN_ACTIVITY_MARKER_FILE, RUN_CONTROL_ACTION_COMPLETED, RUN_CONTROL_ACTION_FAILED, - RUN_CONTROL_ACTION_FALLBACK, RUN_CONTROL_ACTION_TIMED_OUT, RUN_OPERATION_REPO_GATE, - ReviewHandoffMarker, ReviewOrchestrationMarker, ReviewPolicyCheckpointInput, - RunControlActionRequest, StateStore, + ConnectorBackoffInput, DispatchSlotLimit, EffectiveRuntimeMarker, + LoopGuardrailCheckpointInput, PreacquiredLeaseGuards, ProjectRegistration, + ProtocolActivityMarker, ProtocolActivitySummary, RUN_ACTIVITY_MARKER_FILE, + RUN_CONTROL_ACTION_COMPLETED, RUN_CONTROL_ACTION_FAILED, RUN_CONTROL_ACTION_FALLBACK, + RUN_CONTROL_ACTION_TIMED_OUT, RUN_OPERATION_REPO_GATE, ReviewHandoffMarker, + ReviewOrchestrationMarker, ReviewPolicyCheckpointInput, RunControlActionRequest, + StateStore, }, tracker::records::{LinearExecutionEventIdentity, LinearExecutionEventRecord}, }; @@ -126,6 +127,92 @@ fn upsert_handoff_review_policy_checkpoint( .expect("review policy checkpoint should persist"); } +#[test] +fn loop_guardrail_checkpoints_track_fingerprints_and_retarget_issue() { + let store = StateStore::open_in_memory().expect("state store should open"); + let first = store + .observe_loop_guardrail_checkpoint(LoopGuardrailCheckpointInput { + project_id: "pubfi", + issue_id: "PUB-101", + reason: "validation_repeat", + fingerprint: "fp-a", + run_id: "run-1", + attempt_number: 1, + details_json: "{}", + }) + .expect("first loop guardrail observation should persist"); + + assert_eq!(first.consecutive_count(), 1); + assert_eq!(first.reason(), "validation_repeat"); + + let second = store + .observe_loop_guardrail_checkpoint(LoopGuardrailCheckpointInput { + project_id: "pubfi", + issue_id: "PUB-101", + reason: "validation_repeat", + fingerprint: "fp-a", + run_id: "run-2", + attempt_number: 2, + details_json: "{\"attempt\":2}", + }) + .expect("same fingerprint should increment"); + + assert_eq!(second.consecutive_count(), 2); + assert_eq!(second.run_id(), "run-2"); + assert_eq!(second.attempt_number(), 2); + assert!(second.updated_at_unix() > 0); + + let reset = store + .observe_loop_guardrail_checkpoint(LoopGuardrailCheckpointInput { + project_id: "pubfi", + issue_id: "PUB-101", + reason: "validation_repeat", + fingerprint: "fp-b", + run_id: "run-3", + attempt_number: 3, + details_json: "{\"attempt\":3}", + }) + .expect("new fingerprint should reset"); + + assert_eq!(reset.consecutive_count(), 1); + assert_eq!(reset.fingerprint(), "fp-b"); + assert_eq!(reset.details_json(), "{\"attempt\":3}"); + assert!(!reset.updated_at().is_empty()); + + store + .canonicalize_issue_identity("PUB-101", "linear-id-101") + .expect("issue identity should retarget"); + + assert!( + store + .loop_guardrail_checkpoint("pubfi", "PUB-101", "validation_repeat") + .expect("old checkpoint should read") + .is_none(), + "legacy issue identity should be cleared after retarget" + ); + + let canonical = store + .loop_guardrail_checkpoint("pubfi", "linear-id-101", "validation_repeat") + .expect("canonical checkpoint should read") + .expect("canonical checkpoint should exist"); + + assert_eq!(canonical.project_id(), "pubfi"); + assert_eq!(canonical.issue_id(), "linear-id-101"); + assert_eq!(canonical.fingerprint(), "fp-b"); + assert_eq!(canonical.consecutive_count(), 1); + + store + .clear_loop_guardrail_checkpoints_for_issue("pubfi", "linear-id-101") + .expect("checkpoint should clear"); + + assert!( + store + .loop_guardrail_checkpoint("pubfi", "linear-id-101", "validation_repeat") + .expect("cleared checkpoint should read") + .is_none() + ); +} + #[test] fn review_markers_roundtrip_preserve_required_fields() { let store = StateStore::open_in_memory().expect("state store should open"); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 7d055b940..c49286845 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -148,7 +148,7 @@ work that needs full private payload values. | Surface | Owns | Does Not Own | | --- | --- | --- | -| Runtime SQLite DB | active leases, attempts, run-control channels, protocol events, private execution events, Decision Contracts, worktree mappings, retry state, retained PR state, review-policy checkpoints with structured independent-review detail, phase-goal signals, phase timing, connector backoff, project registry | human backlog grooming or durable team-visible issue history | +| Runtime SQLite DB | active leases, attempts, run-control channels, protocol events, private execution events, Decision Contracts, worktree mappings, retry state, retained PR state, review-policy checkpoints with structured independent-review detail, loop-guardrail checkpoints, phase-goal signals, phase timing, connector backoff, project registry | human backlog grooming or durable team-visible issue history | | Central project config | `service_id`, repo root, worktree root, tracker/GitHub credential env-var names, enabled project registration | per-run state or issue ownership | | Project `WORKFLOW.md` | repo policy, validation gate, state names, retry/review policy | runtime ownership, queue labels, credentials, model overrides | | Linear | team-visible issue state, queue/active/manual-attention labels, coarse execution ledger comments, progress/failure/handoff/closeout summaries | high-frequency runtime truth, heartbeat, token pressure, raw attempts, private execution evidence, connector retry budgets | @@ -237,7 +237,7 @@ protocol activity durable outside the local operator surface. | `Accounts` | Shared Codex account pool and usage table from `~/.codex/decodex/accounts.jsonl` when `[codex.accounts]` is enabled for a project. Account identity can be obscured from the `Account` column header eye without changing the underlying snapshot. The row weight column shows the capacity multiplier used for pool usage estimates: `pro` accounts count as `20x`, and all other plans count as `1x`. Usage probes read Codex `/wham/usage` for window capacity and `/wham/profiles/me` for profile token stats such as lifetime tokens, peak daily tokens, longest task, streaks, and daily token activity. Selecting an account writes the global `[codex.accounts].fixed_account` selector in `~/.codex/decodex/config.toml`; clearing it returns all new account-pool runs to balanced account selection. Account display-name rerolls write `[codex.account_names.offsets]` in the same global config so Decodex App and the dashboard share the privacy-preserving names. Theme, sort, and identity-visibility preferences are client-local presentation state. The selector is global and does not pin a project to an account. | | `Projects` | Fleet-level project table. The section-level filter toggles between active project work and the full registry. Location is its own compact path column and can be obscured from the location header eye. `Activity` shows a relative timestamp or `-`; `Work` is `running/waiting/attention`. It should not duplicate per-lane details already shown below. | | `Running Lanes` | Active leased or live-executing issue lanes. A lane here is currently owned by this local control plane, or a live process/thread/protocol marker still explains active execution even when the queue lease is not held. It shows issue identity, phase, operation, attempt, queue lease state, execution liveness, thread/protocol status, child-agent activity when captured, phase-goal status when app-server reports it, timing, branch, and worktree. | -| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after stalled reconciliation, failure writeback, or retries, the candidate is partial retained progress and not just a generic stalled or retry-budget hold. Running lanes are not repeated as normal intake work. | +| `Intake Queue` | Queued tracker issues before execution. Candidates are classified as `ready`, capacity-waiting, claimed without a matching local lane, blocked, or closed/stale. Repeated identical open dependency blockers surface as `dependency_program_stale` after the guardrail threshold so operators can distinguish a stale Execution Program/dependency plan from a newly blocked queue item. A blocked queued candidate can still show an attached `.worktrees/XY-*` path when the queue owns the attention state; if that worktree has tracked changes after stalled reconciliation, failure writeback, or retries, the candidate is partial retained progress and not just a generic stalled or retry-budget hold. Running lanes are not repeated as normal intake work. | | `Review & Landing` | Retained PR lanes after review handoff. This section owns post-review repair, wait-for-review, ready-to-land, closeout, cleanup, and blocked retained-lane visibility. | | `Recovery Worktrees` | Retained local worktrees that are not currently owned by `Running Lanes`, `Review & Landing`, or queued attention in `Intake Queue`. This is the cleanup or recovery inbox for recovered paths, retained PR leftovers, and cleanup-only local worktrees. Empty is the normal healthy state. | | `Run Ledger` | Completed or non-running issue history, grouped by issue/lane. Decodex Linear execution ledger comments provide the durable completed outcome when available. If no `decodex.linear_execution_event` record exists, the row reports `missing` / `execution_ledger_missing`; the control plane does not derive a completed or landed outcome from tracker state, local attempts, or non-ledger comments. Raw local attempts and heartbeat details stay in debug expansion. | @@ -334,6 +334,11 @@ Worktree visibility follows the owning dashboard section: diagnostic phase evidence only; it is not a Run Ledger outcome and does not replace repo validation, bounded review, PR handoff, manual attention, landing, closeout, or terminal finalization. +- Loop guardrail stops may appear as `validation_repeat`, `no_effective_diff`, + `remaining_delta_unchanged`, `review_churn`, `dependency_program_stale`, + `uncovered_direction`, or `ambiguous_retained_progress`. Use the public reason to + choose the recovery path, then inspect `decodex evidence` or local status for + `loop_guardrail_checkpoint` evidence before clearing attention labels or retrying. - `Review & Landing` means a retained PR lane still owns the path for review repair, landing, closeout, or retained-lane cleanup. - `missing_review_handoff_record` in `Review & Landing` means Decodex found a retained @@ -360,6 +365,10 @@ Worktree visibility follows the owning dashboard section: command output. - `Intake Queue` means queued attention still owns the path, including partial retained progress after retries. +- `dependency_program_stale` in `Intake Queue` means the same open blocker fingerprint + has repeated through the guardrail threshold. Refresh the dependency issue, split or + repair the Execution Program, or route research/decision work before requeueing; do + not clear it as a transient queue delay. - `linear_active_label_present` in `Intake Queue` means the issue still carries service active ownership while it is also queued, but local status could not prove a matching active lease. Treat it as a recovery/attention row, not ready work. If its diff --git a/docs/runbook/lane-control-recovery.md b/docs/runbook/lane-control-recovery.md index 9786ae544..54fa313c2 100644 --- a/docs/runbook/lane-control-recovery.md +++ b/docs/runbook/lane-control-recovery.md @@ -88,8 +88,30 @@ or clean labels. | Queue label should be added, removed, or interpreted. | Use service-scoped label policy. | Follow the `labels` skill; do not guess `` or clear `needs-attention` before fixing the blocker. | | Broad steer materially changes the objective or acceptance contract. | Preserve audit and resolve lifecycle explicitly. | Update and requeue the same issue, create a new issue/lane, or route the owned run to manual attention. | | Operator wants a different issue or replacement task. | Treat as task replacement, not steer. | Stop or pause through supported controls as needed, then create/update/requeue through the supported lifecycle. | +| Status or Linear failure summary reports a loop guardrail reason. | Stop automatic recovery and inspect the reason-specific evidence. | Follow the loop guardrail recovery table below before clearing `decodex:needs-attention` or requeueing. | | Evidence is missing, contradictory, or would require guessing whether local work is safe to overwrite. | Stop automatic recovery. | Use manual attention with structured public blockers and keep private evidence local. | +## Loop Guardrail Recovery + +Loop guardrails stop non-converging automation after three consecutive matching +observations. They preserve retained worktrees and private evidence; they do not mean +the operator should delete local progress to make the queue clean. + +| Guardrail reason | Inspect first | Resume only after | +| --- | --- | --- | +| `validation_repeat` | The repeated validation failure, repo-gate output, retained worktree, and prior repair attempts. | The repair strategy changes, the validation cause is fixed manually, or the issue is routed to architecture/research review. | +| `no_effective_diff` | The retained worktree status, private retry evidence, and whether any useful tracked delta exists. | A human identifies a concrete next diff, commits/resets the retained work intentionally, or updates the issue scope. | +| `remaining_delta_unchanged` | The unchanged tracked delta and latest validation evidence. | The next repair is bounded and materially different, or the retained patch is accepted/reset manually. | +| `review_churn` or `review_policy_exhausted` | Fresh-context review checkpoints, accepted findings, rejected findings, and the current head. | A new repair strategy, architecture review, or manual decision is recorded. | +| `dependency_program_stale` | The open blocker issue, Execution Program readiness, and whether the dependency split is still correct. | The dependency is resolved, the program is refreshed/split, or a research/decision contract updates execution authority. | +| `uncovered_direction` | The missing requirement, decision, or research gap named in public/private evidence. | A research or Decision Contract captures the missing direction and the issue is updated or requeued from that authority. | +| `ambiguous_retained_progress` | Retained worktree diff, ownership markers, PR lineage if present, and private evidence. | A human chooses one path: resume same lane, finish manual repair, or reset/discard the retained patch explicitly. | + +For every guardrail stop, keep `decodex:needs-attention` until the blocker above is +resolved. If the issue returns to automation, request a Linear scan or let the next +scheduled scan observe the corrected tracker state; do not bypass the guardrail with a +manual retry that leaves the same evidence unchanged. + ## Broad Steer Examples Broad steer can be delivered by the runtime, but it does not erase lifecycle authority. diff --git a/docs/spec/lane-control.md b/docs/spec/lane-control.md index 62bd5aaae..c827b4b8a 100644 --- a/docs/spec/lane-control.md +++ b/docs/spec/lane-control.md @@ -213,6 +213,16 @@ the next lane action from authoritative signals. It is also the correct route wh requested control would overwrite useful partial work, hide a blocker, or require guessing human intent. +Loop guardrail outcomes are manual-attention stop reasons, not active-lane controls. +When status or failure writeback reports `validation_repeat`, `no_effective_diff`, +`remaining_delta_unchanged`, `review_churn`, `dependency_program_stale`, +`uncovered_direction`, or `ambiguous_retained_progress`, operators must inspect the +retained worktree, private evidence, blocker state, or review findings named by that +reason before clearing `decodex:needs-attention`. Do not use steer, retry, label +cleanup, or hard interrupt to bypass the guardrail without changing the underlying +repair strategy, dependency readiness, research contract, or retained-progress +ownership decision. + Agents must not simulate manual attention by editing tracker state directly. The valid agent path is: diff --git a/docs/spec/linear-execution-ledger.md b/docs/spec/linear-execution-ledger.md index 3f0eb39c0..727422129 100644 --- a/docs/spec/linear-execution-ledger.md +++ b/docs/spec/linear-execution-ledger.md @@ -232,6 +232,16 @@ worktree changes and must not be emitted as `terminal_failure`. If the retained disposition absorbs a later runtime failure, the producer should preserve the source failure class in `evidence` instead of changing the event type or terminal path. +Loop guardrail stops are public `needs_attention` or `terminal_failure` records with +the runtime-owned `terminal_path = "manual_attention"` unless retained partial +progress has already taken precedence. The public record may use these normalized +`error_class` values: `validation_repeat`, `no_effective_diff`, +`remaining_delta_unchanged`, `review_churn`, `dependency_program_stale`, +`uncovered_direction`, or `ambiguous_retained_progress`. The Linear record must carry +only the public reason and next action; fingerprints, full checkpoint payloads, +review details, and worktree diagnostics remain in runtime SQLite private execution +events and `loop_guardrail_checkpoints`. + `failed_command` and `raw_error` are public-summary fields, not private evidence escape hatches. Producers must validate those values before writing a Linear comment. When the exact failed command or raw error contains private information, producers must diff --git a/docs/spec/loop-runtime.md b/docs/spec/loop-runtime.md index 94a7734e2..effeaea61 100644 --- a/docs/spec/loop-runtime.md +++ b/docs/spec/loop-runtime.md @@ -286,14 +286,15 @@ generic retry bucket. Normalized outcomes include: | Outcome | Use when | | --- | --- | -| `validation_failure_repeated` | The same validation class repeats after bounded repair. | -| `no_effective_diff` | Repeated attempts do not change the head, evidence, or decision state. | -| `review_policy_exhausted` | Review findings exceed the accepted repair convergence budget. | +| `validation_repeat` | The same validation class repeats after bounded repair. Legacy public summaries may still use `validation_failure_repeated`. | +| `no_effective_diff` | Repeated attempts do not change the tracked delta, evidence, or decision state. | +| `remaining_delta_unchanged` | Validation text or failure presentation changes, but the remaining tracked delta is unchanged across bounded repair attempts. | +| `review_churn` | Review findings exceed the accepted repair convergence budget. Existing review-policy stops may still appear as `review_policy_exhausted`. | | `architecture_review_required` | The lane needs architecture direction before more repair. | | `review_policy_blocked` | Review cannot proceed from available evidence. | -| `dependency_blocked` | The node is waiting on dependency state that is not progressing. | -| `research_contract_required` | Execution uncovered a missing or contradictory decision contract. | -| `ownership_ambiguous` | Tracker, PR, branch, or runtime ownership evidence is contradictory. | +| `dependency_program_stale` | The node is waiting on dependency state that is not progressing. Legacy summaries may still use `dependency_blocked`. | +| `uncovered_direction` | Execution uncovered a missing or contradictory decision contract. Legacy summaries may still use `research_contract_required`. | +| `ambiguous_retained_progress` | Tracker, PR, branch, retained worktree, or runtime ownership evidence is contradictory. Legacy summaries may still use `ownership_ambiguous`. | These outcomes should route to failure attribution, research-contract feedback, architecture review, or manual attention. They must not spin in automatic retries. diff --git a/docs/spec/post-review-lifecycle.md b/docs/spec/post-review-lifecycle.md index 558a2aae4..a2aeeaf42 100644 --- a/docs/spec/post-review-lifecycle.md +++ b/docs/spec/post-review-lifecycle.md @@ -305,6 +305,11 @@ At any phase, contradictory signals or exhausted repair/convergence budgets forc - Operator status must report an exhausted retained post-review lane as `blocked` with a retry-budget reason instead of continuing to classify it as `needs_review_repair` or `ready_to_land`. - If the lane's PR is already merged, exhausted local closeout, default-branch sync, or cleanup retries must be reported as `closeout_blocked` or `cleanup_blocked` with the retained PR URL from the runtime handoff row. - Structural churn is not a generic retry case. If repair rounds exceed the configured convergence budget, the runtime must stop for human intervention or architecture rethink rather than patching indefinitely. +- Three consecutive non-clean fresh-context review rounds in the same phase are a + review-churn guardrail. Public writeback may use `review_policy_exhausted` or the + normalized loop reason `review_churn`, but the recovery rule is the same: inspect the + repeated findings for the exact current head and choose a new repair strategy, + architecture review, or manual resolution before requeueing. - A repair batch that changes the head must return to `review_wait` for that new head instead of continuing downstream on stale review state. ### Landing and closeout failures diff --git a/docs/spec/review-orchestration.md b/docs/spec/review-orchestration.md index 3f6301c15..98ac2f988 100644 --- a/docs/spec/review-orchestration.md +++ b/docs/spec/review-orchestration.md @@ -253,7 +253,7 @@ Current required behavior: - The terminal failure path must preserve the normalized review-stop class instead of collapsing it into a generic retry failure: - `architecture_review_required` - - `review_policy_exhausted` + - `review_policy_exhausted` or the loop-guardrail projection `review_churn` - `review_policy_blocked` - A terminal failure comment may point the operator toward a bounded research follow-up, but that guidance is not a research dispatch signal. diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index f280d839f..7cc8edba0 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -44,7 +44,7 @@ state or this state machine. ## Source of truth boundaries -- The Decodex runtime SQLite database is the single-machine source of truth for active leases, attempts, run-control channels, protocol events, private execution events, latent and promoted Decision Contracts, worktree mappings, retained PR state, review-policy checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. +- The Decodex runtime SQLite database is the single-machine source of truth for active leases, attempts, run-control channels, protocol events, private execution events, latent and promoted Decision Contracts, worktree mappings, retained PR state, review-policy checkpoints, loop-guardrail checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. - Linear remains the team-visible tracker surface for issue lifecycle, queue/active/manual-attention labels, and coarse lifecycle summaries such as start, PR-ready, blocked, failed, landed, and done. - Versioned Linear execution event comments use the schema in [`linear-execution-ledger.md`](./linear-execution-ledger.md), but fine-grained runtime truth must not be rebuilt from comments every tick. @@ -73,6 +73,7 @@ mirror: | Runtime SQLite `decision_contracts` | Versioned `decodex.decision_contract/1` payloads produced by research/design and later promoted into execution authority. The row status is indexed for local runtime lookup, but the JSON payload remains the contract authority. | | Runtime SQLite `run_control_channels` | Local control capability metadata for active run attempts. It records the project, issue, run id, attempt, transport, local channel path, channel status, and publish/update timestamps needed to route future control requests without bypassing active lease ownership. | | Runtime SQLite `review_policy_checkpoints` | Latest bounded-review checkpoint state for one project, issue, run, attempt, and phase, including structured independent-review detail. This row is the authority for review handoff and retained repair gating. | +| Runtime SQLite `loop_guardrail_checkpoints` | Latest convergence checkpoint for one project, issue, and guardrail reason. It stores the fingerprint, consecutive count, run id, attempt number, and structured detail used to stop non-converging loops without replaying Linear comments. | | Agent evidence under `~/.codex/decodex/agent-evidence//` | Derived local handoff view for repair agents. It may reference private evidence readback commands and compact run capsules, but it is not scheduling authority and is not a public mirror. | | Logs under `~/.codex/decodex/logs/` and `.decodex-run-activity` | Diagnostic process and liveness signals. They may explain what a local process did, but they are not the structured execution ledger and must not be replayed as tracker state. | | Linear execution ledger comments | Low-frequency public projection for team-visible lifecycle state. They carry coarse start, progress phase, PR, handoff, failure, landing, closeout, and cleanup summaries only. | @@ -89,6 +90,8 @@ The following facts are local runtime truth and must not be rebuilt from Linear - private execution events carrying structured local evidence for an issue/run/attempt - review-policy checkpoint state: current phase, normalized status, lane head, consecutive non-clean round count, and structured independent-review detail +- loop-guardrail checkpoint state: normalized reason, fingerprint, consecutive + observation count, source run, and structured stop detail - retry and backoff state: queued retry kind, due time, retry budget, and connector backoff - phase timing and operator activity summaries - retained worktree mappings, retained PR handoff identity, post-review phase, and cleanup or repair ownership @@ -336,6 +339,16 @@ When `decodex` runs the repo-native gate during `validating`, it must preserve t The continued-repair classes above are ordinary bounded churn: the coding agent should keep repairing code and rerun the repo gate rather than requesting `manual_attention` just because the gate has not passed yet. Human-attention exits remain reserved for environment, toolchain, or operator-owned blockers that the coding agent cannot clear from the retained worktree alone. +Continued repair is still bounded by loop guardrails. For each retryable failure, +Decodex records local `loop_guardrail_checkpoint` evidence and updates the +`loop_guardrail_checkpoints` row for any matching convergence reason. Three +consecutive observations with the same fingerprint stop automatic retry and route the +lane through the human-required failure path. Repo-gate failures record both +`validation_repeat` and `remaining_delta_unchanged` observations; retryable failures +with no changing tracked delta record `no_effective_diff`. Fingerprints use the lane +HEAD plus the tracked worktree status and diff against `HEAD`, so retained partial +progress remains inspectable instead of being deleted or hidden. + When `codex.internal_review_mode = "loop"`, handoff and retained review-repair runs also consume the latest structured `issue_review_checkpoint` state for the current phase and current lane head from the owned lane: - no checkpoint and no terminal path: allow a clean continuation boundary @@ -435,6 +448,27 @@ Runtime-owned review-policy stops use the same human-required failure path, but - `architecture_review_required` - `review_policy_blocked` +Runtime loop guardrails use the same human-required failure path, but preserve a +structured failure-attribution `error_class` so operator status and Linear summaries +can distinguish the stop class: + +- `validation_repeat`: the same validation failure repeated three times. +- `no_effective_diff`: retryable attempts repeated without a changed tracked delta. +- `remaining_delta_unchanged`: validation text changed but the remaining tracked delta + stayed unchanged for three attempts. +- `dependency_program_stale`: a queued issue kept the same open dependency blocker + fingerprint across three status observations, indicating Execution Program readiness + or issue decomposition is stale. +- `uncovered_direction`: execution found missing direction that must feed back into a + research or Decision Contract before more implementation. +- `ambiguous_retained_progress`: retained local work or ownership evidence is useful + but ambiguous enough that a human must choose resume, reset, or manual repair. + +Review repair churn uses the bounded review policy above and may appear publicly as +`review_policy_exhausted` or the normalized loop reason `review_churn`; both mean the +operator should inspect repeated review findings and stop patch-on-patch repair until +the next strategy is explicit. + If the configured `decodex:needs-attention` label is unavailable on the team and the configured failure state is startable, `decodex` must still block automatic reselection by leaving the issue in a non-startable guard state such as `In Progress`. In that case the failure comment must explain that the label could not be applied and that a human must move the issue back to a startable state manually after repair. Restart recovery must preserve that guard by writing a retained-worktree marker under `.worktrees//.decodex-terminal-guarded` and consulting it before redispatching recovered `In Progress` lanes. Any issue carrying `decodex:needs-attention` is ineligible for another automatic run until a human clears the label and returns the issue to a startable state. @@ -472,6 +506,7 @@ The runtime database stores at least: - worktree mappings - retained PR and post-review state - review-policy checkpoints +- loop-guardrail checkpoints - retry state and retry budgets - phase timing and operator activity summaries - tracker and PR cache rows needed to survive connector outages