From d9c3c056f0eeb0e73fc71b541ed6c77e404bb6ec Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Wed, 10 Jun 2026 19:32:42 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Implement autonomous architecture recovery","authority":"XY-873"} --- apps/decodex/src/agent/app_server.rs | 5 +- .../src/agent/tracker_tool_bridge/tools.rs | 5 +- apps/decodex/src/cli.rs | 10 +- .../src/orchestrator/agent_evidence.rs | 92 +++ apps/decodex/src/orchestrator/daemon.rs | 29 +- apps/decodex/src/orchestrator/execution.rs | 757 +++++++++++++++++- .../src/orchestrator/harness_improvement.rs | 22 + apps/decodex/src/orchestrator/prompting.rs | 66 +- apps/decodex/src/orchestrator/selection.rs | 4 +- .../tests/intake/run_and_prompting.rs | 88 ++ .../tests/operator/status/agent_evidence.rs | 43 + .../tests/recovery/terminal_failures.rs | 30 +- .../src/orchestrator/tests/runtime/failure.rs | 157 ++++ .../tests/runtime/loop_scenarios.rs | 34 +- apps/decodex/src/orchestrator/types.rs | 47 +- apps/decodex/src/state/store.rs | 19 + docs/runbook/lane-control-recovery.md | 40 +- docs/spec/agent-evidence.md | 8 +- docs/spec/loop-runtime.md | 68 +- docs/spec/runtime.md | 42 +- 20 files changed, 1508 insertions(+), 58 deletions(-) diff --git a/apps/decodex/src/agent/app_server.rs b/apps/decodex/src/agent/app_server.rs index bb27ef3d2..ca0cd978d 100644 --- a/apps/decodex/src/agent/app_server.rs +++ b/apps/decodex/src/agent/app_server.rs @@ -3702,8 +3702,9 @@ fn clear_thread_phase_goal_best_effort( tracing::warn!(?error, "Failed to record app-server goal clear response."); } }, - Err(error) => - tracing::warn!(?error, "Failed to clear app-server phase goal after terminal path."), + Err(error) => { + tracing::warn!(?error, "Failed to clear app-server phase goal after terminal path.") + }, } } diff --git a/apps/decodex/src/agent/tracker_tool_bridge/tools.rs b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs index 996044bde..9d9209ece 100644 --- a/apps/decodex/src/agent/tracker_tool_bridge/tools.rs +++ b/apps/decodex/src/agent/tracker_tool_bridge/tools.rs @@ -1263,11 +1263,12 @@ impl<'a> TrackerToolBridge<'a> { }; let details_json = match serde_json::to_string(&checkpoint_payload) { Ok(details_json) => details_json, - Err(error) => + Err(error) => { return DynamicToolCallResponse::failure(format!( "Failed to serialize the structured review checkpoint for issue `{}`: {error}", self.issue.identifier - )), + )); + }, }; let nonclean_rounds = match self.review_checkpoint_nonclean_rounds( review_context, diff --git a/apps/decodex/src/cli.rs b/apps/decodex/src/cli.rs index 5b477a681..3519b6e01 100644 --- a/apps/decodex/src/cli.rs +++ b/apps/decodex/src/cli.rs @@ -680,10 +680,12 @@ impl ResearchCompileCommand { self.source_issue.clone(), self.outcome.into(), )), - (None, None) => - eyre::bail!("research compile requires either --input or --intent ."), - (Some(_), Some(_)) => - eyre::bail!("research compile accepts --input or --intent, not both."), + (None, None) => { + eyre::bail!("research compile requires either --input or --intent .") + }, + (Some(_), Some(_)) => { + eyre::bail!("research compile accepts --input or --intent, not both.") + }, } } } diff --git a/apps/decodex/src/orchestrator/agent_evidence.rs b/apps/decodex/src/orchestrator/agent_evidence.rs index b6ab20654..151e74ce4 100644 --- a/apps/decodex/src/orchestrator/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/agent_evidence.rs @@ -260,6 +260,7 @@ struct PrivateEvidenceReadback { latest_event_type: Option, latest_event_at: Option, decision_requests: Vec, + architecture_recoveries: Vec, improvement_candidates: Vec, events: Vec, warnings: Vec, @@ -276,6 +277,16 @@ struct PrivateEvidenceDecisionRequestSummary { resume_condition: Option, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct PrivateEvidenceArchitectureRecoverySummary { + reason_code: String, + guardrail_reason: Option, + boundary_disposition: Option, + recovery_budget_attempt: Option, + recovery_budget_max_attempts: Option, + next_action: String, +} + #[derive(Clone, Debug, PartialEq, Serialize)] struct PrivateEvidenceReadbackEvent { record_id: i64, @@ -660,6 +671,7 @@ fn build_private_evidence_readback( latest_event_type: latest_event.map(|event| event.event_type().to_owned()), latest_event_at: latest_event.map(|event| event.recorded_at().to_owned()), decision_requests: authority_decision_requests_from_private_events(&events), + architecture_recoveries: architecture_recoveries_from_private_events(&events), improvement_candidates: harness_improvement_candidates_from_private_events(&events), events: events .iter() @@ -802,6 +814,86 @@ fn authority_decision_request_from_private_event( }) } +fn architecture_recoveries_from_private_events( + events: &[state::PrivateExecutionEvent], +) -> Vec { + events + .iter() + .filter(|event| { + matches!( + event.event_type(), + ARCHITECTURE_RECOVERY_PACKET_EVENT_TYPE + | ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE + | ARCHITECTURE_RECOVERY_TERMINAL_EVENT_TYPE + ) + }) + .filter_map(architecture_recovery_from_private_event) + .collect() +} + +fn architecture_recovery_from_private_event( + event: &state::PrivateExecutionEvent, +) -> Option { + let payload = event.payload(); + let reason_code = payload.get("reason_code")?.as_str()?.to_owned(); + let guardrail_reason = payload + .get("guardrail_reason") + .and_then(Value::as_str) + .or_else(|| { + payload + .get("loop_guardrail") + .and_then(|guardrail| guardrail.get("reason")) + .and_then(Value::as_str) + }) + .map(str::to_owned); + let boundary_disposition = payload + .get("boundary_disposition") + .and_then(Value::as_str) + .or_else(|| { + payload + .get("authority_boundary_check") + .and_then(|boundary| boundary.get("disposition")) + .and_then(Value::as_str) + }) + .map(str::to_owned); + let recovery_budget_attempt = payload + .get("recovery_budget") + .and_then(|budget| budget.get("attempt")) + .and_then(Value::as_u64); + let recovery_budget_max_attempts = payload + .get("recovery_budget") + .and_then(|budget| budget.get("max_attempts")) + .and_then(Value::as_u64); + let next_action = architecture_recovery_next_action(&reason_code); + + Some(PrivateEvidenceArchitectureRecoverySummary { + reason_code, + guardrail_reason, + boundary_disposition, + recovery_budget_attempt, + recovery_budget_max_attempts, + next_action, + }) +} + +fn architecture_recovery_next_action(reason_code: &str) -> String { + match reason_code { + "architecture_recovery_started" => { + String::from("Retry with a materially different implementation strategy inside authority.") + }, + "architecture_recovery_exhausted" => { + String::from("Require a new accepted recovery strategy or architecture decision before retrying.") + }, + "external_dependency_required" => { + String::from("Resolve the dependency or Execution Program readiness blocker before retrying.") + }, + "contract_boundary_required" => { + String::from("Resolve the Decision Contract or Authority Envelope boundary before retrying.") + }, + _ => String::from("Inspect the Architecture Recovery Packet before retrying."), + } +} + fn private_evidence_run_matches_issue( project: &ServiceConfig, run: &ProjectRunStatus, diff --git a/apps/decodex/src/orchestrator/daemon.rs b/apps/decodex/src/orchestrator/daemon.rs index 054038904..0831c3bc1 100644 --- a/apps/decodex/src/orchestrator/daemon.rs +++ b/apps/decodex/src/orchestrator/daemon.rs @@ -1103,8 +1103,10 @@ where } else { let retry_budget_attempts = child_exit_retry_budget_attempt_count(&context, &issue, child)?; + let retry_budget_limit = + child_exit_retry_budget_limit(&context, &issue, child)?; - if retry_budget_attempts >= context.workflow.frontmatter().execution().max_attempts() { + if retry_budget_attempts >= retry_budget_limit { return terminalize_exhausted_child_exit_retry( context, issue, @@ -1227,6 +1229,31 @@ where Ok(u32::try_from(retry_budget_attempts).unwrap_or(u32::MAX).max(1)) } +fn child_exit_retry_budget_limit( + context: &ChildExitRetryContext<'_, T>, + issue: &TrackerIssue, + child: ChildRunRef<'_>, +) -> Result +where + T: IssueTracker, +{ + let max_attempts = context.workflow.frontmatter().execution().max_attempts(); + let worktree = child_exit_worktree_spec(context, issue)?; + let Some(marker) = state::read_run_activity_marker_snapshot(&worktree.path)? else { + return Ok(max_attempts); + }; + + if marker.run_id() == child.run_id + && marker.attempt_number() == child.attempt_number + && marker.retry_kind() == Some(ARCHITECTURE_RECOVERY_RETRY_KIND) + { + return Ok(max_attempts + .saturating_add(u32::try_from(ARCHITECTURE_RECOVERY_BUDGET).unwrap_or(0))); + } + + Ok(max_attempts) +} + fn terminalize_exhausted_child_exit_retry( context: ChildExitRetryContext<'_, T>, issue: TrackerIssue, diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index 9788f1c49..c9eee11d5 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -4,10 +4,13 @@ use agent::CodexAccountProvider; use agent::AppServerThreadArchiveRequest; use records::LinearExecutionEventPublicProjection; use sha2::Digest; +use state::DecisionContractRecord; use crate::tracker::privacy_classifier::PublicProjectionPrivacyClassifier; const LOOP_GUARDRAIL_CONVERGENCE_BUDGET: i64 = 3; +const ARCHITECTURE_RECOVERY_BUDGET: usize = 1; +const ARCHITECTURE_RECOVERY_RETRY_KIND: &str = "architecture_recovery"; #[derive(Debug)] pub(crate) struct AppServerZeroEvidenceStartFailure { @@ -100,6 +103,12 @@ struct PreparedTerminalFailureWriteback { retry_guarded_by_state: bool, } +struct ArchitectureRecoveryStart { + attempt_number: usize, + max_attempts: usize, + detail: String, +} + struct CompletedAppServerRun<'a, T> where T: IssueTracker, @@ -200,7 +209,27 @@ impl RepoGatePhaseGoalController<'_> { self.issue_run, &error, )? { - return Err(Report::new(loop_guardrail_stop).wrap_err(error)); + match loop_guardrail_architecture_recovery_decision( + self.project, + self.state_store, + self.issue_run, + loop_guardrail_stop, + &error, + )? { + LoopGuardrailRecoveryDecision::Start(recovery) => { + let next_goal = self.phase_goal_spec( + PhaseGoalKind::RepairValidationFailures, + Some(&recovery.detail), + ); + + self.persist_next_phase_goal(&next_goal, "architecture_recovery_started")?; + + return Ok(PhaseGoalTransition::Continue(next_goal)); + }, + LoopGuardrailRecoveryDecision::HumanRequired(loop_guardrail_stop) => { + return Err(Report::new(loop_guardrail_stop).wrap_err(error)); + }, + } } let detail = format!( @@ -359,6 +388,41 @@ where retry_budget_attempts: i64, } +struct ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition, + final_reason: &'static str, + boundary_type: &'static str, +} + +struct ArchitectureRecoveryPacketInput<'a> { + project: &'a ServiceConfig, + issue_run: &'a IssueRunPlan, + loop_guardrail_stop: &'a LoopGuardrailStopRequested, + error: &'a Report, + contracts: &'a [DecisionContractRecord], + boundary_check_record_id: i64, + boundary_disposition: AuthorityBoundaryDisposition, + boundary_final_reason: &'a str, + reason_code: &'a str, + recovery_attempt_number: usize, + prior_started_count: usize, +} + +struct ArchitectureRecoveryTerminalEventInput<'a> { + project: &'a ServiceConfig, + issue_run: &'a IssueRunPlan, + stop: &'a LoopGuardrailStopRequested, + boundary_check_record_id: i64, + boundary_disposition: AuthorityBoundaryDisposition, + reason_code: &'a str, + recovery_attempt_number: usize, +} + +enum LoopGuardrailRecoveryDecision { + Start(ArchitectureRecoveryStart), + HumanRequired(LoopGuardrailStopRequested), +} + #[derive(Clone, Copy, Eq, PartialEq)] enum TerminalFailureEventRecordStatus { Recorded, @@ -1841,6 +1905,7 @@ fn retryable_failure_loop_guardrail_stop( consecutive_count: checkpoint.consecutive_count(), fingerprint, source_error_class: source_error_class.map(ToOwned::to_owned), + architecture_recovery_reason_code: None, })); } } @@ -1927,6 +1992,595 @@ fn record_loop_guardrail_private_event( .map(|_| ()) } +fn loop_guardrail_stop_from_review_policy( + review_policy_stop: &ReviewPolicyStopRequested, +) -> LoopGuardrailStopRequested { + LoopGuardrailStopRequested { + issue_identifier: review_policy_stop.issue_identifier.clone(), + run_id: review_policy_stop.run_id.clone(), + reason: LoopGuardrailReason::ReviewChurn, + consecutive_count: review_policy_stop.nonclean_rounds.unwrap_or_default(), + fingerprint: format!( + "{}:{}", + review_policy_stop.head_sha, + review_policy_stop.nonclean_rounds.unwrap_or_default() + ), + source_error_class: Some(review_policy_stop.reason.error_class().to_owned()), + architecture_recovery_reason_code: None, + } +} + +fn loop_guardrail_architecture_recovery_decision( + project: &ServiceConfig, + state_store: &StateStore, + issue_run: &IssueRunPlan, + mut loop_guardrail_stop: LoopGuardrailStopRequested, + error: &Report, +) -> Result { + let prior_started_count = + architecture_recovery_started_count(state_store, project, issue_run)?; + let recovery_attempt_number = prior_started_count.saturating_add(1); + let boundary = classify_loop_guardrail_authority_boundary(&loop_guardrail_stop, error); + let contracts = architecture_recovery_contracts_for_issue(state_store, project, issue_run)?; + let decision_contract_ids = + contracts.iter().map(|contract| contract.contract_id().to_owned()).collect::>(); + let decision_contract_id_refs = + decision_contract_ids.iter().map(String::as_str).collect::>(); + let boundary_event = record_authority_boundary_check_private_event( + state_store, + AuthorityBoundaryCheckInput { + project_id: project.service_id(), + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + decision_contract_ids: decision_contract_id_refs, + attempted_recovery_reason: loop_guardrail_stop.reason.error_class(), + changed_surfaces: architecture_recovery_changed_surfaces(&boundary), + disposition: boundary.disposition, + final_disposition_reason: boundary.final_reason, + improvement_signals: architecture_recovery_improvement_signals( + loop_guardrail_stop.reason, + &boundary, + ), + }, + )?; + let budget_exhausted = prior_started_count >= ARCHITECTURE_RECOVERY_BUDGET; + let reason_code = architecture_recovery_reason_code(&boundary, budget_exhausted); + + record_architecture_recovery_packet( + state_store, + ArchitectureRecoveryPacketInput { + project, + issue_run, + loop_guardrail_stop: &loop_guardrail_stop, + error, + contracts: &contracts, + boundary_check_record_id: boundary_event.record_id(), + boundary_disposition: boundary.disposition, + boundary_final_reason: boundary.final_reason, + reason_code, + recovery_attempt_number, + prior_started_count, + }, + )?; + + if budget_exhausted || boundary.disposition != AuthorityBoundaryDisposition::WithinAuthority { + loop_guardrail_stop.architecture_recovery_reason_code = Some(reason_code.to_owned()); + + record_architecture_recovery_terminal_event( + state_store, + ArchitectureRecoveryTerminalEventInput { + project, + issue_run, + stop: &loop_guardrail_stop, + boundary_check_record_id: boundary_event.record_id(), + boundary_disposition: boundary.disposition, + reason_code, + recovery_attempt_number, + }, + )?; + + if boundary.disposition != AuthorityBoundaryDisposition::WithinAuthority { + let decision_request_id = format!( + "{}-{}-{}-{}", + issue_run.issue.identifier, + issue_run.run_id, + issue_run.attempt_number, + reason_code + ); + + record_authority_decision_request_private_event( + state_store, + architecture_recovery_decision_request_input( + project, + issue_run, + &loop_guardrail_stop, + boundary_event.record_id(), + &decision_request_id, + reason_code, + boundary.final_reason, + ), + )?; + } + + return Ok(LoopGuardrailRecoveryDecision::HumanRequired(loop_guardrail_stop)); + } + + state_store.clear_loop_guardrail_checkpoint( + project.service_id(), + &issue_run.issue.id, + loop_guardrail_stop.reason.error_class(), + )?; + + record_architecture_recovery_started_event( + state_store, + project, + issue_run, + &loop_guardrail_stop, + boundary_event.record_id(), + recovery_attempt_number, + )?; + + Ok(LoopGuardrailRecoveryDecision::Start(ArchitectureRecoveryStart { + attempt_number: recovery_attempt_number, + max_attempts: ARCHITECTURE_RECOVERY_BUDGET, + detail: architecture_recovery_goal_detail(&loop_guardrail_stop, recovery_attempt_number), + })) +} + +fn classify_loop_guardrail_authority_boundary( + stop: &LoopGuardrailStopRequested, + error: &Report, +) -> ArchitectureRecoveryBoundary { + let source_is_repo_gate = + stop.source_error_class.as_deref().is_some_and(|class| class.starts_with("repo_gate_")) + || error.downcast_ref::().is_some_and(|failure| { + failure.disposition() == RepoGateFailureDisposition::ContinueRepair + }); + + match stop.reason { + LoopGuardrailReason::ValidationRepeat | LoopGuardrailReason::RemainingDeltaUnchanged + if source_is_repo_gate => + { + ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::WithinAuthority, + final_reason: "Repo-gate convergence failed on an engineering implementation problem; architecture recovery may change implementation strategy without weakening validation.", + boundary_type: "implementation_strategy", + } + }, + LoopGuardrailReason::NoEffectiveDiff if source_is_repo_gate => { + ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::WithinAuthority, + final_reason: "No-effective-diff convergence followed repo-gate repair work; architecture recovery may replace the ineffective implementation strategy.", + boundary_type: "implementation_strategy", + } + }, + LoopGuardrailReason::ReviewChurn => ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::WithinAuthority, + final_reason: "Review churn can be recovered autonomously only by changing implementation architecture while preserving accepted behavior and review standards.", + boundary_type: "implementation_strategy", + }, + LoopGuardrailReason::DependencyProgramStale => ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::RequiresHuman, + final_reason: "The next viable action changes dependency or Execution Program readiness and requires accepted authority.", + boundary_type: "external_dependency", + }, + LoopGuardrailReason::UncoveredDirection => ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::RequiresHuman, + final_reason: "Execution uncovered missing direction that changes the accepted Decision Contract.", + boundary_type: "decision_contract", + }, + LoopGuardrailReason::AmbiguousRetainedProgress => ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::InsufficientEvidence, + final_reason: "Retained progress ownership is underspecified, so Decodex lacks evidence that recovery is inside authority.", + boundary_type: "retained_ownership", + }, + _ => ArchitectureRecoveryBoundary { + disposition: AuthorityBoundaryDisposition::InsufficientEvidence, + final_reason: "Guardrail evidence is insufficient to prove autonomous recovery stays inside the Authority Envelope.", + boundary_type: "authority_evidence", + }, + } +} + +fn architecture_recovery_started_count( + state_store: &StateStore, + project: &ServiceConfig, + issue_run: &IssueRunPlan, +) -> Result { + Ok(state_store + .list_private_execution_events_for_issue(project.service_id(), &issue_run.issue.id)? + .iter() + .filter(|event| event.event_type() == ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE) + .count()) +} + +fn architecture_recovery_contracts_for_issue( + state_store: &StateStore, + project: &ServiceConfig, + issue_run: &IssueRunPlan, +) -> Result> { + let mut records = Vec::new(); + + for issue_id in [&issue_run.issue.id, &issue_run.issue.identifier] { + for record in state_store.list_decision_contracts_for_issue(project.service_id(), issue_id)? { + if records + .iter() + .all(|existing: &DecisionContractRecord| existing.contract_id() != record.contract_id()) + { + records.push(record); + } + } + } + + records.sort_by(|left, right| left.contract_id().cmp(right.contract_id())); + + Ok(records) +} + +fn architecture_recovery_changed_surfaces( + boundary: &ArchitectureRecoveryBoundary, +) -> Vec> { + vec![AuthorityBoundaryChangedSurface { + surface: boundary.boundary_type, + change_summary: "Replace the non-converging guardrail repair strategy with a materially different architecture recovery strategy.", + classification: boundary.disposition, + }] +} + +fn architecture_recovery_improvement_signals( + reason: LoopGuardrailReason, + boundary: &ArchitectureRecoveryBoundary, +) -> Vec> { + match boundary.disposition { + AuthorityBoundaryDisposition::WithinAuthority => match reason { + LoopGuardrailReason::ValidationRepeat | LoopGuardrailReason::RemainingDeltaUnchanged => { + vec![AuthorityBoundaryImprovementSignal { + kind: "missing_validator", + reason_code: "validation_guardrail_repeated", + target: "validator:repo_gate", + recommendation: "Promote the repeated repo-gate failure into an earlier deterministic validator or fixture.", + }] + }, + _ => vec![AuthorityBoundaryImprovementSignal { + kind: "weak_prompt", + reason_code: "architecture_recovery_strategy_needed", + target: "prompt:phase_goal_repair", + recommendation: "Prompt recovery agents to replace the ineffective strategy instead of repeating patch-only repair.", + }], + }, + AuthorityBoundaryDisposition::RequiresHuman => vec![AuthorityBoundaryImprovementSignal { + kind: "underspecified_decision_contract", + reason_code: "contract_boundary_required", + target: "decision_contract:authority_envelope", + recommendation: "Record explicit accepted authority before retrying autonomous recovery.", + }], + AuthorityBoundaryDisposition::InsufficientEvidence => { + vec![AuthorityBoundaryImprovementSignal { + kind: "underspecified_decision_contract", + reason_code: "authority_evidence_missing", + target: "issue_template:loop_recovery", + recommendation: "Capture retained ownership, validation, and Decision Contract evidence before recovery.", + }] + }, + } +} + +fn architecture_recovery_reason_code( + boundary: &ArchitectureRecoveryBoundary, + budget_exhausted: bool, +) -> &'static str { + if budget_exhausted { + "architecture_recovery_exhausted" + } else if boundary.boundary_type == "external_dependency" { + "external_dependency_required" + } else if boundary.disposition == AuthorityBoundaryDisposition::WithinAuthority { + "architecture_recovery_started" + } else { + "contract_boundary_required" + } +} + +fn record_architecture_recovery_packet( + state_store: &StateStore, + input: ArchitectureRecoveryPacketInput<'_>, +) -> Result<()> { + let programs = architecture_recovery_programs_for_contracts( + state_store, + input.project.service_id(), + input.contracts, + )?; + let retained = architecture_recovery_retained_worktree(&input.issue_run.worktree.path)?; + let review = architecture_recovery_review_findings(state_store, input.project, input.issue_run)?; + + state_store + .append_private_execution_event( + input.project.service_id(), + &input.issue_run.issue.id, + &input.issue_run.run_id, + input.issue_run.attempt_number, + ARCHITECTURE_RECOVERY_PACKET_EVENT_TYPE, + json!({ + "schema": ARCHITECTURE_RECOVERY_PACKET_SCHEMA, + "record_version": 1, + "state": input.reason_code, + "reason_code": input.reason_code, + "issue": architecture_recovery_issue_payload(input.issue_run), + "run": architecture_recovery_run_payload(input.issue_run), + "decision_contract_context": input.contracts + .iter() + .map(architecture_recovery_contract_payload) + .collect::>(), + "execution_program_context": programs + .iter() + .map(architecture_recovery_program_payload) + .collect::>(), + "retained_worktree": retained, + "validation_failures": architecture_recovery_validation_failures( + input.loop_guardrail_stop, + input.error, + ), + "review_findings": review, + "prior_recovery_attempts": { + "started_count": input.prior_started_count, + }, + "recovery_budget": { + "attempt": input.recovery_attempt_number, + "max_attempts": ARCHITECTURE_RECOVERY_BUDGET, + }, + "loop_guardrail": { + "reason": input.loop_guardrail_stop.reason.error_class(), + "consecutive_count": input.loop_guardrail_stop.consecutive_count, + "threshold": LOOP_GUARDRAIL_CONVERGENCE_BUDGET, + "fingerprint": input.loop_guardrail_stop.fingerprint.as_str(), + "source_error_class": input.loop_guardrail_stop.source_error_class.as_deref(), + }, + "authority_boundary_check": { + "record_id": input.boundary_check_record_id, + "disposition": input.boundary_disposition.as_str(), + "reason": input.boundary_final_reason, + }, + }), + ) + .map(|_| ()) +} + +fn architecture_recovery_programs_for_contracts( + state_store: &StateStore, + project_id: &str, + contracts: &[DecisionContractRecord], +) -> Result> { + let mut programs = Vec::new(); + + for contract in contracts { + for program in state_store + .list_execution_programs_for_contract(project_id, contract.contract_id())? + { + if programs + .iter() + .all(|existing: &ExecutionProgramRecord| existing.program_id() != program.program_id()) + { + programs.push(program); + } + } + } + + programs.sort_by(|left, right| left.program_id().cmp(right.program_id())); + + Ok(programs) +} + +fn architecture_recovery_retained_worktree(worktree_path: &Path) -> Result { + let fingerprint = loop_guardrail_worktree_fingerprint(worktree_path)?; + let tracked_status = + git_guardrail_output(worktree_path, &["status", "--porcelain", "--untracked-files=no"])?; + let diff_stat = + git_guardrail_output(worktree_path, &["diff", "--stat", "--no-ext-diff", "HEAD", "--"])?; + + Ok(json!({ + "head_sha": fingerprint.as_ref().map(|value| value.head_sha.as_str()), + "tracked_status_hash": fingerprint + .as_ref() + .map(|value| value.tracked_status_hash.as_str()), + "tracked_diff_hash": fingerprint.as_ref().map(|value| value.tracked_diff_hash.as_str()), + "tracked_status": tracked_status.unwrap_or_default(), + "diff_stat": diff_stat.unwrap_or_default(), + })) +} + +fn architecture_recovery_review_findings( + state_store: &StateStore, + project: &ServiceConfig, + issue_run: &IssueRunPlan, +) -> Result { + let events = state_store.list_private_execution_events_for_issue( + project.service_id(), + &issue_run.issue.id, + )?; + let latest_review = events + .iter() + .rev() + .find(|event| event.event_type() == "review_checkpoint") + .map(|event| event.payload()); + let Some(payload) = latest_review else { + return Ok(json!({ + "latest_status": null, + "accepted_finding_count": 0, + "rejected_finding_count": 0, + })); + }; + let review = payload.get("review").unwrap_or(payload); + + Ok(json!({ + "latest_status": payload.get("status").and_then(Value::as_str), + "accepted_finding_count": review + .get("accepted_findings") + .and_then(Value::as_array) + .map_or(0, Vec::len), + "rejected_finding_count": review + .get("rejected_findings") + .and_then(Value::as_array) + .map_or(0, Vec::len), + "nonclean_rounds": payload.get("nonclean_rounds").and_then(Value::as_i64).unwrap_or(0), + })) +} + +fn architecture_recovery_issue_payload(issue_run: &IssueRunPlan) -> Value { + json!({ + "id": issue_run.issue.id.as_str(), + "identifier": issue_run.issue.identifier.as_str(), + "title": issue_run.issue.title.as_str(), + }) +} + +fn architecture_recovery_run_payload(issue_run: &IssueRunPlan) -> Value { + json!({ + "run_id": issue_run.run_id.as_str(), + "attempt_number": issue_run.attempt_number, + "branch": issue_run.worktree.branch_name.as_str(), + "dispatch_mode": issue_run.dispatch_mode.as_str(), + }) +} + +fn architecture_recovery_contract_payload(record: &DecisionContractRecord) -> Value { + json!({ + "contract_id": record.contract_id(), + "source_issue_id": record.source_issue_id(), + "status": record.status().as_str(), + "updated_at": record.updated_at(), + }) +} + +fn architecture_recovery_program_payload(record: &ExecutionProgramRecord) -> Value { + json!({ + "program_id": record.program_id(), + "source_contract_id": record.source_contract_id(), + "updated_at": record.updated_at(), + }) +} + +fn architecture_recovery_validation_failures( + stop: &LoopGuardrailStopRequested, + error: &Report, +) -> Value { + json!({ + "guardrail_reason": stop.reason.error_class(), + "source_error_class": stop.source_error_class.as_deref(), + "error_summary": truncate_private_diagnostic_text(&error.to_string()), + }) +} + +fn record_architecture_recovery_started_event( + state_store: &StateStore, + project: &ServiceConfig, + issue_run: &IssueRunPlan, + stop: &LoopGuardrailStopRequested, + boundary_check_record_id: i64, + recovery_attempt_number: usize, +) -> Result<()> { + state_store + .append_private_execution_event( + project.service_id(), + &issue_run.issue.id, + &issue_run.run_id, + issue_run.attempt_number, + ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE, + json!({ + "schema": "decodex.architecture_recovery_started/1", + "record_version": 1, + "reason_code": "architecture_recovery_started", + "guardrail_reason": stop.reason.error_class(), + "authority_boundary_check_record_id": boundary_check_record_id, + "recovery_budget": { + "attempt": recovery_attempt_number, + "max_attempts": ARCHITECTURE_RECOVERY_BUDGET, + }, + "next_strategy": "materially_different_architecture_recovery", + }), + ) + .map(|_| ()) +} + +fn record_architecture_recovery_terminal_event( + state_store: &StateStore, + input: ArchitectureRecoveryTerminalEventInput<'_>, +) -> Result<()> { + state_store + .append_private_execution_event( + input.project.service_id(), + &input.issue_run.issue.id, + &input.issue_run.run_id, + input.issue_run.attempt_number, + ARCHITECTURE_RECOVERY_TERMINAL_EVENT_TYPE, + json!({ + "schema": "decodex.architecture_recovery_terminal/1", + "record_version": 1, + "reason_code": input.reason_code, + "guardrail_reason": input.stop.reason.error_class(), + "authority_boundary_check_record_id": input.boundary_check_record_id, + "boundary_disposition": input.boundary_disposition.as_str(), + "recovery_budget": { + "attempt": input.recovery_attempt_number, + "max_attempts": ARCHITECTURE_RECOVERY_BUDGET, + }, + }), + ) + .map(|_| ()) +} + +fn architecture_recovery_decision_request_input<'a>( + project: &'a ServiceConfig, + issue_run: &'a IssueRunPlan, + stop: &'a LoopGuardrailStopRequested, + boundary_check_record_id: i64, + decision_request_id: &'a str, + reason_code: &'a str, + final_reason: &'a str, +) -> AuthorityDecisionRequestInput<'a> { + AuthorityDecisionRequestInput { + project_id: project.service_id(), + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + boundary_check_record_id, + decision_request_id, + reason_code, + boundary_type: "architecture_recovery", + proposed_change: "Continue loop recovery with a materially different architecture strategy.", + why_exceeds_authority: final_reason, + options: vec![ + AuthorityDecisionOption { + label: "Authorize recovery", + description: "Update the issue, Decision Contract, or policy to allow this recovery.", + }, + AuthorityDecisionOption { + label: "Keep stopped", + description: "Leave the lane in manual attention until the boundary is resolved.", + }, + ], + recommendation: "Resolve the authority boundary before requeueing the lane.", + resume_condition: "Accept, reject, or revise the requested authority in the issue, Decision Contract, or project policy before clearing needs-attention.", + retained_worktree_evidence: vec![issue_run.worktree.branch_name.as_str()], + retained_diff_evidence: vec![stop.fingerprint.as_str()], + recovery_attempt_context: vec![stop.reason.error_class()], + } +} + +fn architecture_recovery_goal_detail( + stop: &LoopGuardrailStopRequested, + recovery_attempt_number: usize, +) -> String { + format!( + "Loop guardrail `{}` stopped the current ineffective strategy after {} matching observations. Decodex recorded an Architecture Recovery Packet and an Authority Boundary Check with `within_authority`; use autonomous architecture recovery attempt {} of {}. Start a materially different implementation strategy, preserve the accepted Decision Contract and all validation/review gates, and request human attention only if the next viable action would change product behavior, public API/config contract, security, data, credential, billing, validation standards, or accepted authority.", + stop.reason.error_class(), + stop.consecutive_count, + recovery_attempt_number, + ARCHITECTURE_RECOVERY_BUDGET + ) +} + fn run_failure_requires_terminal_attention(error: &Report) -> bool { error.downcast_ref::().is_some() || error.downcast_ref::().is_some() @@ -1982,8 +2636,35 @@ where retryable_failure_loop_guardrail_stop(project, state_store, issue_run, error)? }; + if let Some(review_policy_stop) = error.downcast_ref::() + && review_policy_stop.reason == ReviewPolicyStopReason::Exhausted + { + return match loop_guardrail_architecture_recovery_decision( + project, + state_store, + issue_run, + loop_guardrail_stop_from_review_policy(review_policy_stop), + error, + )? { + LoopGuardrailRecoveryDecision::Start(recovery) => + apply_architecture_recovery_retry_writeback(&failure_context, recovery, max_attempts), + LoopGuardrailRecoveryDecision::HumanRequired(loop_guardrail_stop) => + apply_loop_guardrail_failure_writeback(&failure_context, loop_guardrail_stop), + }; + } if let Some(loop_guardrail_stop) = loop_guardrail_stop { - return apply_loop_guardrail_failure_writeback(&failure_context, loop_guardrail_stop); + return match loop_guardrail_architecture_recovery_decision( + project, + state_store, + issue_run, + loop_guardrail_stop, + error, + )? { + LoopGuardrailRecoveryDecision::Start(recovery) => + apply_architecture_recovery_retry_writeback(&failure_context, recovery, max_attempts), + LoopGuardrailRecoveryDecision::HumanRequired(loop_guardrail_stop) => + apply_loop_guardrail_failure_writeback(&failure_context, loop_guardrail_stop), + }; } if !requires_terminal_attention && retry_budget_attempts < max_attempts { @@ -2108,6 +2789,78 @@ where Ok(()) } +fn apply_architecture_recovery_retry_writeback( + context: &FailureHandlingContext<'_, T>, + recovery: ArchitectureRecoveryStart, + max_attempts: i64, +) -> Result<()> +where + T: IssueTracker, +{ + let retry_attempt = u32::try_from(context.retry_budget_attempts).unwrap_or(u32::MAX).max(1); + let delay = retry_delay(RetryKind::Failure, retry_attempt, context.workflow); + let retry_ready_at_unix_epoch = OffsetDateTime::now_utc().unix_timestamp().saturating_add( + i64::try_from((delay.as_millis().saturating_add(999)) / 1_000).unwrap_or(i64::MAX), + ); + let recovery_max_attempts = + max_attempts.saturating_add(i64::try_from(recovery.max_attempts).unwrap_or(0)); + + state::write_run_retry_schedule( + &context.issue_run.worktree.path, + &context.issue_run.run_id, + context.issue_run.attempt_number, + ARCHITECTURE_RECOVERY_RETRY_KIND, + retry_ready_at_unix_epoch, + )?; + + write_retry_budget_marker( + &context.issue_run.worktree.path, + &context.issue_run.run_id, + context.issue_run.attempt_number, + context.retry_budget_attempts, + )?; + + tracing::warn!( + project_id = context.project.service_id(), + issue_id = context.issue_run.issue.id, + issue = context.issue_run.issue.identifier, + run_id = context.issue_run.run_id, + attempt = context.issue_run.attempt_number, + recovery_attempt = recovery.attempt_number, + max_recovery_attempts = recovery.max_attempts, + branch = context.issue_run.worktree.branch_name, + worktree_path = %context.worktree_path, + "Loop guardrail started autonomous architecture recovery." + ); + + tracker::create_public_comment( + context.tracker, + &context.issue_run.issue.id, + &format_retry_comment(RetryComment { + run_id: &context.issue_run.run_id, + attempt_number: context.issue_run.attempt_number, + retry_budget_attempt_number: context.retry_budget_attempts, + max_attempts: recovery_max_attempts, + worktree_path: context.worktree_path.to_owned(), + branch_name: &context.issue_run.worktree.branch_name, + error_class: "architecture_recovery_started", + next_action: "decodex recorded a within-authority boundary check and will retry with a materially different architecture recovery strategy", + }), + )?; + + record_harness_outcome_best_effort( + context.state_store, + context.project.service_id(), + context.issue_run, + HarnessOutcomeKind::RetryableFailure, + Some("architecture_recovery_started"), + Some("architecture_recovery_started"), + None, + ); + + Ok(()) +} + fn apply_loop_guardrail_failure_writeback( context: &FailureHandlingContext<'_, T>, loop_guardrail_stop: LoopGuardrailStopRequested, diff --git a/apps/decodex/src/orchestrator/harness_improvement.rs b/apps/decodex/src/orchestrator/harness_improvement.rs index 0338a9674..641f33678 100644 --- a/apps/decodex/src/orchestrator/harness_improvement.rs +++ b/apps/decodex/src/orchestrator/harness_improvement.rs @@ -202,6 +202,7 @@ struct HarnessOutcomeSignals { authority_boundary_dispositions: std::collections::BTreeSet, authority_boundary_failed_check_count: usize, authority_boundary_candidates: Vec, + architecture_recovery_budget_exhausted_count: usize, } pub(crate) fn record_harness_outcome_for_issue_run( @@ -524,6 +525,9 @@ fn harness_outcome_signals( "review_checkpoint" => push_review_signal(&mut signals, event.payload()), "loop_guardrail_checkpoint" => push_guardrail_signal(&mut signals, event.payload()), "authority_boundary_check" => push_authority_boundary_signal(&mut signals, event.payload()), + "architecture_recovery_terminal" => { + push_architecture_recovery_signal(&mut signals, event.payload()); + }, "progress_checkpoint" => push_progress_signal(&mut signals, event.payload()), _ => {}, } @@ -637,6 +641,14 @@ fn push_authority_boundary_signal(signals: &mut HarnessOutcomeSignals, payload: } } +fn push_architecture_recovery_signal(signals: &mut HarnessOutcomeSignals, payload: &Value) { + if json_string(payload.get("reason_code")).as_deref() + == Some("architecture_recovery_exhausted") + { + signals.architecture_recovery_budget_exhausted_count += 1; + } +} + fn push_progress_signal(signals: &mut HarnessOutcomeSignals, payload: &Value) { if json_string(payload.get("phase")).is_some_and(|phase| phase.contains("repair")) { signals.repair_phase_events += 1; @@ -813,6 +825,16 @@ fn push_signal_candidates( ); } + if signals.architecture_recovery_budget_exhausted_count > 0 { + insert_candidate( + candidates, + "recovery_budget_exhausted", + "architecture_recovery_exhausted", + &format!("issue:{}", input.issue_identifier), + signals.architecture_recovery_budget_exhausted_count, + "Increase recovery evidence quality or require a new accepted architecture decision before retrying.", + ); + } if input.error_class == Some("uncovered_direction") || linear_projection.final_error_class.as_deref() == Some("uncovered_direction") { diff --git a/apps/decodex/src/orchestrator/prompting.rs b/apps/decodex/src/orchestrator/prompting.rs index d52af966f..d49ec4b7b 100644 --- a/apps/decodex/src/orchestrator/prompting.rs +++ b/apps/decodex/src/orchestrator/prompting.rs @@ -12,6 +12,63 @@ fn build_retry_recovery_context(dispatch_mode: IssueDispatchMode) -> Option Option { + let events = match state_store + .list_private_execution_events_for_issue(project.service_id(), &issue_run.issue.id) + { + Ok(events) => events, + Err(error) => { + tracing::warn!( + ?error, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + "Prompt could not read architecture recovery evidence." + ); + + return None; + }, + }; + let event = events + .iter() + .rev() + .find(|event| { + matches!( + event.event_type(), + ARCHITECTURE_RECOVERY_PACKET_EVENT_TYPE + | ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE + | ARCHITECTURE_RECOVERY_TERMINAL_EVENT_TYPE + ) + })?; + + if event.event_type() != ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE { + return None; + } + + let payload = event.payload(); + let guardrail_reason = payload + .get("guardrail_reason") + .and_then(Value::as_str) + .unwrap_or("loop_guardrail"); + let recovery_attempt = payload + .get("recovery_budget") + .and_then(|budget| budget.get("attempt")) + .and_then(Value::as_u64) + .unwrap_or(1); + let recovery_max = payload + .get("recovery_budget") + .and_then(|budget| budget.get("max_attempts")) + .and_then(Value::as_u64) + .unwrap_or(1); + + Some(format!( + "Architecture recovery context\n- Decodex recorded `architecture_recovery_started` for guardrail `{guardrail_reason}` after an Authority Boundary Check returned `within_authority`.\n- This is autonomous architecture recovery attempt {recovery_attempt} of {recovery_max}; start a materially different implementation strategy instead of repeating the ineffective repair.\n- Preserve the accepted Decision Contract, public API/config behavior, and validation/review gates. Do not ask the user through chat while detached; use manual attention only if the next viable action crosses authority." + )) +} + fn review_pull_request_title(issue: &TrackerIssue) -> String { let title = issue.title.trim(); let prefix = format!("{}:", issue.identifier); @@ -71,6 +128,11 @@ where if let Some(recovery_context) = build_retry_recovery_context(issue_run.dispatch_mode) { sections.push(recovery_context); } + if let Some(recovery_context) = + build_architecture_recovery_context(project, state_store, issue_run) + { + sections.push(recovery_context); + } let repair_architecture_guidance = build_external_repair_architecture_guidance(project, state_store, issue_run); @@ -168,8 +230,10 @@ where .resolved_completed_state(); let review_level = project.codex().review_level(); let recovery_context = build_retry_recovery_context(issue_run.dispatch_mode) + .into_iter() + .chain(build_architecture_recovery_context(project, state_store, issue_run)) .map(|section| format!("{section}\n\n")) - .unwrap_or_default(); + .collect::(); match issue_run.dispatch_mode { IssueDispatchMode::ReviewRepair => format!( diff --git a/apps/decodex/src/orchestrator/selection.rs b/apps/decodex/src/orchestrator/selection.rs index 2879d6138..a290a0de4 100644 --- a/apps/decodex/src/orchestrator/selection.rs +++ b/apps/decodex/src/orchestrator/selection.rs @@ -199,8 +199,8 @@ fn terminal_failure_comment_details( ) } else if let Some(loop_guardrail_stop) = error.downcast_ref::() { ( - loop_guardrail_stop.reason.error_class(), - loop_guardrail_stop.reason.terminal_next_action(recovery_gate), + loop_guardrail_stop.terminal_error_class(), + loop_guardrail_stop.terminal_next_action(recovery_gate), ) } else if manual_attention_requested { if let Some(manual_attention) = error.downcast_ref::() diff --git a/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs index 45ba7cce1..73ffbf8d2 100644 --- a/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs +++ b/apps/decodex/src/orchestrator/tests/intake/run_and_prompting.rs @@ -672,6 +672,94 @@ fn retry_prompts_include_recovery_context() { } } +#[test] +fn architecture_recovery_prompt_uses_only_latest_active_recovery_start() { + let (_temp_dir, config, workflow) = temp_project_layout(); + let issue = sample_issue("In Progress", &[]); + let tracker = FakeTracker::new(vec![issue.clone()]); + let issue_run = orchestrator::IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("In Progress"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: String::from("PUB-101"), + path: config.worktree_root().join("PUB-101"), + reused_existing: true, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Retry, + attempt_number: 3, + run_id: String::from("pub-101-attempt-3-123"), + retry_budget_base: 2, + }; + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + "pub-101-attempt-2-123", + 2, + "architecture_recovery_started", + serde_json::json!({ + "schema": "decodex.architecture_recovery_started/1", + "reason_code": "architecture_recovery_started", + "guardrail_reason": "review_churn", + "recovery_budget": { + "attempt": 1, + "max_attempts": 1, + }, + }), + ) + .expect("architecture recovery start event should record"); + + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &state_store, + None, + ) + .expect("developer instructions should build"); + + assert!(developer_instructions.contains("Architecture recovery context")); + assert!(developer_instructions.contains("guardrail `review_churn`")); + + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + &issue.id, + "pub-101-attempt-2-123", + 2, + "architecture_recovery_terminal", + serde_json::json!({ + "schema": "decodex.architecture_recovery_terminal/1", + "reason_code": "architecture_recovery_exhausted", + "guardrail_reason": "review_churn", + "recovery_budget": { + "attempt": 2, + "max_attempts": 1, + }, + }), + ) + .expect("architecture recovery terminal event should record"); + + let developer_instructions = orchestrator::build_developer_instructions( + &tracker, + &config, + &workflow, + &issue_run, + &state_store, + None, + ) + .expect("developer instructions should build"); + + assert!(!developer_instructions.contains("Architecture recovery context")); + assert!(!developer_instructions.contains("guardrail `review_churn`")); +} + #[test] fn normal_prompts_respect_non_standard_review_levels() { for (mode, expected, forbidden_checkpoint) in [ diff --git a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs index 86e38ea92..47265f96c 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs @@ -569,6 +569,13 @@ fn harness_outcome_records_validation_review_and_repair_signals() { .iter() .any(|candidate| candidate["reason_code"] == "authority_underspecified") ); + assert!( + payload["improvement_candidates"] + .as_array() + .expect("candidates should be an array") + .iter() + .any(|candidate| candidate["reason_code"] == "architecture_recovery_exhausted") + ); let request = EvidenceRequest { config_path: None, @@ -597,6 +604,21 @@ fn harness_outcome_records_validation_review_and_repair_signals() { .iter() .any(|candidate| candidate.reason_code == "authority_underspecified") ); + assert!( + readback + .improvement_candidates + .iter() + .any(|candidate| candidate.reason_code == "architecture_recovery_exhausted") + ); + assert!( + readback.architecture_recoveries.iter().any(|recovery| { + recovery.reason_code == "architecture_recovery_exhausted" + && recovery.guardrail_reason.as_deref() == Some("validation_repeat") + && recovery.boundary_disposition.as_deref() == Some("within_authority") + && recovery.recovery_budget_attempt == Some(2) + && recovery.recovery_budget_max_attempts == Some(1) + }) + ); assert!(readback.events.iter().all(|event| event.payload.is_none())); } @@ -677,6 +699,27 @@ fn record_harness_signal_fixture_events(state_store: &StateStore) { }, ) .expect("authority boundary evidence should append"); + + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + "issue-harness", + "run-harness", + 2, + "architecture_recovery_terminal", + serde_json::json!({ + "schema": "decodex.architecture_recovery_terminal/1", + "record_version": 1, + "reason_code": "architecture_recovery_exhausted", + "guardrail_reason": "validation_repeat", + "boundary_disposition": "within_authority", + "recovery_budget": { + "attempt": 2, + "max_attempts": 1, + }, + }), + ) + .expect("architecture recovery evidence should append"); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs index f70f2cfb3..8ba31f841 100644 --- a/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs +++ b/apps/decodex/src/orchestrator/tests/recovery/terminal_failures.rs @@ -444,7 +444,7 @@ fn duplicate_passive_retained_review_attention_event_does_not_reapply_tracker_wr } #[test] -fn review_policy_exhausted_failures_skip_retry_and_require_attention_pre_pr() { +fn review_policy_exhausted_failures_start_architecture_recovery_pre_pr() { let (_temp_dir, config, workflow) = temp_project_layout(); let tracker = FakeTracker::new(vec![]); let state_store = StateStore::open_in_memory().expect("state store should open"); @@ -485,16 +485,10 @@ fn review_policy_exhausted_failures_skip_retry_and_require_attention_pre_pr() { orchestrator::handle_failure(&tracker, &config, &workflow, &state_store, &issue_run, &error) .expect("review policy failure handling should succeed"); - assert_eq!( - tracker.state_updates.borrow().last(), - Some(&(issue.id.clone(), String::from("state-todo"))) - ); + assert!(tracker.state_updates.borrow().is_empty()); assert!(tracker.comments.borrow().iter().any(|comment| { - comment.contains("review_policy_exhausted") - && comment.contains("decide the next repair or redesign manually") - && comment.contains("bounded convergence research follow-up") - && comment.contains("machine-checkable") - && comment.contains("clear label `decodex:needs-attention`") + comment.contains("architecture_recovery_started") + && comment.contains("materially different architecture recovery strategy") })); assert!( !tracker @@ -503,6 +497,22 @@ fn review_policy_exhausted_failures_skip_retry_and_require_attention_pre_pr() { .iter() .any(|comment| { comment.contains("retryable_execution_failure") }) ); + + let marker = state::read_run_activity_marker_snapshot(&issue_run.worktree.path) + .expect("run marker should read") + .expect("run marker should exist"); + + assert_eq!(marker.retry_kind(), Some("architecture_recovery")); + + let events = state_store + .list_private_execution_events(config.service_id(), &issue.id, &issue_run.run_id, 1) + .expect("private events should list"); + + assert!(events.iter().any(|event| { + event.event_type() == "architecture_recovery_packet" + && event.payload()["loop_guardrail"]["reason"] == "review_churn" + && event.payload()["authority_boundary_check"]["disposition"] == "within_authority" + })); } #[test] diff --git a/apps/decodex/src/orchestrator/tests/runtime/failure.rs b/apps/decodex/src/orchestrator/tests/runtime/failure.rs index 761ca4c7d..f06b49250 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/failure.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/failure.rs @@ -152,6 +152,7 @@ fn loop_guardrail_terminal_failure_details_normalize_stop_classes() { consecutive_count: 3, fingerprint: String::from("fingerprint"), source_error_class: Some(String::from("repo_gate_verify_failed")), + architecture_recovery_reason_code: None, }); let (actual_error_class, next_action) = orchestrator::terminal_failure_comment_details(false, &error, recovery_gate); @@ -663,6 +664,162 @@ fn loop_guardrail_stops_repeated_validation_failures_after_three_observations() assert_eq!(events[0].payload()["reason"], "validation_repeat"); } +#[test] +fn loop_guardrail_starts_architecture_recovery_when_boundary_is_within_authority() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + let error = || { + Report::new(orchestrator::RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + String::from("Repo verify command `cargo make test` failed: same assertion failed"), + )) + }; + + for attempt_number in 1..=2 { + let issue_run = loop_guardrail_issue_run(&config, &issue, attempt_number); + + assert!( + orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error(), + ) + .expect("guardrail observation should persist") + .is_none() + ); + } + + let issue_run = loop_guardrail_issue_run(&config, &issue, 3); + let error = error(); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("third matching failure should evaluate") + .expect("third matching validation failure should stop"); + let decision = orchestrator::loop_guardrail_architecture_recovery_decision( + &config, + &state_store, + &issue_run, + stop, + &error, + ) + .expect("architecture recovery decision should record"); + let recovery = match decision { + orchestrator::LoopGuardrailRecoveryDecision::Start(recovery) => recovery, + orchestrator::LoopGuardrailRecoveryDecision::HumanRequired(_) => { + panic!("repo-gate validation repeat should recover autonomously") + }, + }; + + assert_eq!(recovery.attempt_number, 1); + assert!(recovery.detail.contains("materially different implementation strategy")); + assert!( + state_store + .loop_guardrail_checkpoint(config.service_id(), &issue.id, "validation_repeat") + .expect("checkpoint read should succeed") + .is_none(), + "started recovery should clear the stopped guardrail reason" + ); + + let events = state_store + .list_private_execution_events(config.service_id(), &issue.id, &issue_run.run_id, 3) + .expect("private events should list"); + + assert!(events.iter().any(|event| { + event.event_type() == "authority_boundary_check" + && event.payload()["disposition"] == "within_authority" + })); + assert!(events.iter().any(|event| { + event.event_type() == "architecture_recovery_packet" + && event.payload()["reason_code"] == "architecture_recovery_started" + && event.payload()["authority_boundary_check"]["disposition"] == "within_authority" + && event.payload()["retained_worktree"]["tracked_status"].is_string() + && event.payload()["validation_failures"]["source_error_class"] + == "repo_gate_verify_failed" + })); + assert!(events.iter().any(|event| { + event.event_type() == "architecture_recovery_started" + && event.payload()["next_strategy"] == "materially_different_architecture_recovery" + })); +} + +#[test] +fn loop_guardrail_requires_human_when_boundary_evidence_is_missing() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue = sample_issue("In Progress", &[]); + + for attempt_number in 1..=2 { + let issue_run = loop_guardrail_issue_run(&config, &issue, attempt_number); + let error = Report::msg("child exited without useful change"); + + assert!( + orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("guardrail observation should persist") + .is_none() + ); + } + + let issue_run = loop_guardrail_issue_run(&config, &issue, 3); + let error = Report::msg("child exited without useful change"); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &config, + &state_store, + &issue_run, + &error, + ) + .expect("third no-diff observation should evaluate") + .expect("no effective diff should stop"); + let decision = orchestrator::loop_guardrail_architecture_recovery_decision( + &config, + &state_store, + &issue_run, + stop, + &error, + ) + .expect("architecture recovery decision should record"); + let terminal_stop = match decision { + orchestrator::LoopGuardrailRecoveryDecision::Start(_) => { + panic!("missing authority evidence must not start recovery") + }, + orchestrator::LoopGuardrailRecoveryDecision::HumanRequired(stop) => stop, + }; + + assert_eq!(terminal_stop.terminal_error_class(), "contract_boundary_required"); + + let events = state_store + .list_private_execution_events(config.service_id(), &issue.id, &issue_run.run_id, 3) + .expect("private events should list"); + + assert!(events.iter().any(|event| { + event.event_type() == "authority_boundary_check" + && event.payload()["disposition"] == "insufficient_evidence" + })); + assert!(events.iter().any(|event| { + event.event_type() == "architecture_recovery_packet" + && event.payload()["reason_code"] == "contract_boundary_required" + && event.payload()["authority_boundary_check"]["disposition"] == "insufficient_evidence" + })); + assert!(events.iter().any(|event| { + event.event_type() == "architecture_recovery_terminal" + && event.payload()["reason_code"] == "contract_boundary_required" + })); + assert!(events.iter().any(|event| { + event.event_type() == "authority_decision_request" + && event.payload()["reason"] == "contract_boundary_required" + })); +} + #[test] fn loop_guardrail_stops_unchanged_remaining_delta_when_validation_text_changes() { let (_temp_dir, config, _workflow) = temp_project_layout(); diff --git a/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs b/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs index f978d6315..a6ce2ccce 100644 --- a/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs +++ b/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs @@ -1,6 +1,6 @@ mod loop_scenarios { use color_eyre::Report; -use serde_json::Value; +use serde_json::{Value, json}; use crate::agent::PhaseGoalController; use crate::agent::PhaseGoalKind; @@ -170,6 +170,7 @@ impl LoopScenarioHarness { self.record_requires_human_authority_boundary(contract.contract_id()); self.record_uncovered_direction_guardrail(); + self.record_architecture_recovery_exhausted(); self.state_store .record_run_attempt( self.run_id, @@ -205,6 +206,30 @@ impl LoopScenarioHarness { ); } + fn record_architecture_recovery_exhausted(&self) { + self.state_store + .append_private_execution_event( + "decodex", + self.issue_id, + self.run_id, + self.attempt_number, + "architecture_recovery_terminal", + json!({ + "schema": "decodex.architecture_recovery_terminal/1", + "record_version": 1, + "reason_code": "architecture_recovery_exhausted", + "guardrail_reason": "validation_repeat", + "authority_boundary_check_record_id": 99, + "boundary_disposition": "within_authority", + "recovery_budget": { + "attempt": 2, + "max_attempts": 1, + }, + }), + ) + .expect("architecture recovery terminal event should persist"); + } + fn record_review_checkpoint_and_assert_repair_then_escalation(&self) -> &'static str { let private_review_marker = "PRIVATE_REVIEW_PAYLOAD_SHOULD_NOT_SURFACE"; let review_payload = loop_scenario_review_payload(1, private_review_marker); @@ -676,6 +701,13 @@ fn loop_scenario_assert_harness_candidates( }), "authority gaps should recommend validator hardening" ); + assert!( + candidates.iter().any(|candidate| { + candidate["kind"] == "recovery_budget_exhausted" + && candidate["reason_code"] == "architecture_recovery_exhausted" + }), + "exhausted architecture recovery should recommend recovery-budget hardening" + ); assert!( !candidate_json.contains(private_review_marker), "harness recommendations must summarize private events without leaking raw payloads" diff --git a/apps/decodex/src/orchestrator/types.rs b/apps/decodex/src/orchestrator/types.rs index 7cc6d07d0..007d0a3ea 100644 --- a/apps/decodex/src/orchestrator/types.rs +++ b/apps/decodex/src/orchestrator/types.rs @@ -10,9 +10,16 @@ pub(crate) const AUTHORITY_DECISION_REQUEST_SCHEMA: &str = pub(crate) const AUTHORITY_DECISION_REQUEST_EVENT_TYPE: &str = "authority_decision_request"; #[allow(dead_code)] pub(crate) const AUTHORITY_BOUNDARY_CHECK_EVENT_TYPE: &str = "authority_boundary_check"; +pub(crate) const ARCHITECTURE_RECOVERY_PACKET_EVENT_TYPE: &str = + "architecture_recovery_packet"; +pub(crate) const ARCHITECTURE_RECOVERY_STARTED_EVENT_TYPE: &str = + "architecture_recovery_started"; +pub(crate) const ARCHITECTURE_RECOVERY_TERMINAL_EVENT_TYPE: &str = + "architecture_recovery_terminal"; #[allow(dead_code)] const AUTHORITY_BOUNDARY_CHECK_SCHEMA: &str = "decodex.authority_boundary_check/1"; +const ARCHITECTURE_RECOVERY_PACKET_SCHEMA: &str = "decodex.architecture_recovery_packet/1"; trait PullRequestReviewStateInspector { fn inspect_review_state( @@ -809,7 +816,7 @@ impl Display for RetainedPartialProgress { impl Error for RetainedPartialProgress {} -#[derive(Debug)] +#[derive(Clone, Debug)] struct LoopGuardrailStopRequested { issue_identifier: String, run_id: String, @@ -817,20 +824,54 @@ struct LoopGuardrailStopRequested { consecutive_count: i64, fingerprint: String, source_error_class: Option, + architecture_recovery_reason_code: Option, +} +impl LoopGuardrailStopRequested { + fn terminal_error_class(&self) -> &'static str { + match self.architecture_recovery_reason_code.as_deref() { + Some("architecture_recovery_exhausted") => "architecture_recovery_exhausted", + Some("contract_boundary_required") => "contract_boundary_required", + Some("external_dependency_required") => "external_dependency_required", + Some("architecture_recovery_started") | None => self.reason.error_class(), + Some(_) => self.reason.error_class(), + } + } + + fn terminal_next_action(&self, recovery_gate: &str) -> String { + match self.architecture_recovery_reason_code.as_deref() { + Some("architecture_recovery_exhausted") => format!( + "inspect the Architecture Recovery Packet and prior recovery attempts; recovery budget is exhausted, {recovery_gate}" + ), + Some("contract_boundary_required") => format!( + "inspect the Authority Boundary Check and resolve the Decision Contract or authority evidence before retrying, {recovery_gate}" + ), + Some("external_dependency_required") => format!( + "inspect the dependency or Execution Program readiness blocker and resolve that external dependency before retrying, {recovery_gate}" + ), + Some("architecture_recovery_started") | None => + self.reason.terminal_next_action(recovery_gate), + Some(_) => self.reason.terminal_next_action(recovery_gate), + } + } } impl Display for LoopGuardrailStopRequested { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let source = self.source_error_class.as_deref().unwrap_or("none"); + let architecture_recovery = self + .architecture_recovery_reason_code + .as_deref() + .unwrap_or("none"); write!( f, - "Run `{}` for issue `{}` hit loop guardrail `{}` after {} consecutive matching observations with source `{}` and fingerprint `{}`; stop automatic retries.", + "Run `{}` for issue `{}` hit loop guardrail `{}` after {} consecutive matching observations with source `{}` and fingerprint `{}`; architecture recovery reason `{}`.", self.run_id, self.issue_identifier, self.reason.error_class(), self.consecutive_count, source, - self.fingerprint + self.fingerprint, + architecture_recovery ) } } diff --git a/apps/decodex/src/state/store.rs b/apps/decodex/src/state/store.rs index 35624a4bc..de2906044 100644 --- a/apps/decodex/src/state/store.rs +++ b/apps/decodex/src/state/store.rs @@ -1519,6 +1519,25 @@ impl StateStore { Ok(records.into_iter().map(|record| record.as_public()).collect()) } + /// List private execution events for one project/issue tuple. + pub(crate) fn list_private_execution_events_for_issue( + &self, + project_id: &str, + issue_id: &str, + ) -> Result> { + let state = self.lock()?; + let mut records = state + .private_execution_events + .iter() + .filter(|record| record.project_id == project_id && record.issue_id == issue_id) + .cloned() + .collect::>(); + + records.sort_by(compare_private_execution_event_runtime_records); + + Ok(records.into_iter().map(|record| record.as_public()).collect()) + } + /// List private execution events for one project/run/attempt tuple. pub fn list_private_execution_events_for_run_attempt( &self, diff --git a/docs/runbook/lane-control-recovery.md b/docs/runbook/lane-control-recovery.md index e07351bd1..8610077f3 100644 --- a/docs/runbook/lane-control-recovery.md +++ b/docs/runbook/lane-control-recovery.md @@ -91,7 +91,7 @@ or clean labels. | Queue label should be added, removed, or interpreted. | Use service-scoped label policy. | Follow the `labels` skill; do not guess `` or clear `needs-attention` before fixing the blocker. | | Broad steer materially changes the objective or acceptance contract. | Preserve audit and resolve lifecycle explicitly. | Update and requeue the same issue, create a new issue/lane, or route the owned run to manual attention. | | Operator wants a different issue or replacement task. | Treat as task replacement, not steer. | Stop or pause through supported controls as needed, then create/update/requeue through the supported lifecycle. | -| Status or Linear failure summary reports a loop guardrail reason. | Stop automatic recovery and inspect the reason-specific evidence. | Follow the loop guardrail recovery table below before clearing `decodex:needs-attention` or requeueing. | +| Status or Linear failure summary reports a loop guardrail reason. | Inspect the reason-specific evidence, Architecture Recovery Packet, and Authority Boundary Check. | Follow the loop guardrail recovery table below before clearing `decodex:needs-attention` or requeueing. | | Authority Boundary Check reports `within_authority`. | Continue only if lane identity, ownership, and validation evidence still match. | Resume through the supported retained-lane path; keep the boundary-check event as private evidence. | | Authority Boundary Check reports `requires_human`. | Stop automatic recovery and preserve the durable decision request. | Keep or apply `decodex:needs-attention`, inspect the Linear decision request and private `authority_decision_request` evidence, then continue only after the issue, Decision Contract, or policy accepts, rejects, or revises the requested authority change. | | Authority Boundary Check reports `insufficient_evidence`. | Stop automatic recovery unless later evidence proves the change is inside authority. | Keep or apply `decodex:needs-attention`, capture the missing evidence, and continue only after the Decision Contract, issue, policy, or human direction explicitly authorizes the change. | @@ -103,20 +103,36 @@ Loop guardrails stop non-converging automation after three consecutive matching observations. They preserve retained worktrees and private evidence; they do not mean the operator should delete local progress to make the queue clean. +Current runtime guardrail handling is two-stage: + +1. stop the current ineffective repair strategy and record a private + `architecture_recovery_packet` +2. record or consume an Authority Boundary Check to decide whether autonomous + architecture recovery may continue + +If the boundary check is `within_authority` and recovery budget remains, Decodex may +record `architecture_recovery_started` and retry with a materially different +implementation strategy. If the check is `requires_human` or `insufficient_evidence`, +or if recovery budget is exhausted, Decodex must keep or apply manual attention with a +typed reason such as `contract_boundary_required`, `external_dependency_required`, or +`architecture_recovery_exhausted`. + | Guardrail reason | Inspect first | Resume only after | | --- | --- | --- | -| `validation_repeat` | The repeated validation failure, repo-gate output, retained worktree, and prior repair attempts. | The repair strategy changes, the validation cause is fixed manually, or the issue is routed to architecture/research review. | -| `no_effective_diff` | The retained worktree status, private retry evidence, and whether any useful tracked delta exists. | A human identifies a concrete next diff, commits/resets the retained work intentionally, or updates the issue scope. | -| `remaining_delta_unchanged` | The unchanged tracked delta and latest validation evidence. | The next repair is bounded and materially different, or the retained patch is accepted/reset manually. | -| `review_churn` or `review_policy_exhausted` | Fresh-context review checkpoints, accepted findings, rejected findings, and the current head. | A new repair strategy, architecture review, or manual decision is recorded. | -| `dependency_program_stale` | The open blocker issue, Execution Program readiness, and whether the dependency split is still correct. | The dependency is resolved, the program is refreshed/split, or a research/decision contract updates execution authority. | +| `validation_repeat` | The repeated validation failure, repo-gate output, retained worktree, prior repair attempts, Architecture Recovery Packet, and boundary disposition. | Autonomous recovery may continue only when the boundary is `within_authority` and budget remains; otherwise a human fixes the cause or records new authority. | +| `no_effective_diff` | The retained worktree status, private retry evidence, whether any useful tracked delta exists, Architecture Recovery Packet, and boundary disposition. | Autonomous recovery may continue only when evidence proves the next strategy is an engineering implementation change inside authority; otherwise a human identifies the next diff, resets intentionally, or updates authority. | +| `remaining_delta_unchanged` | The unchanged tracked delta, latest validation evidence, Architecture Recovery Packet, and boundary disposition. | The next repair must be bounded, materially different, and inside authority; otherwise a human accepts/resets the patch or updates authority. | +| `review_churn` or `review_policy_exhausted` | Fresh-context review checkpoints, accepted findings, rejected findings, current head, Architecture Recovery Packet, and boundary disposition. | A materially different implementation strategy may continue only inside authority; architecture/product direction changes require human authority. | +| `dependency_program_stale` | The open blocker issue, Execution Program readiness, and whether the dependency split is still correct. | Resolve the dependency, refresh/split the program, or update the research/Decision Contract; do not auto-recover as ordinary implementation work. | | `uncovered_direction` | The missing requirement, decision, or research gap named in public/private evidence. | A research or Decision Contract captures the missing direction and the issue is updated or requeued from that authority. | -| `ambiguous_retained_progress` | Retained worktree diff, ownership markers, PR lineage if present, and private evidence. | A human chooses one path: resume same lane, finish manual repair, or reset/discard the retained patch explicitly. | - -For every guardrail stop, keep `decodex:needs-attention` until the blocker above is -resolved. If the issue returns to automation, request a Linear scan or let the next -scheduled scan observe the corrected tracker state; do not bypass the guardrail with a -manual retry that leaves the same evidence unchanged. +| `ambiguous_retained_progress` | Retained worktree diff, ownership markers, PR lineage if present, private evidence, and boundary disposition. | A human chooses one path: resume same lane, finish manual repair, or reset/discard the retained patch explicitly. | + +For every human-required guardrail stop, keep `decodex:needs-attention` until the +blocker above is resolved. If the issue returns to automation, request a Linear scan +or let the next scheduled scan observe the corrected tracker state; do not bypass the +guardrail with a manual retry that leaves the same evidence unchanged. Do not clear +manual attention for `architecture_recovery_exhausted` until a new accepted recovery +strategy, issue update, or Decision Contract update exists. For authority-boundary decision requests, the supported resume sequence is: diff --git a/docs/spec/agent-evidence.md b/docs/spec/agent-evidence.md index 0ecc4ccac..071c0a83f 100644 --- a/docs/spec/agent-evidence.md +++ b/docs/spec/agent-evidence.md @@ -165,9 +165,15 @@ The readback includes: boundary, recommendation, resume condition, and next action, but not retained worktree evidence, raw diff payloads, recovery context, transcripts, logs, or credentials unless `--include-payload` is explicitly requested for local repair. +- Architecture Recovery Packet summaries when `architecture_recovery_packet`, + `architecture_recovery_started`, or `architecture_recovery_terminal` events are + present. Default readback may expose the recovery reason code, guardrail reason, + boundary disposition, recovery budget state, and compact next action, but not raw + retained diffs, transcript text, logs, credentials, or full private packet payloads + unless `--include-payload` is explicitly requested for local repair. - harness improvement candidates derived from `decodex.harness_outcome/1` events or, when no harness outcome has been recorded yet, directly from private validation, - review, guardrail, and authority-boundary signals + review, guardrail, authority-boundary, and architecture-recovery signals - `private_execution_evidence_missing` when the selected run is known but has no private execution events diff --git a/docs/spec/loop-runtime.md b/docs/spec/loop-runtime.md index bbb24b8c0..a7e641a73 100644 --- a/docs/spec/loop-runtime.md +++ b/docs/spec/loop-runtime.md @@ -181,8 +181,9 @@ Contract, issue, project policy, lane ownership, and private evidence. Before autonomous loop recovery continues a detached or guardrail-pressured lane, the runtime must record a private Authority Boundary Check when the attempted recovery could change the Authority Envelope or when evidence is too weak to prove that it does -not. This check is evidence plumbing for downstream recovery workers; this spec does -not implement the full autonomous architecture recovery execution loop. +not. Guardrail recovery may continue only when this check returns +`within_authority` and the recovery budget still has room. `requires_human` and +`insufficient_evidence` are hard stops for the current autonomous lane. The private payload is versioned as `decodex.authority_boundary_check/1` with `event_type = "authority_boundary_check"` in `private_execution_events`. It records: @@ -235,6 +236,44 @@ or readiness-model hardening from boundary-check failures. Those recommendations advisory. They do not modify the accepted Decision Contract, queue eligibility, or project policy by themselves. +## Architecture Recovery Packet + +When loop guardrails detect non-converging automation, Decodex first stops the current +ineffective strategy. It then records a private Architecture Recovery Packet before +deciding whether to continue autonomously or require human attention. + +The private payload is versioned as `decodex.architecture_recovery_packet/1` with +`event_type = "architecture_recovery_packet"` in `private_execution_events`. It +records: + +- issue id, issue identifier, title, run id, attempt number, branch, and dispatch mode +- Decision Contract ids, statuses, source issue ids, and update timestamps when known +- Execution Program ids linked to those contracts when known +- retained worktree HEAD, tracked status, diff/status hashes, and compact diff stat +- validation failure or loop-guardrail source class and private error summary +- latest review checkpoint status and accepted/rejected finding counts when present +- prior architecture recovery attempts for the issue +- recovery budget attempt and maximum +- loop-guardrail reason, threshold, consecutive count, fingerprint, and source class +- linked Authority Boundary Check record id, disposition, and final reason + +If the boundary check is `within_authority` and budget remains, Decodex records +`event_type = "architecture_recovery_started"` with reason code +`architecture_recovery_started`, clears the stopped guardrail reason, and starts a +materially different implementation strategy. This recovery may change internal +architecture, plumbing, tests, and docs needed to satisfy the same accepted objective. +It must not weaken validation or review gates. + +If recovery would exceed the Authority Envelope, lacks enough evidence to prove that +it stays inside the envelope, depends on external/manual state, or exhausts its +bounded recovery budget, Decodex records `event_type = +"architecture_recovery_terminal"` with a reason code such as +`contract_boundary_required`, `external_dependency_required`, or +`architecture_recovery_exhausted`, then routes through the human-required failure +path. Boundary stops should also record an `authority_decision_request` private event +so operator status can surface the decision request without exposing raw private +payloads. + ## Promotion Boundary Promotion is the boundary between design and execution authority. @@ -429,8 +468,25 @@ generic retry bucket. Normalized outcomes include: | `uncovered_direction` | Execution uncovered a missing or contradictory decision contract. Legacy summaries may still use `research_contract_required`. | | `ambiguous_retained_progress` | Tracker, PR, branch, retained worktree, or runtime ownership evidence is contradictory. Legacy summaries may still use `ownership_ambiguous`. | -These outcomes should route to failure attribution, research-contract feedback, -architecture review, or manual attention. They must not spin in automatic retries. +These outcomes first stop the ineffective strategy. The runtime must then classify +whether autonomous architecture recovery is allowed: + +- Engineering implementation problems such as repeated repo-gate validation failures + or no-effective-diff repair loops may continue only after an Authority Boundary + Check records `within_authority` and recovery budget remains. +- Product goal, accepted behavior, public API/config/workflow contract, security, + credential, billing, privacy, destructive data-loss, validation-weakening, review + weakening, lane-ownership, or accepted Decision Contract changes must stop with a + human-required reason. +- Dependency or Execution Program staleness that requires external/manual state must + stop with `external_dependency_required` or an equivalent typed reason. +- Missing or contradictory evidence must stop with `contract_boundary_required`, + `insufficient_evidence`, or an equivalent typed reason until accepted authority is + recorded. + +Allowed recovery attempts are bounded and recorded. Exhausting the recovery budget is +itself a terminal recovery outcome, not a reason to silently fall back to the same +patch strategy. ## Harness Improvement Loop @@ -460,8 +516,8 @@ versioned payload is `decodex.harness_outcome/1` with `event_type = - PR handoff, retained repair, closeout, cleanup, and terminal failure outcomes as summarized from cached Linear execution records and local runtime state - candidate harness improvements such as `missing_validator`, `weak_prompt`, - `missing_issue_template_field`, `underspecified_decision_contract`, and - `stale_readiness_model` + `missing_issue_template_field`, `underspecified_decision_contract`, + `stale_readiness_model`, and `recovery_budget_exhausted` Operator readback may summarize improvement candidates through `decodex evidence` or derived agent-evidence files. Default readback must expose only compact candidate kind, diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 40b3038fe..503f50794 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -363,7 +363,13 @@ The continued-repair classes above are ordinary bounded churn: the coding agent Continued repair is still bounded by loop guardrails. For each retryable failure, Decodex records local `loop_guardrail_checkpoint` evidence and updates the `loop_guardrail_checkpoints` row for any matching convergence reason. Three -consecutive observations with the same fingerprint stop automatic retry and route the +consecutive observations with the same fingerprint stop the current ineffective +strategy. Before terminalizing the lane, Decodex records an Architecture Recovery +Packet and an Authority Boundary Check. If the disposition is `within_authority` and +the bounded recovery budget remains, Decodex records `architecture_recovery_started` +and schedules a materially different recovery strategy. Otherwise it records a +terminal recovery reason such as `contract_boundary_required`, +`external_dependency_required`, or `architecture_recovery_exhausted` and routes the lane through the human-required failure path. Repo-gate failures record both `validation_repeat` and `remaining_delta_unchanged` observations; retryable failures with no changing tracked delta record `no_effective_diff`. Fingerprints use the lane @@ -377,7 +383,7 @@ state for the current phase and current lane head from the owned lane: - no checkpoint and no terminal path: allow a clean continuation boundary - latest checkpoint `clean` and no terminal path: allow continuation so the agent can finish handoff or repair completion - latest checkpoint `findings` with fewer than three consecutive non-clean rounds in the same phase: allow continuation -- latest checkpoint `findings` with three or more consecutive non-clean rounds in the same phase: fail the turn through the human-required failure path +- latest checkpoint `findings` with three or more consecutive non-clean rounds in the same phase: treat this as `review_churn`, stop the current repair strategy, and run the architecture recovery boundary check before either retrying with a materially different implementation strategy or routing to the human-required path - latest checkpoint `needs_architecture_review` or `blocked`: fail the turn through the human-required failure path `decodex` persists this review-policy state in the runtime SQLite @@ -404,9 +410,13 @@ or repair completion, and ignores stale review-policy state while classifying cl turn boundaries. The review-policy human-required failure path is also the boundary for any later -runtime-owned research escalation. The current runtime must not dispatch research from a -review stop. Future escalation may only consume structured review-stop evidence through -the adapter contract defined by [`review-orchestration.md`](./review-orchestration.md). +runtime-owned research escalation. The current runtime must not dispatch research from +a review stop. Exhausted review findings may enter architecture recovery only as an +implementation-strategy change after the Authority Boundary Check returns +`within_authority`; `needs_architecture_review` and `blocked` review stops remain +human-required. Future research escalation may only consume structured review-stop +evidence through the adapter contract defined by +[`review-orchestration.md`](./review-orchestration.md). ### Success writeback @@ -473,15 +483,18 @@ linked to the Authority Boundary Check record. Status JSON and dashboard snapsho must expose the compact request fields (`phase = human_required`, reason, boundary, `decision_request_id`, and `next_action`) so the lane is operable without inspecting SQLite directly. -Runtime-owned review-policy stops use the same human-required failure path, but with dedicated `error_class` values: +Runtime-owned review-policy stops use either bounded architecture recovery or the same +human-required failure path, with dedicated `error_class` values: -- `review_policy_exhausted` +- `review_policy_exhausted`: normalized to `review_churn` for the architecture + recovery boundary check before terminal human attention is considered. - `architecture_review_required` - `review_policy_blocked` -Runtime loop guardrails use the same human-required failure path, but preserve a -structured failure-attribution `error_class` so operator status and Linear summaries -can distinguish the stop class: +Runtime loop guardrails may start bounded architecture recovery before using the +human-required failure path. When recovery cannot proceed, the terminal writeback +preserves a structured failure-attribution `error_class` so operator status and +Linear summaries can distinguish the stop class or recovery boundary: - `validation_repeat`: the same validation failure repeated three times. - `no_effective_diff`: retryable attempts repeated without a changed tracked delta. @@ -494,6 +507,12 @@ can distinguish the stop class: research or Decision Contract before more implementation. - `ambiguous_retained_progress`: retained local work or ownership evidence is useful but ambiguous enough that a human must choose resume, reset, or manual repair. +- `contract_boundary_required`: recovery would change accepted authority, or evidence + is insufficient to prove that it would not. +- `external_dependency_required`: recovery depends on a dependency, project policy, + or Execution Program readiness change outside the current lane. +- `architecture_recovery_exhausted`: the lane already used its bounded autonomous + architecture recovery budget. Review repair churn uses the bounded review policy above and may appear publicly as `review_policy_exhausted` or the normalized loop reason `review_churn`; both mean the @@ -502,7 +521,8 @@ the next strategy is explicit. When a stopped lane is eligible for future autonomous recovery, the recovery worker must consume the latest Authority Boundary Check or record a fresh one before changing -implementation direction. `requires_human` and `insufficient_evidence` dispositions +implementation direction. `requires_human` and `insufficient_evidence` dispositions, +external/manual blockers, validation/review weakening, and exhausted recovery budgets must route through the human-required path or a later accepted recovery contract; they must not be treated as retryable repo-gate failures. The supported resume path is deliberate: accept, reject, or revise the requested