From c374e840472ce102ee8eeffe403061058b0caad2 Mon Sep 17 00:00:00 2001 From: Yvette Carlisle Date: Wed, 10 Jun 2026 04:25:24 +0800 Subject: [PATCH] {"schema":"decodex/commit/1","summary":"Add research-to-execution loop scenario tests","authority":"XY-859"} --- apps/decodex/src/orchestrator/tests.rs | 2 + .../tests/runtime/loop_scenarios.rs | 638 ++++++++++++++++++ docs/reference/test-suite.md | 28 +- docs/runbook/index.md | 3 + docs/runbook/research-to-execution-loop.md | 100 +++ 5 files changed, 758 insertions(+), 13 deletions(-) create mode 100644 apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs create mode 100644 docs/runbook/research-to-execution-loop.md diff --git a/apps/decodex/src/orchestrator/tests.rs b/apps/decodex/src/orchestrator/tests.rs index dcb27cac8..6ca2a8722 100644 --- a/apps/decodex/src/orchestrator/tests.rs +++ b/apps/decodex/src/orchestrator/tests.rs @@ -73,6 +73,8 @@ include!("tests/runtime/repo_gate.rs"); include!("tests/runtime/failure.rs"); +include!("tests/runtime/loop_scenarios.rs"); + include!("tests/runtime/thread_archive.rs"); include!("tests/recovery/reconciliation.rs"); diff --git a/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs b/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs new file mode 100644 index 000000000..ffb1043c7 --- /dev/null +++ b/apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs @@ -0,0 +1,638 @@ +mod loop_scenarios { +use color_eyre::Report; +use serde_json::Value; + +use crate::agent::PhaseGoalController; +use crate::agent::PhaseGoalKind; +use crate::agent::PhaseGoalSpec; +use crate::agent::PhaseGoalTransition; +use crate::agent::ReviewPolicyStopReason; +use crate::execution_program::ExecutionLinearIssueMapping; +use crate::execution_program::ExecutionProgram; +use crate::execution_program::ExecutionProgramEvaluation; +use crate::execution_program::ExecutionProgramNodeStage; +use crate::execution_program::ExecutionProgramReadinessContext; +use crate::execution_program::ExecutionQueueIntent; +use crate::execution_program::ExecutionQueueLabelAction; +use crate::execution_program::ExecutionWorkflowPolicy; +use crate::loop_contract::DecisionContract; +use crate::loop_contract::DecisionContractStatus; +use crate::loop_contract::DecisionPromotion; +use crate::loop_contract::DecisionPromotionActorKind; +use crate::orchestrator; +use crate::orchestrator::HarnessOutcomeKind; +use crate::orchestrator::HarnessOutcomeRecordInput; +use crate::orchestrator::IssueDispatchMode; +use crate::orchestrator::IssueRunPlan; +use crate::orchestrator::RepoGateFailure; +use crate::orchestrator::RepoGateFailureKind; +use crate::orchestrator::RepoGatePhaseGoalController; +use crate::state::LoopGuardrailCheckpointInput; +use crate::state::ReviewPolicyCheckpointInput; +use crate::state::StateStore; +use crate::tracker; +use crate::worktree::WorktreeSpec; + +const LOOP_SCENARIO_GATE_SERVICE_ID: &str = "pubfi"; + +struct LoopScenarioHarness { + state_store: StateStore, + issue_id: &'static str, + issue_identifier: &'static str, + run_id: &'static str, + attempt_number: i64, +} +impl LoopScenarioHarness { + fn new() -> Self { + Self { + state_store: StateStore::open_in_memory().expect("state store should open"), + issue_id: "issue-xy-859", + issue_identifier: "XY-859", + run_id: "run-loop-scenario", + attempt_number: 3, + } + } + + fn assert_latent_research_stays_non_executable(&self) -> DecisionContract { + let contract = loop_scenario_research_x_contract(); + + assert_eq!(contract.status(), DecisionContractStatus::DraftLatent); + + self.state_store + .upsert_decision_contract("decodex", Some(self.issue_identifier), contract.clone()) + .expect("latent contract should persist"); + + assert!( + ExecutionProgram::from_accepted_contract( + "program-before-promotion", + "decodex", + &contract, + Vec::new(), + ) + .is_err(), + "latent research output must not authorize execution" + ); + assert!( + self.state_store + .list_execution_programs_for_contract("decodex", contract.contract_id()) + .expect("programs should load") + .is_empty(), + "latent research output must not create executable programs" + ); + + contract + } + + fn promote_and_evaluate_program( + &self, + mut contract: DecisionContract, + ) -> ( + DecisionContract, + ExecutionWorkflowPolicy, + ExecutionProgramEvaluation, + ) { + contract + .promote( + DecisionPromotion::new( + "operator", + DecisionPromotionActorKind::User, + "2026-06-10T00:00:00Z", + "conversation", + Some(String::from("User promoted the research result for implementation.")), + ) + .expect("promotion metadata should build"), + ) + .expect("operator promotion should accept the contract"); + self.state_store + .upsert_decision_contract("decodex", Some(self.issue_identifier), contract.clone()) + .expect("promoted contract should persist"); + + let policy = loop_scenario_decodex_workflow_policy(); + let program = ExecutionProgram::from_accepted_contract( + "program-loop-scenario", + "decodex", + &contract, + loop_scenario_program_nodes(), + ) + .expect("accepted contract should shape a program"); + + self.state_store + .upsert_execution_program("decodex", program.clone()) + .expect("program should persist"); + + let evaluation = program + .evaluate(&contract, &policy, &ExecutionProgramReadinessContext::new()) + .expect("program should evaluate"); + + (contract, policy, evaluation) + } + + fn assert_queue_shaping( + &self, + policy: &ExecutionWorkflowPolicy, + evaluation: &ExecutionProgramEvaluation, + ) { + let summary = evaluation.operator_summary(); + + assert_eq!(policy.queue_label(), "decodex:queued:decodex"); + assert_eq!(evaluation.ready_node_ids(), vec!["node-ready"]); + assert_eq!(evaluation.startable_node_ids(), vec!["node-ready"]); + assert_eq!(summary.ready_count, 1); + assert_eq!(summary.blocked_count, 1); + assert_eq!(summary.paused_count, 1); + assert_eq!(summary.active_count, 1); + assert_eq!(summary.queue_label_eligible_count, 1); + assert_eq!( + loop_scenario_queue_action(evaluation, "node-ready"), + Some(ExecutionQueueLabelAction::Apply) + ); + assert_eq!( + loop_scenario_queue_action(evaluation, "node-uncovered-direction"), + Some(ExecutionQueueLabelAction::Remove) + ); + assert_eq!( + loop_scenario_queue_action(evaluation, "node-active"), + Some(ExecutionQueueLabelAction::Remove) + ); + } + + fn record_review_guardrail_and_assert_harness_feedback( + &self, + contract: DecisionContract, + ) { + let private_review_marker = + self.record_review_checkpoint_and_assert_repair_then_escalation(); + + self.record_uncovered_direction_guardrail(); + self.state_store + .record_run_attempt( + self.run_id, + self.issue_id, + self.attempt_number, + "terminal_guarded", + ) + .expect("run should persist"); + self.state_store + .upsert_decision_contract("decodex", Some(self.issue_id), contract) + .expect("issue-linked contract should persist"); + + let recorded = orchestrator::record_harness_outcome_for_issue_run( + &self.state_store, + HarnessOutcomeRecordInput { + project_id: "decodex", + issue_id: self.issue_id, + issue_identifier: self.issue_identifier, + run_id: self.run_id, + attempt_number: self.attempt_number, + outcome: HarnessOutcomeKind::ManualAttention, + error_class: Some("uncovered_direction"), + validation_result: None, + pr_url: None, + }, + ) + .expect("harness outcome should record"); + + loop_scenario_assert_harness_candidates(recorded.payload(), private_review_marker); + } + + fn record_review_checkpoint_and_assert_repair_then_escalation(&self) -> &'static str { + let private_review_marker = "PRIVATE_REVIEW_PAYLOAD_SHOULD_NOT_SURFACE"; + let review_payload = loop_scenario_review_payload(1, private_review_marker); + + self.state_store + .upsert_review_policy_checkpoint(ReviewPolicyCheckpointInput { + project_id: "decodex", + issue_id: self.issue_id, + run_id: self.run_id, + attempt_number: self.attempt_number, + phase: "handoff", + status: "findings", + head_sha: "0123456789abcdef0123456789abcdef01234567", + nonclean_rounds: 1, + details_json: &review_payload.to_string(), + }) + .expect("review checkpoint should persist"); + self.state_store + .append_private_execution_event( + "decodex", + self.issue_id, + self.run_id, + self.attempt_number, + "review_checkpoint", + review_payload, + ) + .expect("review event should persist"); + self.assert_review_checkpoint(1); + self.record_review_churn_escalation(); + + private_review_marker + } + + fn assert_review_checkpoint(&self, expected_rounds: i64) { + let checkpoint = self + .state_store + .review_policy_checkpoint( + "decodex", + self.issue_id, + self.run_id, + self.attempt_number, + "handoff", + ) + .expect("checkpoint lookup should succeed") + .expect("review checkpoint should exist"); + + assert_eq!(checkpoint.status(), "findings"); + assert_eq!(checkpoint.nonclean_rounds(), expected_rounds); + } + + fn record_review_churn_escalation(&self) { + let review_churn_payload = loop_scenario_review_payload(3, "review churn"); + + self.state_store + .upsert_review_policy_checkpoint(ReviewPolicyCheckpointInput { + project_id: "decodex", + issue_id: self.issue_id, + run_id: self.run_id, + attempt_number: self.attempt_number, + phase: "handoff", + status: "findings", + head_sha: "0123456789abcdef0123456789abcdef01234567", + nonclean_rounds: 3, + details_json: &review_churn_payload.to_string(), + }) + .expect("review churn checkpoint should persist"); + self.assert_review_checkpoint(3); + + assert_eq!( + ReviewPolicyStopReason::Exhausted.error_class(), + "review_policy_exhausted" + ); + } + + fn record_uncovered_direction_guardrail(&self) { + let uncovered_payload = loop_scenario_uncovered_payload(); + let uncovered_checkpoint = self + .state_store + .observe_loop_guardrail_checkpoint(LoopGuardrailCheckpointInput { + project_id: "decodex", + issue_id: self.issue_id, + reason: "uncovered_direction", + fingerprint: "contract-gap:operator-feedback", + run_id: self.run_id, + attempt_number: self.attempt_number, + details_json: &uncovered_payload.to_string(), + }) + .expect("uncovered direction checkpoint should persist"); + + self.state_store + .append_private_execution_event( + "decodex", + self.issue_id, + self.run_id, + self.attempt_number, + "loop_guardrail_checkpoint", + uncovered_payload, + ) + .expect("guardrail event should persist"); + + assert_eq!(uncovered_checkpoint.reason(), "uncovered_direction"); + assert_eq!(uncovered_checkpoint.consecutive_count(), 1); + } +} + +#[test] +fn research_to_execution_loop_scenario_shapes_ready_work_and_records_feedback() { + let harness = LoopScenarioHarness::new(); + let contract = harness.assert_latent_research_stays_non_executable(); + let (contract, policy, evaluation) = harness.promote_and_evaluate_program(contract); + + harness.assert_queue_shaping(&policy, &evaluation); + harness.record_review_guardrail_and_assert_harness_feedback(contract); +} + +#[test] +fn loop_scenario_phase_completion_runs_validation_and_guardrails_bound_repair() { + loop_scenario_assert_phase_goal_completion_runs_validation(); + loop_scenario_assert_validation_guardrail_stops_after_threshold(); +} + +fn loop_scenario_assert_phase_goal_completion_runs_validation() { + let workflow_markdown = super::sample_workflow_markdown( + "pubfi", + &[], + "Phase goal validation policy.\n", + 3, + ) + .replace( + "canonicalize_commands = []", + "canonicalize_commands = [\"printf canonicalized > phase-canonicalized.txt\"]", + ) + .replace( + "verify_commands = []", + "verify_commands = [\"test -f phase-canonicalized.txt && printf verified > phase-verified.txt\"]", + ); + let (_temp_dir, config, workflow) = + super::temp_project_layout_with_workflow_markdown(&workflow_markdown); + let issue = + super::sample_issue("In Progress", &[tracker::automation_active_label(LOOP_SCENARIO_GATE_SERVICE_ID).as_str()]); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let issue_run = IssueRunPlan { + issue: issue.clone(), + issue_state: String::from("In Progress"), + initial_issue_state: String::from("Todo"), + worktree: WorktreeSpec { + branch_name: String::from("x/pubfi-pub-101"), + issue_identifier: issue.identifier.clone(), + path: config.repo_root().to_path_buf(), + reused_existing: false, + }, + retry_project_slug: String::from("pubfi"), + dispatch_mode: IssueDispatchMode::Normal, + attempt_number: 1, + run_id: String::from("pub-101-attempt-1"), + retry_budget_base: 0, + }; + let transition = RepoGatePhaseGoalController { + project: &config, + workflow: &workflow, + state_store: &state_store, + issue_run: &issue_run, + } + .phase_goal_completed(PhaseGoalKind::ImplementToValidationReady) + .expect("completed implementation phase should run validation"); + let events = state_store + .list_private_execution_events( + LOOP_SCENARIO_GATE_SERVICE_ID, + &issue.id, + &issue_run.run_id, + 1, + ) + .expect("phase goal events should load"); + + assert!(config.repo_root().join("phase-canonicalized.txt").exists()); + assert!(config.repo_root().join("phase-verified.txt").exists()); + assert!(matches!( + transition, + PhaseGoalTransition::Continue(PhaseGoalSpec { + phase: PhaseGoalKind::HandoffEvidence, + .. + }) + )); + assert!(!matches!(transition, PhaseGoalTransition::CompleteRun)); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_transition" + && event.payload()["signal"] == "validation_pass" + })); + assert!(events.iter().any(|event| { + event.event_type() == "phase_goal_next" + && event.payload()["phase"] == "handoff_evidence" + })); +} + +fn loop_scenario_assert_validation_guardrail_stops_after_threshold() { + let (_guardrail_temp_dir, guardrail_config, _guardrail_workflow) = super::temp_project_layout(); + let guardrail_store = StateStore::open_in_memory().expect("guardrail store should open"); + let guardrail_issue = super::sample_issue("In Progress", &[]); + + for round in 1..=2 { + let guardrail_issue_run = + super::loop_guardrail_issue_run(&guardrail_config, &guardrail_issue, round); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &guardrail_config, + &guardrail_store, + &guardrail_issue_run, + &loop_scenario_repo_gate_failure(), + ) + .expect("guardrail observation should persist"); + + assert!( + stop.is_none(), + "round {round} should keep repairing before the threshold" + ); + } + + let guardrail_issue_run = super::loop_guardrail_issue_run(&guardrail_config, &guardrail_issue, 3); + let stop = orchestrator::retryable_failure_loop_guardrail_stop( + &guardrail_config, + &guardrail_store, + &guardrail_issue_run, + &loop_scenario_repo_gate_failure(), + ) + .expect("third guardrail observation should persist") + .expect("third identical failure should stop repair churn"); + let checkpoint = guardrail_store + .loop_guardrail_checkpoint( + LOOP_SCENARIO_GATE_SERVICE_ID, + &guardrail_issue_run.issue.id, + "validation_repeat", + ) + .expect("checkpoint lookup should succeed") + .expect("validation repeat checkpoint should exist"); + let guardrail_events = guardrail_store + .list_private_execution_events( + LOOP_SCENARIO_GATE_SERVICE_ID, + &guardrail_issue_run.issue.id, + &guardrail_issue_run.run_id, + guardrail_issue_run.attempt_number, + ) + .expect("guardrail events should load"); + + assert_eq!(stop.reason, orchestrator::LoopGuardrailReason::ValidationRepeat); + assert_eq!(checkpoint.consecutive_count(), 3); + assert!(guardrail_events.iter().any(|event| { + event.event_type() == "loop_guardrail_checkpoint" + && event.payload()["reason"] == "validation_repeat" + && event.payload()["consecutive_count"] == 3 + })); +} + +fn loop_scenario_decodex_workflow_policy() -> ExecutionWorkflowPolicy { + ExecutionWorkflowPolicy::new( + "decodex", + vec![String::from("Todo")], + vec![String::from("Done"), String::from("Canceled")], + "decodex:manual-only", + "decodex:needs-attention", + ) + .expect("policy should build") +} + +fn loop_scenario_program_nodes() -> Vec { + vec![ + loop_scenario_node( + "node-ready", + ExecutionProgramNodeStage::Runtime, + "Implement the accepted runtime node", + ExecutionQueueIntent::ReadyToQueue, + "XY-861", + false, + ), + loop_scenario_node( + "node-blocked", + ExecutionProgramNodeStage::Eval, + "Validate after the runtime node completes", + ExecutionQueueIntent::ReadyToQueue, + "XY-862", + false, + ) + .with_dependencies(vec![ + crate::execution_program::ExecutionProgramDependency::new("node-ready") + .expect("dependency should build"), + ]) + .expect("dependency should attach"), + loop_scenario_node( + "node-uncovered-direction", + ExecutionProgramNodeStage::Research, + "Pause uncovered direction for contract feedback", + ExecutionQueueIntent::Paused, + "XY-863", + true, + ), + loop_scenario_active_node(), + ] +} + +fn loop_scenario_active_node() -> crate::execution_program::ExecutionProgramNode { + loop_scenario_node( + "node-active", + ExecutionProgramNodeStage::Handoff, + "Retain handoff ownership without queueing", + ExecutionQueueIntent::Active, + "XY-864", + true, + ) + .with_linear_issue( + ExecutionLinearIssueMapping::new("linear-node-active", "XY-864", "Todo") + .expect("issue mapping should build") + .with_queue_label(true) + .with_active_label(true), + ) + .expect("active issue should attach") +} + +fn loop_scenario_review_payload( + nonclean_rounds: i64, + private_marker: &'static str, +) -> Value { + serde_json::json!({ + "phase": "handoff", + "status": "findings", + "head_sha": "0123456789abcdef0123456789abcdef01234567", + "nonclean_rounds": nonclean_rounds, + "review": { + "accepted_findings": [{ + "summary": "Ready node lacks a scenario fixture", + "evidence": [private_marker] + }], + "rejected_findings": [{ + "summary": "Stale comment did not apply to current head" + }] + } + }) +} + +fn loop_scenario_uncovered_payload() -> Value { + serde_json::json!({ + "reason": "uncovered_direction", + "fingerprint": "contract-gap:operator-feedback", + "paused_node_id": "node-uncovered-direction", + "feedback_target": "decision_contract", + "other_ready_node_ids": ["node-ready"], + }) +} + +fn loop_scenario_repo_gate_failure() -> Report { + Report::new(RepoGateFailure::new( + RepoGateFailureKind::VerifyCommandFailed, + String::from("Repo verify command `cargo make test` failed: same assertion failed"), + )) +} + +fn loop_scenario_assert_harness_candidates( + payload: &Value, + private_review_marker: &str, +) { + let candidates = payload["improvement_candidates"] + .as_array() + .expect("improvement candidates should be an array"); + let candidate_json = + serde_json::to_string(candidates).expect("candidate summaries should serialize"); + + assert!( + candidates.iter().any(|candidate| { + candidate["kind"] == "underspecified_decision_contract" + && candidate["reason_code"] == "uncovered_direction" + }), + "uncovered direction should recommend a contract improvement" + ); + assert!( + candidates.iter().any(|candidate| { + candidate["kind"] == "weak_prompt" + && candidate["reason_code"] == "accepted_review_findings" + }), + "accepted review findings should recommend a prompt or fixture improvement" + ); + assert!( + !candidate_json.contains(private_review_marker), + "harness recommendations must summarize private events without leaking raw payloads" + ); +} + +fn loop_scenario_research_x_contract() -> DecisionContract { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/decision_contract/research_x_latent_contract.json" + ))) + .expect("research X fixture should deserialize") +} + +fn loop_scenario_node( + node_id: &str, + stage: ExecutionProgramNodeStage, + objective: &str, + queue_intent: ExecutionQueueIntent, + issue_identifier: &str, + has_queue_label: bool, +) -> crate::execution_program::ExecutionProgramNode { + crate::execution_program::ExecutionProgramNode::new(node_id, stage, objective, queue_intent) + .expect("node should build") + .with_conflict_domains(vec![ + crate::execution_program::ExecutionConflictDomain::new( + crate::execution_program::ExecutionConflictDomainKind::Module, + format!("runtime/{node_id}"), + ) + .expect("conflict domain should build"), + ]) + .expect("conflict domain should attach") + .with_acceptance_expectations(vec![format!( + "{issue_identifier} satisfies the promoted contract", + )]) + .expect("acceptance expectations should attach") + .with_validation_expectations(vec![format!( + "{issue_identifier} has deterministic validation", + )]) + .expect("validation expectations should attach") + .with_linear_issue( + ExecutionLinearIssueMapping::new( + format!("linear-{node_id}"), + issue_identifier, + "Todo", + ) + .expect("issue mapping should build") + .with_queue_label(has_queue_label), + ) + .expect("issue mapping should attach") +} + +fn loop_scenario_queue_action( + evaluation: &ExecutionProgramEvaluation, + node_id: &str, +) -> Option { + evaluation + .nodes() + .iter() + .find(|node| node.node_id() == node_id) + .unwrap_or_else(|| panic!("missing node evaluation `{node_id}`")) + .queue_label_action() +} +} diff --git a/docs/reference/test-suite.md b/docs/reference/test-suite.md index f1f5a765d..eab5d441c 100644 --- a/docs/reference/test-suite.md +++ b/docs/reference/test-suite.md @@ -14,9 +14,9 @@ standards for keeping, merging, or deleting tests. ## Current Snapshot -This snapshot lists 971 default-runnable `nextest` tests. One additional ignored +This snapshot lists 1024 default-runnable `nextest` tests. One additional ignored live app-server test is listed only with verbose or JSON inventory output. The repo -gate run for this inventory reported 971 passed tests and 1 skipped test. Regenerate +gate run for this inventory reported 1024 passed tests and 1 skipped test. Regenerate the runnable inventory with: ```sh @@ -45,14 +45,15 @@ cargo nextest list --workspace --all-targets --all-features 2>/dev/null \ | Group | Count | Primary surfaces | Owns | | --- | ---: | --- | --- | -| Orchestrator | 460 | `apps/decodex/src/orchestrator/tests.rs`, `apps/decodex/src/orchestrator/tests/**/*.rs` | Intake, retry, review/landing, runtime cleanup, operator status, repo gates | -| Tracker tool bridge | 91 | `apps/decodex/src/agent/tracker_tool_bridge/tests.rs`, `apps/decodex/src/agent/tracker_tool_bridge/tests/**/*.rs` | Dynamic tracker tools, continuation guards, review handoff writes, closeout writes | -| App-server protocol/runtime | 80 | `apps/decodex/src/agent/app_server/tests.rs`, `apps/decodex/src/agent/json_rpc.rs`, app-server protocol tests | JSON-RPC parsing, turn execution, dynamic tools, thread config, transport failures | -| Runtime state, locks, and maintenance | 60 | `state::tests`, `runtime::tests`, `maintenance::tests` | Persistent local state, lock ownership, runtime database contracts, local retention | -| Workflow and config parsing | 45 | `workflow::tests`, `config::tests`, `codex_config::tests` | `WORKFLOW.md`, project config, Codex config edits, removed-field rejection, default policy | +| Orchestrator | 472 | `apps/decodex/src/orchestrator/tests.rs`, `apps/decodex/src/orchestrator/tests/**/*.rs` | Intake, retry, review/landing, runtime cleanup, operator status, repo gates | +| Tracker tool bridge | 96 | `apps/decodex/src/agent/tracker_tool_bridge/tests.rs`, `apps/decodex/src/agent/tracker_tool_bridge/tests/**/*.rs` | Dynamic tracker tools, continuation guards, review handoff writes, closeout writes | +| App-server protocol/runtime | 86 | `apps/decodex/src/agent/app_server/tests.rs`, `apps/decodex/src/agent/json_rpc.rs`, app-server protocol tests | JSON-RPC parsing, turn execution, dynamic tools, thread config, transport failures | +| Runtime state, locks, and maintenance | 66 | `state::tests`, `runtime::tests`, `maintenance::tests` | Persistent local state, lock ownership, runtime database contracts, local retention | +| Workflow and config parsing | 46 | `workflow::tests`, `config::tests`, `codex_config::tests` | `WORKFLOW.md`, project config, Codex config edits, removed-field rejection, default policy | | Git, worktree, landing, and recovery helpers | 120 | `worktree::tests`, `manual::tests`, `commit_message::tests`, `github::tests`, `default_branch_sync::tests`, `pull_request::tests`, `recovery::tests`, `git_credentials::tests` | Git/worktree behavior, manual landing, GitHub/PR helpers, recovery commands, commit-message policy | | Radar content validation | 33 | `radar::tests` | Upstream Radar schemas, bundles, signal rendering, social publication ledgers | -| Account, CLI, archive, and tracker integration | 82 | `accounts::tests`, `agent::codex_accounts::tests`, `agent::decodex_tool_bridge::tests`, `app_bridge::tests`, `cli::tests`, `archive_hygiene::tests`, `tracker::*::tests` | User-facing commands, account pools, app bridge parsing, archive hygiene, direct tracker adapter and public-text behavior | +| Account, CLI, archive, and tracker integration | 84 | `accounts::tests`, `agent::codex_accounts::tests`, `agent::decodex_tool_bridge::tests`, `app_bridge::tests`, `cli::tests`, `archive_hygiene::tests`, `tracker::*::tests` | User-facing commands, account pools, app bridge parsing, archive hygiene, direct tracker adapter and public-text behavior | +| Loop contract and research surfaces | 21 | `loop_contract::tests`, `execution_program::tests`, `research_design::tests`, `plugin_surface_tests` | Decision Contract schema, Execution Program readiness, research compile/promote, and plugin trigger behavior | ## Orchestrator Inventory @@ -68,8 +69,9 @@ large catch-all test file unless the behavior crosses several of these stages. | `apps/decodex/src/orchestrator/tests/intake/candidate_selection.rs` | 22 | Candidate ordering, retained lane preference, closeout dispatch policy | | `apps/decodex/src/orchestrator/tests/retry/scheduling.rs` | 24 | Retry timing, dry-run behavior, retry marker semantics | | `apps/decodex/src/orchestrator/tests/retry/selection.rs` | 14 | Retry queue selection and blocked retry candidates | -| `apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs` | 8 | Repo gate command selection, cleanliness, shell fallback, and failure classification | -| `apps/decodex/src/orchestrator/tests/runtime/failure.rs` | 33 | Failure comments, runtime credentials, cleanup, lease release | +| `apps/decodex/src/orchestrator/tests/runtime/repo_gate.rs` | 9 | Repo gate command selection, cleanliness, shell fallback, and failure classification | +| `apps/decodex/src/orchestrator/tests/runtime/failure.rs` | 38 | Failure comments, runtime credentials, cleanup, lease release | +| `apps/decodex/src/orchestrator/tests/runtime/loop_scenarios.rs` | 2 | Research-to-execution loop scenarios, queue shaping, phase-goal validation, review, guardrails, and harness feedback | | `apps/decodex/src/orchestrator/tests/runtime/thread_archive.rs` | 1 | Completed-thread archive candidate filtering | | `apps/decodex/src/orchestrator/tests/recovery/reconciliation.rs` | 22 | Stale lease, recovery worktree, and reconciliation behavior | | `apps/decodex/src/orchestrator/tests/recovery/terminal_support.rs` | 0 | Shared retained recovery and closeout fixtures | @@ -82,12 +84,12 @@ large catch-all test file unless the behavior crosses several of these stages. | `apps/decodex/src/orchestrator/tests/operator/status/control_plane.rs` | 10 | Registered project control-plane rows | | `apps/decodex/src/orchestrator/tests/operator/status/running_lanes.rs` | 34 | Running lanes, stalled lanes, active-run hydration, and local worktrees | | `apps/decodex/src/orchestrator/tests/operator/status/history.rs` | 7 | Run ledger and Linear history hydration | -| `apps/decodex/src/orchestrator/tests/operator/status/text.rs` | 9 | Human-readable operator status text | +| `apps/decodex/src/orchestrator/tests/operator/status/text.rs` | 10 | Human-readable operator status text | | `apps/decodex/src/orchestrator/tests/operator/status/publishing.rs` | 11 | Snapshot publishing, degraded observers, and tracker backoff | -| `apps/decodex/src/orchestrator/tests/operator/status/queue.rs` | 17 | Intake queue classifications and shared-claim visibility | +| `apps/decodex/src/orchestrator/tests/operator/status/queue.rs` | 18 | Intake queue classifications and shared-claim visibility | | `apps/decodex/src/orchestrator/tests/operator/status/http.rs` | 31 | Operator dashboard HTTP pages/assets, `/livez`, WebSocket control, and removed snapshot-route responses | | `apps/decodex/src/orchestrator/tests/operator/status/dashboard.rs` | 34 | Dashboard client rendering contracts | -| `apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs` | 4 | Agent evidence snapshots and private evidence readback | +| `apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs` | 6 | Agent evidence snapshots and private evidence readback | | `apps/decodex/src/orchestrator/tests/review_landing/status_support.rs` | 0 | Shared Review & Landing status fixtures | | `apps/decodex/src/orchestrator/tests/review_landing/status_rows.rs` | 18 | Review & Landing status rows and handoff lineage | | `apps/decodex/src/orchestrator/tests/review_landing/orchestration.rs` | 17 | Review orchestration, admin merge, and repair routing | diff --git a/docs/runbook/index.md b/docs/runbook/index.md index ee01a0888..f2b6d93ea 100644 --- a/docs/runbook/index.md +++ b/docs/runbook/index.md @@ -46,5 +46,8 @@ Question this index answers: "which sequence should I execute?" explicitly rebinding retained review lanes blocked by a missing or stale runtime DB handoff marker. +- [`research-to-execution-loop.md`](./research-to-execution-loop.md) for compiling + latent research contracts, promoting accepted results, inspecting Execution Program + queue shaping, and following validation, review, guardrail, and harness feedback. - [`self-dogfood-pilot.md`](./self-dogfood-pilot.md) for the retained-lane pilot run against `decodex` itself and the bounded live-operation sequence. diff --git a/docs/runbook/research-to-execution-loop.md b/docs/runbook/research-to-execution-loop.md new file mode 100644 index 000000000..208209aeb --- /dev/null +++ b/docs/runbook/research-to-execution-loop.md @@ -0,0 +1,100 @@ +# Research-To-Execution Loop + +Purpose: Operate and inspect the natural-language research-to-execution loop without +turning latent research into automatic execution. + +Read this when: A user asks Decodex to research something, promotes the result, or +needs to inspect why only part of the resulting work queued. + +Not this document: The normative schema or runtime invariants. Use +[`../spec/loop-runtime.md`](../spec/loop-runtime.md) for the authority contract. + +Covers: Latent research contracts, promotion, Execution Program queue shaping, +phase-goal validation, independent review, guardrails, and harness improvement +evidence. + +## Preconditions + +- The project is registered and has a service id. +- The operator can inspect private local Decodex evidence for the project. +- Promotion is a human or runtime-policy decision; latent research output is not + execution authority. + +## Operator Path + +1. Compile the research result into a latent contract. + + ```sh + decodex research compile --intent "research X" + ``` + + Inspect the contract id and status. The expected status is `draft_latent`. + This step must not enqueue Linear issues, mutate trackers, set phase goals, or + authorize implementation. + +2. Promote only an accepted result. + + ```sh + decodex research promote + ``` + + Promotion records the accepted Decision Contract. It authorizes the runtime to + shape an internal Execution Program, but it is still not a request to queue every + possible node. + +3. Inspect Execution Program readiness. + + ```sh + decodex status --json + ``` + + Check the execution-program summary for ready, blocked, paused, active, and + completed counts. Only ready nodes mapped to startable issues, without active, + opt-out, needs-attention, terminal-state, dependency, or conflict blockers may + receive or retain `decodex:queued:decodex`. + +4. Treat phase-goal completion as a validation boundary. + + A child goal reaching `complete` means Decodex should run the registered repo gate + and move to validation, review, repair, handoff evidence, or manual attention. It + does not mean the issue is terminally complete. + +5. Follow review and guardrail outcomes. + + Accepted independent-review findings route to repair. Three non-clean review + rounds, `needs_architecture_review`, or `blocked` escalates instead of continuing + patch churn. Repeated validation failures stop after the configured threshold. + +6. Preserve uncovered direction as research feedback. + + If execution discovers direction not covered by the accepted contract, pause the + affected branch, record Decision Contract feedback, and allow unrelated ready + nodes to continue when their dependencies and conflict domains permit it. + +7. Inspect private harness feedback locally. + + ```sh + decodex evidence --run-id --attempt --json + ``` + + Review improvement candidates in the summarized evidence. Use raw payload + readback only for local debugging, and do not paste private execution payloads into + public tracker comments or PR descriptions. + +## Dogfood Assessment + +The loop is meaningfully better than the old external research to manual Linear issue +flow when the lane evidence shows all of the following: + +- research output stays latent until promotion; +- promotion creates an internal source of truth for execution shape; +- only ready nodes get the service queue label; +- child-goal completion triggers validation and review instead of terminal success; +- repair churn has bounded guardrails; +- harness telemetry produces at least one concrete fixture, prompt, or contract + improvement recommendation. + +Remaining gaps should be recorded with the lane evidence. Common gaps are missing +native issue-generation ergonomics, too little scenario coverage for a new branch of +the architecture, or operator-owned promotion decisions that still require manual +judgment.