diff --git a/apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json b/apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json new file mode 100644 index 000000000..ff043ff91 --- /dev/null +++ b/apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json @@ -0,0 +1,93 @@ +{ + "schema": "decodex.harness_improvement_eval/1", + "record_version": 1, + "project_id": "decodex", + "issue_id": "issue-857-fixture", + "issue_identifier": "XY-857-EVAL", + "run_id": "xy-857-eval-run", + "attempt_number": 1, + "decision_contract": { + "schema": "decodex.decision_contract/1", + "record_version": 1, + "contract_id": "incomplete-contract-eval", + "status": "needs_human_decision", + "source_intent": { + "summary": "Arrange a research result whose generated issue brief omitted the contract authority boundary.", + "user_utterance": "push this forward", + "source_issue_identifier": "XY-857-EVAL" + }, + "research_provenance": [ + { + "kind": "fixture", + "reference": "apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json", + "summary": "Fixture intentionally leaves execution authority underspecified." + } + ], + "research_evidence": [ + { + "claim": "Generated work can be unsafe when the issue brief omits the accepted contract id.", + "support": "The lane stopped with uncovered direction instead of guessing the missing authority boundary.", + "source_ref": "fixture" + } + ], + "accepted_authority": { + "accepted_objectives": [], + "non_goals": [ + "Do not queue generated issues when promotion authority is missing." + ], + "constraints": [ + "Keep private research evidence in runtime SQLite." + ], + "assumptions": [], + "objections": [ + "The generated issue lacks a contract id and conflict-domain field." + ], + "stop_conditions": [ + "Stop for human decision when generated issue authority is missing." + ] + }, + "execution_readiness": { + "summary": "Not ready for issue shaping because the generated issue template omitted the accepted contract boundary.", + "ready_for_issue_shaping": false, + "missing_decisions": [ + "Decide how generated issues must cite accepted Decision Contract provenance before queueing." + ], + "validation_expectations": [ + "A harness eval should emit an underspecified_decision_contract recommendation." + ], + "risk_notes": [ + "A future agent could implement from latent evidence if the issue template omits the authority field." + ], + "proposed_issue_summaries": [], + "conflict_domains": [], + "queue_intent": [] + }, + "links": { + "generated_issue_ids": [], + "generated_issue_identifiers": [], + "execution_program_node_ids": [] + }, + "evidence_boundary": { + "private_evidence_refs": [], + "public_projection_refs": [], + "public_summary": "Incomplete contract fixture for harness-improvement evaluation." + } + }, + "private_events": [ + { + "event_type": "loop_guardrail_checkpoint", + "payload": { + "schema": "decodex.loop_guardrail_checkpoint/1", + "reason": "uncovered_direction", + "consecutive_count": 3, + "threshold": 3, + "source_error_class": "uncovered_direction" + } + } + ], + "expected_candidate": { + "kind": "underspecified_decision_contract", + "reason_code": "missing_decisions", + "target": "decision_contract:incomplete-contract-eval" + } +} diff --git a/apps/decodex/src/execution_program.rs b/apps/decodex/src/execution_program.rs index f51df7c23..d5b8b3caf 100644 --- a/apps/decodex/src/execution_program.rs +++ b/apps/decodex/src/execution_program.rs @@ -38,6 +38,21 @@ pub(crate) enum ExecutionProgramNodeStage { /// Review, PR, delivery, or handoff work. Handoff, } +impl ExecutionProgramNodeStage { + /// Stable machine-readable stage name. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Research => "research", + Self::Design => "design", + Self::Spec => "spec", + Self::Schema => "schema", + Self::Runtime => "runtime", + Self::Plugin => "plugin", + Self::Eval => "eval", + Self::Handoff => "handoff", + } + } +} /// Queue intent for one internal Execution Program node. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)] @@ -59,6 +74,19 @@ pub(crate) enum ExecutionQueueIntent { Canceled, } impl ExecutionQueueIntent { + /// Stable machine-readable queue-intent name. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::NotReady => "not_ready", + Self::ReadyToQueue => "ready_to_queue", + Self::Queued => "queued", + Self::Active => "active", + Self::Paused => "paused", + Self::Done => "done", + Self::Canceled => "canceled", + } + } + fn is_terminal(self) -> bool { matches!(self, Self::Done | Self::Canceled) } @@ -82,7 +110,8 @@ pub(crate) enum ExecutionConflictDomainKind { ReviewSurface, } impl ExecutionConflictDomainKind { - fn as_str(self) -> &'static str { + /// Stable machine-readable conflict-domain class name. + pub(crate) fn as_str(self) -> &'static str { match self { Self::File => "file", Self::Module => "module", @@ -159,6 +188,11 @@ impl ExecutionConflictDomain { &self.key } + /// Stable conflict-domain kind. + pub(crate) fn kind(&self) -> ExecutionConflictDomainKind { + self.kind + } + fn validate(&self) -> Result<()> { validate_required("execution program conflict_domain.key", &self.key) } @@ -284,6 +318,11 @@ impl ExecutionLinearIssueMapping { &self.issue_identifier } + /// Linear issue id used by tracker APIs. + pub(crate) fn issue_id(&self) -> &str { + &self.issue_id + } + /// Tracker workflow state for the mapped issue. pub(crate) fn issue_state(&self) -> &str { &self.issue_state @@ -439,11 +478,21 @@ impl ExecutionProgramNode { &self.node_id } + /// Node execution stage. + pub(crate) fn stage(&self) -> ExecutionProgramNodeStage { + self.stage + } + /// Node queue intent. pub(crate) fn queue_intent(&self) -> ExecutionQueueIntent { self.queue_intent } + /// Conflict domains occupied by this node. + pub(crate) fn conflict_domains(&self) -> &[ExecutionConflictDomain] { + &self.conflict_domains + } + /// Linked normal Linear issue, when the node is executable. pub(crate) fn linear_issue(&self) -> Option<&ExecutionLinearIssueMapping> { self.linear_issue.as_ref() diff --git a/apps/decodex/src/loop_contract.rs b/apps/decodex/src/loop_contract.rs index cc41db38e..6da14e9e1 100644 --- a/apps/decodex/src/loop_contract.rs +++ b/apps/decodex/src/loop_contract.rs @@ -535,6 +535,10 @@ pub(crate) struct DecisionContractLinks { } #[allow(dead_code)] impl DecisionContractLinks { + pub(crate) fn generated_issue_ids(&self) -> &[String] { + &self.generated_issue_ids + } + pub(crate) fn generated_issue_identifiers(&self) -> &[String] { &self.generated_issue_identifiers } diff --git a/apps/decodex/src/orchestrator.rs b/apps/decodex/src/orchestrator.rs index 9691e8418..53db7a7fb 100644 --- a/apps/decodex/src/orchestrator.rs +++ b/apps/decodex/src/orchestrator.rs @@ -1,4 +1,9 @@ mod lane_control; +mod harness_improvement { + use crate::orchestrator::{IssueRunPlan, Result, Serialize, StateStore, Value, records, state}; + + include!("orchestrator/harness_improvement.rs"); +} pub(crate) use lane_control::{ DEFAULT_STEER_RESULT_WAIT_TIMEOUT, LaneInspectRequest, LaneInterruptRequest, interrupt_lane, @@ -36,6 +41,12 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::{agent, default_branch_sync, git_credentials, maintenance, state}; #[rustfmt::skip] use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{InternalReviewMode, ServiceConfig}, execution_program::{ExecutionProgramOperatorSummary, ExecutionProgramReadinessContext, ExecutionWorkflowPolicy}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ExecutionProgramRecord, LoopGuardrailCheckpoint, LoopGuardrailCheckpointInput, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}}; +use harness_improvement::{ + HarnessImprovementCandidateSummary, HarnessOutcomeKind, + harness_improvement_candidates_from_private_events, record_harness_outcome_best_effort, +}; +#[cfg(test)] +use harness_improvement::{HarnessOutcomeRecordInput, record_harness_outcome_for_issue_run}; include!("orchestrator/types.rs"); diff --git a/apps/decodex/src/orchestrator/agent_evidence.rs b/apps/decodex/src/orchestrator/agent_evidence.rs index d1554dad0..11fa04ec0 100644 --- a/apps/decodex/src/orchestrator/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/agent_evidence.rs @@ -1,7 +1,5 @@ use std::collections::{self, BTreeMap}; -use state::PrivateExecutionEvent; - const AGENT_HANDOFF_INDEX_SCHEMA: &str = "decodex.agent_handoff_index/1"; const AGENT_BLOCKER_SNAPSHOT_SCHEMA: &str = "decodex.blocker_snapshot/1"; const AGENT_RUN_CAPSULE_SCHEMA: &str = "decodex.run_capsule/1"; @@ -261,6 +259,7 @@ struct PrivateEvidenceReadback { event_count: usize, latest_event_type: Option, latest_event_at: Option, + improvement_candidates: Vec, events: Vec, warnings: Vec, } @@ -648,6 +647,7 @@ fn build_private_evidence_readback( event_count: events.len(), latest_event_type: latest_event.map(|event| event.event_type().to_owned()), latest_event_at: latest_event.map(|event| event.recorded_at().to_owned()), + improvement_candidates: harness_improvement_candidates_from_private_events(&events), events: events .iter() .map(|event| private_evidence_readback_event(event, request.include_payload)) @@ -719,12 +719,12 @@ fn resolve_private_evidence_target( } fn private_evidence_direct_lookup_issue_id( - events: &[PrivateExecutionEvent], + events: &[state::PrivateExecutionEvent], selector: &str, ) -> Result> { let issue_ids = events .iter() - .map(PrivateExecutionEvent::issue_id) + .map(state::PrivateExecutionEvent::issue_id) .collect::>(); if issue_ids.is_empty() { @@ -767,7 +767,7 @@ fn private_evidence_run_matches_issue( } fn private_evidence_readback_event( - event: &PrivateExecutionEvent, + event: &state::PrivateExecutionEvent, include_payload: bool, ) -> PrivateEvidenceReadbackEvent { PrivateEvidenceReadbackEvent { @@ -894,6 +894,10 @@ fn render_private_evidence_readback(readback: &PrivateEvidenceReadback) -> Strin output.push_str(&format!("evidence_ref: {}\n", readback.evidence_ref)); output.push_str(&format!("payload_mode: {}\n", readback.payload_mode)); output.push_str(&format!("event_count: {}\n", readback.event_count)); + output.push_str(&format!( + "improvement_candidate_count: {}\n", + readback.improvement_candidates.len() + )); output.push_str(&format!( "latest_event_type: {}\n", readback.latest_event_type.as_deref().unwrap_or("none") @@ -907,6 +911,23 @@ fn render_private_evidence_readback(readback: &PrivateEvidenceReadback) -> Strin output.push_str(&format!("warnings: {}\n", readback.warnings.join(", "))); } + output.push_str("\nImprovement Candidates\n"); + + if readback.improvement_candidates.is_empty() { + output.push_str("- none\n"); + } else { + for candidate in &readback.improvement_candidates { + output.push_str(&format!( + "- kind: {}\n reason_code: {}\n target: {}\n source_event_count: {}\n recommendation: {}\n", + candidate.kind, + candidate.reason_code, + candidate.target, + candidate.source_event_count, + candidate.recommendation + )); + } + } + output.push_str("\nEvents\n"); if readback.events.is_empty() { diff --git a/apps/decodex/src/orchestrator/execution.rs b/apps/decodex/src/orchestrator/execution.rs index b2a057661..4fc7dbfe3 100644 --- a/apps/decodex/src/orchestrator/execution.rs +++ b/apps/decodex/src/orchestrator/execution.rs @@ -1319,6 +1319,31 @@ where Ok(()) } +fn run_completion_repo_gate(workflow: &WorkflowDocument, issue_run: &IssueRunPlan) -> Result<()> { + let selected_repo_gate = + select_repo_gate_for_worktree(workflow.frontmatter().execution(), &issue_run.worktree.path); + + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REPO_GATE, + ); + run_repo_gate_commands( + selected_repo_gate.canonicalize_commands(), + selected_repo_gate.verify_commands(), + &issue_run.worktree.path, + )?; + write_run_operation_marker_best_effort( + &issue_run.worktree.path, + &issue_run.run_id, + issue_run.attempt_number, + RUN_OPERATION_REVIEW_WRITEBACK, + ); + + Ok(()) +} + fn apply_run_completion_disposition( tracker: &T, project: &ServiceConfig, @@ -1333,27 +1358,7 @@ where match tracker_tool_bridge.completion_disposition()? { RunCompletionDisposition::ReviewHandoff => { validate_review_handoff_runtime(project, false)?; - - let selected_repo_gate = - select_repo_gate_for_worktree(workflow.frontmatter().execution(), &issue_run.worktree.path); - - write_run_operation_marker_best_effort( - &issue_run.worktree.path, - &issue_run.run_id, - issue_run.attempt_number, - RUN_OPERATION_REPO_GATE, - ); - run_repo_gate_commands( - selected_repo_gate.canonicalize_commands(), - selected_repo_gate.verify_commands(), - &issue_run.worktree.path, - )?; - write_run_operation_marker_best_effort( - &issue_run.worktree.path, - &issue_run.run_id, - issue_run.attempt_number, - RUN_OPERATION_REVIEW_WRITEBACK, - ); + run_completion_repo_gate(workflow, issue_run)?; tracker_tool_bridge.apply_review_handoff().map_err(|error| { if let Some(writeback_error) = error.downcast_ref::() @@ -1368,6 +1373,18 @@ where error } })?; + + record_harness_outcome_best_effort( + state_store, + project.service_id(), + issue_run, + HarnessOutcomeKind::ReviewHandoff, + None, + Some("passed"), + tracker_tool_bridge + .review_context() + .and_then(|context| context.recorded_pr_url.as_deref()), + ); }, RunCompletionDisposition::ManualAttention => { return Err(Report::new(ManualAttentionRequested { @@ -1378,28 +1395,21 @@ where })); }, RunCompletionDisposition::ReviewRepair => { - let selected_repo_gate = - select_repo_gate_for_worktree(workflow.frontmatter().execution(), &issue_run.worktree.path); - - write_run_operation_marker_best_effort( - &issue_run.worktree.path, - &issue_run.run_id, - issue_run.attempt_number, - RUN_OPERATION_REPO_GATE, - ); - run_repo_gate_commands( - selected_repo_gate.canonicalize_commands(), - selected_repo_gate.verify_commands(), - &issue_run.worktree.path, - )?; - write_run_operation_marker_best_effort( - &issue_run.worktree.path, - &issue_run.run_id, - issue_run.attempt_number, - RUN_OPERATION_REVIEW_WRITEBACK, - ); + run_completion_repo_gate(workflow, issue_run)?; tracker_tool_bridge.apply_review_repair()?; + + record_harness_outcome_best_effort( + state_store, + project.service_id(), + issue_run, + HarnessOutcomeKind::ReviewRepair, + None, + Some("passed"), + tracker_tool_bridge + .review_context() + .and_then(|context| context.recorded_pr_url.as_deref()), + ); }, RunCompletionDisposition::Closeout => { write_run_operation_marker_best_effort( @@ -1426,6 +1436,18 @@ where )?; tracker_tool_bridge.clear_closeout_issue_scope()?; + + record_harness_outcome_best_effort( + state_store, + project.service_id(), + issue_run, + HarnessOutcomeKind::Closeout, + None, + Some("passed"), + tracker_tool_bridge + .review_context() + .and_then(|context| context.recorded_pr_url.as_deref()), + ); }, } @@ -2076,6 +2098,15 @@ where context.issue_run.attempt_number, context.retry_budget_attempts, )?; + record_harness_outcome_best_effort( + context.state_store, + context.project.service_id(), + context.issue_run, + HarnessOutcomeKind::RetryableFailure, + Some(retry_error_class), + Some("failed"), + None, + ); Ok(()) } @@ -2283,6 +2314,23 @@ where return Err(error); } + if let Some(state_store) = runtime.state_store { + let outcome = if writeback.projection.record.event_type == "needs_attention" { + HarnessOutcomeKind::ManualAttention + } else { + HarnessOutcomeKind::TerminalFailure + }; + + record_harness_outcome_best_effort( + state_store, + runtime.service_id, + issue_run, + outcome, + Some(writeback.error_class), + None, + writeback.projection.record.pr_url.as_deref(), + ); + } Ok(terminal_failure_outcome(&writeback)) } diff --git a/apps/decodex/src/orchestrator/harness_improvement.rs b/apps/decodex/src/orchestrator/harness_improvement.rs new file mode 100644 index 000000000..bdc973691 --- /dev/null +++ b/apps/decodex/src/orchestrator/harness_improvement.rs @@ -0,0 +1,816 @@ +use records::LinearExecutionEventRecord; +use state::{DecisionContractRecord, ExecutionProgramRecord, PrivateExecutionEvent}; + +use crate::execution_program::ExecutionConflictDomain; + +const HARNESS_OUTCOME_SCHEMA: &str = "decodex.harness_outcome/1"; +const HARNESS_OUTCOME_EVENT_TYPE: &str = "harness_outcome"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum HarnessOutcomeKind { + ReviewHandoff, + ReviewRepair, + Closeout, + RetryableFailure, + TerminalFailure, + ManualAttention, +} +impl HarnessOutcomeKind { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::ReviewHandoff => "review_handoff", + Self::ReviewRepair => "review_repair", + Self::Closeout => "closeout", + Self::RetryableFailure => "retryable_failure", + Self::TerminalFailure => "terminal_failure", + Self::ManualAttention => "manual_attention", + } + } + + fn validation_result(self, explicit: Option<&str>, signals: &HarnessOutcomeSignals) -> String { + if let Some(result) = explicit { + return result.to_owned(); + } + + if signals.validation_failure_count > 0 || matches!(self, Self::RetryableFailure) { + return String::from("failed"); + } + if matches!(self, Self::ReviewHandoff | Self::ReviewRepair | Self::Closeout) { + return String::from("passed"); + } + + String::from("not_recorded") + } +} + +pub(crate) struct HarnessOutcomeRecordInput<'a> { + pub(crate) project_id: &'a str, + pub(crate) issue_id: &'a str, + pub(crate) issue_identifier: &'a str, + pub(crate) run_id: &'a str, + pub(crate) attempt_number: i64, + pub(crate) outcome: HarnessOutcomeKind, + pub(crate) error_class: Option<&'a str>, + pub(crate) validation_result: Option<&'a str>, + pub(crate) pr_url: Option<&'a str>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct HarnessImprovementCandidateSummary { + pub(crate) kind: String, + pub(crate) reason_code: String, + pub(crate) target: String, + pub(crate) source_event_count: usize, + pub(crate) recommendation: String, +} + +#[derive(Serialize)] +struct HarnessOutcomePayload { + schema: &'static str, + record_version: u16, + source: HarnessOutcomeSource, + contracts: Vec, + execution_programs: Vec, + phase_goal_outcomes: Vec, + validation: HarnessValidationOutcome, + repair: HarnessRepairOutcome, + review: HarnessReviewOutcome, + manual_attention: Option, + pr_lifecycle: HarnessPrLifecycleOutcome, + linear_projection: HarnessLinearProjectionSummary, + improvement_candidates: Vec, +} + +#[derive(Serialize)] +struct HarnessOutcomeSource { + project_id: String, + issue_id: String, + issue_identifier: String, + run_id: String, + attempt_number: i64, + outcome: String, + source_intents: Vec, +} + +#[derive(Serialize)] +struct HarnessSourceIntent { + contract_id: String, + status: String, + summary: String, + source_issue_identifier: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct HarnessOutcomeContract { + contract_id: String, + status: String, + source_issue_id: Option, + ready_for_issue_shaping: bool, + missing_decision_count: usize, + generated_issue_ids: Vec, + generated_issue_identifiers: Vec, + execution_program_node_ids: Vec, + conflict_domains: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct HarnessOutcomeProgram { + program_id: String, + source_contract_id: String, + node_count: usize, + nodes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct HarnessOutcomeProgramNode { + node_id: String, + stage: String, + queue_intent: String, + linear_issue_id: Option, + linear_issue_identifier: Option, + conflict_domains: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct HarnessPhaseGoalOutcome { + event_type: String, + phase: Option, + signal: Option, + status: Option, +} + +#[derive(Serialize)] +struct HarnessValidationOutcome { + result: String, + failure_count: usize, + failure_classes: Vec, +} + +#[derive(Serialize)] +struct HarnessRepairOutcome { + attempt_number: i64, + repair_attempt_observed: bool, + repair_phase_events: usize, +} + +#[derive(Serialize)] +struct HarnessReviewOutcome { + statuses: Vec, + accepted_finding_count: usize, + rejected_finding_count: usize, + nonclean_rounds: i64, +} + +#[derive(Serialize)] +struct HarnessManualAttentionOutcome { + reason_code: String, +} + +#[derive(Serialize)] +struct HarnessPrLifecycleOutcome { + outcome: String, + pr_urls: Vec, +} + +#[derive(Serialize)] +struct HarnessLinearProjectionSummary { + event_types: Vec, + final_event_type: Option, + final_error_class: Option, + final_terminal_path: Option, +} + +#[derive(Default)] +struct HarnessOutcomeSignals { + phase_goals: Vec, + validation_failure_count: usize, + validation_failure_classes: std::collections::BTreeSet, + review_statuses: std::collections::BTreeSet, + accepted_finding_count: usize, + rejected_finding_count: usize, + nonclean_rounds: i64, + repair_phase_events: usize, + guardrail_reasons: std::collections::BTreeSet, +} + +pub(crate) fn record_harness_outcome_for_issue_run( + state_store: &StateStore, + input: HarnessOutcomeRecordInput<'_>, +) -> Result { + let events = state_store.list_private_execution_events( + input.project_id, + input.issue_id, + input.run_id, + input.attempt_number, + )?; + let contracts = harness_contracts_for_issue(state_store, &input)?; + let programs = harness_programs_for_contracts(state_store, input.project_id, &contracts)?; + let linear_records = state_store.list_linear_execution_events(input.project_id, input.issue_id)?; + let signals = harness_outcome_signals(&events, input.error_class); + let payload = harness_outcome_payload(&input, &contracts, &programs, &linear_records, &signals)?; + + state_store.append_private_execution_event( + input.project_id, + input.issue_id, + input.run_id, + input.attempt_number, + HARNESS_OUTCOME_EVENT_TYPE, + payload, + ) +} + +pub(crate) fn harness_improvement_candidates_from_private_events( + events: &[PrivateExecutionEvent], +) -> Vec { + let mut from_outcome = Vec::new(); + + for event in events.iter().filter(|event| event.event_type() == HARNESS_OUTCOME_EVENT_TYPE) { + from_outcome.extend(harness_candidates_from_payload(event.payload())); + } + + if !from_outcome.is_empty() { + return from_outcome; + } + if events.is_empty() { + return Vec::new(); + } + + let signals = harness_outcome_signals(events, None); + + if signals.validation_failure_count == 0 + && signals.accepted_finding_count == 0 + && signals.guardrail_reasons.is_empty() + { + return Vec::new(); + } + + let input = HarnessOutcomeRecordInput { + project_id: "", + issue_id: "", + issue_identifier: "local-readback", + run_id: "", + attempt_number: 0, + outcome: HarnessOutcomeKind::TerminalFailure, + error_class: None, + validation_result: None, + pr_url: None, + }; + let linear_projection = HarnessLinearProjectionSummary { + event_types: Vec::new(), + final_event_type: None, + final_error_class: None, + final_terminal_path: None, + }; + let mut candidates = std::collections::BTreeMap::new(); + + push_signal_candidates(&mut candidates, &input, &signals, &linear_projection); + + candidates.into_values().collect() +} + +pub(super) fn record_harness_outcome_best_effort( + state_store: &StateStore, + project_id: &str, + issue_run: &IssueRunPlan, + outcome: HarnessOutcomeKind, + error_class: Option<&str>, + validation_result: Option<&str>, + pr_url: Option<&str>, +) { + if let Err(error) = record_harness_outcome_for_issue_run( + state_store, + HarnessOutcomeRecordInput { + project_id, + issue_id: &issue_run.issue.id, + issue_identifier: &issue_run.issue.identifier, + run_id: &issue_run.run_id, + attempt_number: issue_run.attempt_number, + outcome, + error_class, + validation_result, + pr_url, + }, + ) { + tracing::warn!( + ?error, + project_id, + issue_id = issue_run.issue.id, + issue = issue_run.issue.identifier, + run_id = issue_run.run_id, + attempt = issue_run.attempt_number, + "Harness outcome telemetry write failed." + ); + } +} + +fn harness_contracts_for_issue( + state_store: &StateStore, + input: &HarnessOutcomeRecordInput<'_>, +) -> Result> { + let mut records = Vec::new(); + let mut seen = std::collections::BTreeSet::new(); + + for issue_id in [input.issue_id, input.issue_identifier] { + for record in state_store.list_decision_contracts_for_issue(input.project_id, issue_id)? { + let key = record.contract_id().to_owned(); + + if seen.insert(key) { + records.push(record); + } + } + } + + records.sort_by(|left, right| left.contract_id().cmp(right.contract_id())); + + Ok(records) +} + +fn harness_programs_for_contracts( + state_store: &StateStore, + project_id: &str, + contracts: &[DecisionContractRecord], +) -> Result> { + let mut programs = Vec::new(); + let mut seen = std::collections::BTreeSet::new(); + + for contract in contracts { + for program in state_store.list_execution_programs_for_contract( + project_id, + contract.contract_id(), + )? { + let key = program.program_id().to_owned(); + + if seen.insert(key) { + programs.push(program); + } + } + } + + programs.sort_by(|left, right| left.program_id().cmp(right.program_id())); + + Ok(programs) +} + +fn harness_outcome_payload( + input: &HarnessOutcomeRecordInput<'_>, + contracts: &[DecisionContractRecord], + programs: &[ExecutionProgramRecord], + linear_records: &[LinearExecutionEventRecord], + signals: &HarnessOutcomeSignals, +) -> Result { + let source_intents = contracts.iter().map(harness_source_intent).collect(); + let contracts = contracts.iter().map(harness_outcome_contract).collect::>(); + let programs = programs.iter().map(harness_outcome_program).collect::>(); + let linear_projection = harness_linear_projection(linear_records); + let validation = HarnessValidationOutcome { + result: input.outcome.validation_result(input.validation_result, signals), + failure_count: signals.validation_failure_count, + failure_classes: signals.validation_failure_classes.iter().cloned().collect(), + }; + let repair = HarnessRepairOutcome { + attempt_number: input.attempt_number, + repair_attempt_observed: input.attempt_number > 1 || signals.repair_phase_events > 0, + repair_phase_events: signals.repair_phase_events, + }; + let review = HarnessReviewOutcome { + statuses: signals.review_statuses.iter().cloned().collect(), + accepted_finding_count: signals.accepted_finding_count, + rejected_finding_count: signals.rejected_finding_count, + nonclean_rounds: signals.nonclean_rounds, + }; + let manual_attention = + input.error_class.filter(|_| input.outcome == HarnessOutcomeKind::ManualAttention).map( + |reason| HarnessManualAttentionOutcome { reason_code: reason.to_owned() }, + ); + let pr_lifecycle = HarnessPrLifecycleOutcome { + outcome: input.outcome.as_str().to_owned(), + pr_urls: harness_pr_urls(input.pr_url, linear_records), + }; + let improvement_candidates = harness_improvement_candidates( + input, + &contracts, + &programs, + signals, + &linear_projection, + ); + let payload = HarnessOutcomePayload { + schema: HARNESS_OUTCOME_SCHEMA, + record_version: 1, + source: HarnessOutcomeSource { + project_id: input.project_id.to_owned(), + issue_id: input.issue_id.to_owned(), + issue_identifier: input.issue_identifier.to_owned(), + run_id: input.run_id.to_owned(), + attempt_number: input.attempt_number, + outcome: input.outcome.as_str().to_owned(), + source_intents, + }, + contracts, + execution_programs: programs, + phase_goal_outcomes: signals.phase_goals.clone(), + validation, + repair, + review, + manual_attention, + pr_lifecycle, + linear_projection, + improvement_candidates, + }; + + serde_json::to_value(payload).map_err(Into::into) +} + +fn harness_source_intent(record: &DecisionContractRecord) -> HarnessSourceIntent { + let contract = record.contract(); + + HarnessSourceIntent { + contract_id: contract.contract_id().to_owned(), + status: record.status().as_str().to_owned(), + summary: contract.source_intent().summary().to_owned(), + source_issue_identifier: contract + .source_intent() + .source_issue_identifier() + .map(str::to_owned), + } +} + +fn harness_outcome_contract(record: &DecisionContractRecord) -> HarnessOutcomeContract { + let contract = record.contract(); + let readiness = contract.execution_readiness(); + let links = contract.links(); + + HarnessOutcomeContract { + contract_id: contract.contract_id().to_owned(), + status: record.status().as_str().to_owned(), + source_issue_id: record.source_issue_id().map(str::to_owned), + ready_for_issue_shaping: readiness.ready_for_issue_shaping(), + missing_decision_count: readiness.missing_decisions().len(), + generated_issue_ids: links.generated_issue_ids().to_vec(), + generated_issue_identifiers: links.generated_issue_identifiers().to_vec(), + execution_program_node_ids: links.execution_program_node_ids().to_vec(), + conflict_domains: readiness.conflict_domains().to_vec(), + } +} + +fn harness_outcome_program(record: &ExecutionProgramRecord) -> HarnessOutcomeProgram { + let program = record.program(); + let nodes = program + .nodes() + .iter() + .map(|node| { + let linear_issue = node.linear_issue(); + + HarnessOutcomeProgramNode { + node_id: node.node_id().to_owned(), + stage: node.stage().as_str().to_owned(), + queue_intent: node.queue_intent().as_str().to_owned(), + linear_issue_id: linear_issue.map(|issue| issue.issue_id().to_owned()), + linear_issue_identifier: linear_issue + .map(|issue| issue.issue_identifier().to_owned()), + conflict_domains: node + .conflict_domains() + .iter() + .map(harness_conflict_domain_label) + .collect(), + } + }) + .collect::>(); + + HarnessOutcomeProgram { + program_id: record.program_id().to_owned(), + source_contract_id: record.source_contract_id().to_owned(), + node_count: nodes.len(), + nodes, + } +} + +fn harness_conflict_domain_label(domain: &ExecutionConflictDomain) -> String { + format!("{}:{}", domain.kind().as_str(), domain.key()) +} + +fn harness_outcome_signals( + events: &[PrivateExecutionEvent], + error_class: Option<&str>, +) -> HarnessOutcomeSignals { + let mut signals = HarnessOutcomeSignals::default(); + + if let Some(error_class) = error_class.filter(|class| class.starts_with("repo_gate_")) { + signals.validation_failure_count += 1; + + signals.validation_failure_classes.insert(error_class.to_owned()); + } + + for event in events { + match event.event_type() { + "phase_goal_completed" | "phase_goal_set" | "phase_goal_unavailable" => + push_phase_goal_signal(&mut signals, event), + "review_checkpoint" => push_review_signal(&mut signals, event.payload()), + "loop_guardrail_checkpoint" => push_guardrail_signal(&mut signals, event.payload()), + "progress_checkpoint" => push_progress_signal(&mut signals, event.payload()), + _ => {}, + } + } + + signals +} + +fn push_phase_goal_signal(signals: &mut HarnessOutcomeSignals, event: &PrivateExecutionEvent) { + let payload = event.payload(); + let nested = payload.get("payload").unwrap_or(payload); + let signal = json_string(nested.get("signal")).or_else(|| json_string(payload.get("signal"))); + let phase = json_string(nested.get("phase")).or_else(|| json_string(payload.get("phase"))); + let status = json_string(nested.get("status")); + + if signal.as_deref() == Some("validation_fail") { + signals.validation_failure_count += 1; + + signals.validation_failure_classes.insert(String::from("phase_goal_validation_fail")); + } + if phase.as_deref().is_some_and(|phase| phase.contains("repair")) { + signals.repair_phase_events += 1; + } + + signals.phase_goals.push(HarnessPhaseGoalOutcome { + event_type: event.event_type().to_owned(), + phase, + signal, + status, + }); +} + +fn push_review_signal(signals: &mut HarnessOutcomeSignals, payload: &Value) { + if let Some(status) = json_string(payload.get("status")) { + signals.review_statuses.insert(status); + } + + signals.nonclean_rounds = + signals.nonclean_rounds.max(payload.get("nonclean_rounds").and_then(Value::as_i64).unwrap_or(0)); + + let review = payload.get("review").unwrap_or(payload); + + signals.accepted_finding_count += json_array_len(review.get("accepted_findings")); + signals.rejected_finding_count += json_array_len(review.get("rejected_findings")); +} + +fn push_guardrail_signal(signals: &mut HarnessOutcomeSignals, payload: &Value) { + if let Some(reason) = json_string(payload.get("reason")) { + signals.guardrail_reasons.insert(reason); + } + if let Some(error_class) = json_string(payload.get("source_error_class")) + && error_class.starts_with("repo_gate_") + { + signals.validation_failure_count += 1; + + signals.validation_failure_classes.insert(error_class); + } +} + +fn push_progress_signal(signals: &mut HarnessOutcomeSignals, payload: &Value) { + if json_string(payload.get("phase")).is_some_and(|phase| phase.contains("repair")) { + signals.repair_phase_events += 1; + } +} + +fn harness_linear_projection( + linear_records: &[LinearExecutionEventRecord], +) -> HarnessLinearProjectionSummary { + let mut event_types = linear_records + .iter() + .map(|record| record.event_type.clone()) + .collect::>(); + + event_types.sort(); + event_types.dedup(); + + let final_record = linear_records + .iter() + .max_by(|left, right| left.event_timestamp.cmp(&right.event_timestamp)); + + HarnessLinearProjectionSummary { + event_types, + final_event_type: final_record.map(|record| record.event_type.clone()), + final_error_class: final_record.and_then(|record| record.error_class.clone()), + final_terminal_path: final_record.and_then(|record| record.terminal_path.clone()), + } +} + +fn harness_pr_urls( + explicit_pr_url: Option<&str>, + linear_records: &[LinearExecutionEventRecord], +) -> Vec { + let mut pr_urls = explicit_pr_url + .into_iter() + .map(str::to_owned) + .collect::>(); + + for record in linear_records { + if let Some(pr_url) = &record.pr_url { + pr_urls.insert(pr_url.clone()); + } + } + + pr_urls.into_iter().collect() +} + +fn harness_improvement_candidates( + input: &HarnessOutcomeRecordInput<'_>, + contracts: &[HarnessOutcomeContract], + programs: &[HarnessOutcomeProgram], + signals: &HarnessOutcomeSignals, + linear_projection: &HarnessLinearProjectionSummary, +) -> Vec { + let mut candidates = std::collections::BTreeMap::new(); + + push_contract_candidates(&mut candidates, input, contracts, programs); + push_signal_candidates(&mut candidates, input, signals, linear_projection); + + candidates.into_values().collect() +} + +fn push_contract_candidates( + candidates: &mut std::collections::BTreeMap, + input: &HarnessOutcomeRecordInput<'_>, + contracts: &[HarnessOutcomeContract], + programs: &[HarnessOutcomeProgram], +) { + if contracts.is_empty() { + insert_candidate( + candidates, + "missing_issue_template_field", + "contract_provenance_missing", + &format!("issue:{}", input.issue_identifier), + 0, + "Add source intent and Decision Contract id/provenance to generated issue briefs.", + ); + + return; + } + + for contract in contracts { + if contract.missing_decision_count > 0 { + insert_candidate( + candidates, + "underspecified_decision_contract", + "missing_decisions", + &format!("decision_contract:{}", contract.contract_id), + 0, + "Require missing decisions to be resolved before promotion or queueing.", + ); + } + if contract.generated_issue_ids.is_empty() && contract.generated_issue_identifiers.is_empty() + { + insert_candidate( + candidates, + "missing_issue_template_field", + "generated_issue_links_missing", + &format!("decision_contract:{}", contract.contract_id), + 0, + "Record generated issue ids or identifiers when research is promoted.", + ); + } + if contract.conflict_domains.is_empty() { + insert_candidate( + candidates, + "missing_issue_template_field", + "conflict_domains_missing", + &format!("decision_contract:{}", contract.contract_id), + 0, + "Require conflict-domain notes in contracts or generated issue templates.", + ); + } + } + for program in programs.iter().filter(|program| program.node_count == 0) { + insert_candidate( + candidates, + "stale_readiness_model", + "execution_program_has_no_nodes", + &format!("execution_program:{}", program.program_id), + 0, + "Regenerate internal Execution Program readiness from the accepted contract.", + ); + } +} + +fn push_signal_candidates( + candidates: &mut std::collections::BTreeMap, + input: &HarnessOutcomeRecordInput<'_>, + signals: &HarnessOutcomeSignals, + linear_projection: &HarnessLinearProjectionSummary, +) { + if signals.validation_failure_count > 0 { + insert_candidate( + candidates, + "weak_prompt", + "validation_failed_after_generation", + &format!("issue:{}", input.issue_identifier), + signals.validation_failure_count, + "Tighten phase prompts or preflight checks around the failing validation class.", + ); + } + if signals.accepted_finding_count > 0 { + insert_candidate( + candidates, + "weak_prompt", + "accepted_review_findings", + &format!("issue:{}", input.issue_identifier), + signals.accepted_finding_count, + "Convert accepted reviewer findings into prompt, skill, or validator hardening.", + ); + } + + for reason in &signals.guardrail_reasons { + let (kind, recommendation) = guardrail_candidate_kind(reason); + + insert_candidate( + candidates, + kind, + reason, + &format!("issue:{}", input.issue_identifier), + signals.guardrail_reasons.len(), + recommendation, + ); + } + + if input.error_class == Some("uncovered_direction") + || linear_projection.final_error_class.as_deref() == Some("uncovered_direction") + { + insert_candidate( + candidates, + "underspecified_decision_contract", + "uncovered_direction", + &format!("issue:{}", input.issue_identifier), + 1, + "Feed the uncovered direction back into the Decision Contract before retrying.", + ); + } +} + +fn guardrail_candidate_kind(reason: &str) -> (&'static str, &'static str) { + match reason { + "dependency_program_stale" => ( + "stale_readiness_model", + "Refresh dependency readiness and queue-label policy for the affected program node.", + ), + "uncovered_direction" => ( + "underspecified_decision_contract", + "Feed the uncovered direction back into the Decision Contract before retrying.", + ), + "validation_repeat" | "remaining_delta_unchanged" => ( + "missing_validator", + "Promote the repeated failure into an earlier deterministic validator or fixture.", + ), + _ => ( + "weak_prompt", + "Tighten loop instructions so future attempts stop or repair earlier.", + ), + } +} + +fn insert_candidate( + candidates: &mut std::collections::BTreeMap, + kind: &str, + reason_code: &str, + target: &str, + source_event_count: usize, + recommendation: &str, +) { + let key = format!("{kind}:{reason_code}:{target}"); + + candidates.entry(key).or_insert_with(|| HarnessImprovementCandidateSummary { + kind: kind.to_owned(), + reason_code: reason_code.to_owned(), + target: target.to_owned(), + source_event_count, + recommendation: recommendation.to_owned(), + }); +} + +fn harness_candidates_from_payload(payload: &Value) -> Vec { + payload + .get("improvement_candidates") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|candidate| { + Some(HarnessImprovementCandidateSummary { + kind: json_string(candidate.get("kind"))?, + reason_code: json_string(candidate.get("reason_code"))?, + target: json_string(candidate.get("target"))?, + source_event_count: candidate + .get("source_event_count") + .and_then(Value::as_u64) + .and_then(|value| usize::try_from(value).ok()) + .unwrap_or(0), + recommendation: json_string(candidate.get("recommendation"))?, + }) + }) + .collect() +} + +fn json_string(value: Option<&Value>) -> Option { + value.and_then(Value::as_str).filter(|value| !value.is_empty()).map(str::to_owned) +} + +fn json_array_len(value: Option<&Value>) -> usize { + value.and_then(Value::as_array).map_or(0, Vec::len) +} diff --git a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs index 4f379591e..3f95a931d 100644 --- a/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs +++ b/apps/decodex/src/orchestrator/tests/operator/status/agent_evidence.rs @@ -1,3 +1,8 @@ +use orchestrator::HarnessOutcomeKind; +use orchestrator::HarnessOutcomeRecordInput; + +use crate::loop_contract::DecisionContract; + #[test] fn agent_evidence_snapshot_writes_index_blockers_capsules_and_event_stream() { let temp_dir = TempDir::new().expect("temp dir should create"); @@ -325,6 +330,212 @@ fn private_evidence_readback_direct_lookup_uses_stored_issue_id() { assert!(readback.warnings.is_empty()); } +#[test] +fn harness_outcome_records_validation_review_and_repair_signals() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + + state_store + .record_run_attempt("run-harness", "issue-harness", 2, "failed") + .expect("run should persist"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + "issue-harness", + "run-harness", + 2, + "phase_goal_completed", + serde_json::json!({ + "schema": "decodex.phase_goal_signal/1", + "phase": "repair_validation_failures", + "payload": { + "signal": "validation_fail", + "status": "complete" + } + }), + ) + .expect("phase goal evidence should append"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + "issue-harness", + "run-harness", + 2, + "review_checkpoint", + serde_json::json!({ + "phase": "handoff", + "status": "findings", + "head_sha": "abc123", + "nonclean_rounds": 1, + "review": { + "accepted_findings": [{"summary": "cover the missing edge case"}], + "rejected_findings": [] + } + }), + ) + .expect("review evidence should append"); + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + "issue-harness", + "run-harness", + 2, + "progress_checkpoint", + serde_json::json!({ + "phase": "review_repair", + "focus": "repair accepted finding" + }), + ) + .expect("progress evidence should append"); + + let recorded = orchestrator::record_harness_outcome_for_issue_run( + &state_store, + HarnessOutcomeRecordInput { + project_id: TEST_SERVICE_ID, + issue_id: "issue-harness", + issue_identifier: "PUB-110", + run_id: "run-harness", + attempt_number: 2, + outcome: HarnessOutcomeKind::RetryableFailure, + error_class: Some("repo_gate_verify_failed"), + validation_result: Some("failed"), + pr_url: None, + }, + ) + .expect("harness outcome should record"); + let payload = recorded.payload(); + + assert_eq!(recorded.event_type(), "harness_outcome"); + assert_eq!(payload["schema"], "decodex.harness_outcome/1"); + assert_eq!(payload["validation"]["result"], "failed"); + assert_eq!(payload["validation"]["failure_count"], 2); + assert_eq!( + payload["validation"]["failure_classes"], + serde_json::json!(["phase_goal_validation_fail", "repo_gate_verify_failed"]) + ); + assert_eq!(payload["repair"]["repair_attempt_observed"], true); + assert_eq!(payload["review"]["accepted_finding_count"], 1); + assert!( + payload["improvement_candidates"] + .as_array() + .expect("candidates should be an array") + .iter() + .any(|candidate| candidate["reason_code"] == "accepted_review_findings") + ); + + let request = EvidenceRequest { + config_path: None, + issue: "issue-harness", + run_id: Some("run-harness"), + attempt_number: Some(2), + json: true, + include_payload: false, + }; + let readback = orchestrator::build_private_evidence_readback( + &state_store, + &config, + &request, + ) + .expect("private evidence should read"); + + assert!( + readback + .improvement_candidates + .iter() + .any(|candidate| candidate.reason_code == "accepted_review_findings") + ); + assert!(readback.events.iter().all(|event| event.payload.is_none())); +} + +#[test] +fn harness_eval_fixture_recommends_contract_improvement_without_payload_leakage() { + let (_temp_dir, config, _workflow) = temp_project_layout(); + let state_store = StateStore::open_in_memory().expect("state store should open"); + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/harness_improvement/incomplete_contract_eval.json" + ))) + .expect("harness eval fixture should parse"); + let issue_id = fixture["issue_id"].as_str().expect("fixture issue id"); + let issue_identifier = fixture["issue_identifier"].as_str().expect("fixture issue identifier"); + let run_id = fixture["run_id"].as_str().expect("fixture run id"); + let attempt_number = fixture["attempt_number"].as_i64().expect("fixture attempt"); + let contract: DecisionContract = + serde_json::from_value(fixture["decision_contract"].clone()) + .expect("fixture contract should deserialize"); + + state_store + .upsert_worktree(TEST_SERVICE_ID, issue_id, "y/decodex-xy-857-eval", ".worktrees/XY-857-EVAL") + .expect("worktree should persist"); + state_store + .record_run_attempt(run_id, issue_id, attempt_number, "terminal_guarded") + .expect("run should persist"); + state_store + .upsert_decision_contract(TEST_SERVICE_ID, Some(issue_id), contract) + .expect("contract should persist"); + + for event in fixture["private_events"].as_array().expect("fixture events") { + state_store + .append_private_execution_event( + TEST_SERVICE_ID, + issue_id, + run_id, + attempt_number, + event["event_type"].as_str().expect("fixture event type"), + event["payload"].clone(), + ) + .expect("fixture event should append"); + } + + orchestrator::record_harness_outcome_for_issue_run( + &state_store, + HarnessOutcomeRecordInput { + project_id: TEST_SERVICE_ID, + issue_id, + issue_identifier, + run_id, + attempt_number, + outcome: HarnessOutcomeKind::ManualAttention, + error_class: Some("uncovered_direction"), + validation_result: None, + pr_url: None, + }, + ) + .expect("harness outcome should record"); + + let request = EvidenceRequest { + config_path: None, + issue: issue_identifier, + run_id: Some(run_id), + attempt_number: Some(attempt_number), + json: false, + include_payload: false, + }; + let readback = orchestrator::build_private_evidence_readback( + &state_store, + &config, + &request, + ) + .expect("fixture readback should summarize private evidence"); + let expected = &fixture["expected_candidate"]; + + assert!( + readback.improvement_candidates.iter().any(|candidate| { + candidate.kind == expected["kind"].as_str().expect("expected kind") + && candidate.reason_code + == expected["reason_code"].as_str().expect("expected reason") + && candidate.target == expected["target"].as_str().expect("expected target") + }) + ); + + let rendered = orchestrator::render_private_evidence_readback(&readback); + + assert!(rendered.contains("Improvement Candidates")); + assert!(rendered.contains("underspecified_decision_contract")); + assert!(!rendered.contains("Decide how generated issues must cite")); + assert!(readback.events.iter().all(|event| event.payload.is_none())); +} + fn read_json_file(path: &Path) -> Value { let body = fs::read_to_string(path).expect("JSON file should exist"); diff --git a/docs/reference/operator-control-plane.md b/docs/reference/operator-control-plane.md index 6ac49550a..6b4e06db1 100644 --- a/docs/reference/operator-control-plane.md +++ b/docs/reference/operator-control-plane.md @@ -147,6 +147,11 @@ The evidence command reads runtime SQLite directly, so it remains useful when tr or GitHub connectors are unavailable. By default it prints compact payload summaries rather than full structured payloads; add `--include-payload` only for local repair work that needs full private payload values. +When a run has harness-outcome telemetry, the evidence readback also summarizes +candidate improvements by kind, reason code, target, source-event count, and +recommendation. These summaries are local operator guidance; they are not Linear +ledger records and do not automatically edit prompts, skills, validators, issue +templates, or loop policy. ## State Ownership @@ -294,6 +299,10 @@ the next allowlisted lifecycle summary instead of pasting local evidence payload The same boundary applies to Decision Contracts: the operator surface may show status, readiness summary, generated issue links, or public projection references, but the versioned contract payload and private evidence references remain runtime-local. +Harness-outcome telemetry follows the same rule: the operator surface may show compact +improvement-candidate summaries, while the correlated source intent, contract payloads, +review details, validation diagnostics, and guardrail checkpoints remain private +runtime evidence unless explicitly read through the local evidence command. Worktree visibility follows the owning dashboard section: diff --git a/docs/spec/linear-execution-ledger.md b/docs/spec/linear-execution-ledger.md index 3997b5ce9..edc622f58 100644 --- a/docs/spec/linear-execution-ledger.md +++ b/docs/spec/linear-execution-ledger.md @@ -32,8 +32,8 @@ Linear comments, the Decodex runtime database, and short-lived heartbeat markers - Decision Contracts and internal Execution Programs are not Linear execution-ledger records. A ledger record may summarize or link to generated issues after promotion, but the versioned `decodex.decision_contract/1`, - `decodex.execution_program/1`, and private loop evidence payloads stay in runtime - SQLite. + `decodex.execution_program/1`, `decodex.harness_outcome/1`, and private loop + evidence payloads stay in runtime SQLite. ## Comment body format @@ -352,6 +352,8 @@ runtime telemetry: - token counts, largest tool-output sizes, and context-pressure warnings - review-policy convergence counters that only guide the current retained-lane retry loop +- harness-outcome telemetry and improvement-candidate payloads that correlate + contracts, validation, review, repair, PR outcomes, and guardrail stops - full `issue_progress_checkpoint` payloads, including raw focus, next action, blockers, evidence, verification, and local head evidence - transient diagnostic details that help the local operator understand whether an active diff --git a/docs/spec/loop-runtime.md b/docs/spec/loop-runtime.md index 1bdb465a5..84e4bd9df 100644 --- a/docs/spec/loop-runtime.md +++ b/docs/spec/loop-runtime.md @@ -345,6 +345,30 @@ attempts, dependency blockers, and accepted contract gaps should feed improvemen - ready-node selection and conflict-domain policy - future loop guardrails +Runtime outcome feedback is recorded as local private execution evidence. The +versioned payload is `decodex.harness_outcome/1` with `event_type = +"harness_outcome"` in `private_execution_events`. It correlates: + +- source intent, Decision Contract ids, generated issue identifiers, generated node + ids, and conflict domains +- phase-goal signals, validation results, validation failure classes, and repair + attempts +- independent review checkpoint status, accepted findings, rejected findings, and + non-clean review rounds +- manual-attention or guardrail reason codes such as `uncovered_direction`, + `dependency_program_stale`, `validation_repeat`, or `no_effective_diff` +- PR handoff, retained repair, closeout, cleanup, and terminal failure outcomes as + summarized from cached Linear execution records and local runtime state +- candidate harness improvements such as `missing_validator`, `weak_prompt`, + `missing_issue_template_field`, `underspecified_decision_contract`, and + `stale_readiness_model` + +Operator readback may summarize improvement candidates through `decodex evidence` or +derived agent-evidence files. Default readback must expose only compact candidate kind, +reason code, target, source-event count, and recommendation text. Full private payloads +remain local and require explicit private evidence readback such as +`--include-payload`. + Harness improvement does not retroactively change a lane's accepted Decision Contract. It also does not authorize automatic execution from latent research output. Apply future policy changes only after the relevant spec, decision, or project contract is diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index 3ae1eae07..5e122178a 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -44,7 +44,7 @@ state or this state machine. ## Source of truth boundaries -- The Decodex runtime SQLite database is the single-machine source of truth for active leases, attempts, run-control channels, protocol events, private execution events, latent and promoted Decision Contracts, internal Execution Programs, worktree mappings, retained PR state, review-policy checkpoints, loop-guardrail checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. +- The Decodex runtime SQLite database is the single-machine source of truth for active leases, attempts, run-control channels, protocol events, private execution events, harness-outcome telemetry, latent and promoted Decision Contracts, internal Execution Programs, worktree mappings, retained PR state, review-policy checkpoints, loop-guardrail checkpoints, retry state, phase timing, project registration, tracker cache, PR cache, and connector backoff. - Linear remains the team-visible tracker surface for issue lifecycle, queue/active/manual-attention labels, and coarse lifecycle summaries such as start, PR-ready, blocked, failed, landed, and done. - Versioned Linear execution event comments use the schema in [`linear-execution-ledger.md`](./linear-execution-ledger.md), but fine-grained runtime truth must not be rebuilt from comments every tick. @@ -69,7 +69,7 @@ mirror: | Surface | Boundary | | --- | --- | -| Runtime SQLite `private_execution_events` | Structured private execution evidence for the local Decodex installation. This is where full checkpoint payloads, verification notes, local head evidence, and recovery detail belong. | +| Runtime SQLite `private_execution_events` | Structured private execution evidence for the local Decodex installation. This is where full checkpoint payloads, verification notes, local head evidence, recovery detail, and `decodex.harness_outcome/1` feedback records belong. | | Runtime SQLite `decision_contracts` | Versioned `decodex.decision_contract/1` payloads produced by research/design and later promoted into execution authority. The row status is indexed for local runtime lookup, but the JSON payload remains the contract authority. | | Runtime SQLite `execution_programs` | Versioned `decodex.execution_program/1` payloads derived from accepted Decision Contracts. They hold internal node readiness, dependency, conflict-domain, queue-intent, drift, and normal-issue mapping state; Linear issue descriptions and ledger comments are only coarse projections. | | Runtime SQLite `run_control_channels` | Local control capability metadata for active run attempts. It records the project, issue, run id, attempt, transport, local channel path, channel status, and publish/update timestamps needed to route future control requests without bypassing active lease ownership. | @@ -89,6 +89,9 @@ The following facts are local runtime truth and must not be rebuilt from Linear - active run-control channel metadata and local control audit events - protocol events, event counts, event timestamps, and thread/liveness hydration fields - private execution events carrying structured local evidence for an issue/run/attempt +- harness-outcome events correlating Decision Contracts, generated issue or node ids, + validation/review/repair/manual-attention/PR lifecycle outcomes, and private + improvement candidates - review-policy checkpoint state: current phase, normalized status, lane head, consecutive non-clean round count, and structured independent-review detail - loop-guardrail checkpoint state: normalized reason, fingerprint, consecutive diff --git a/plugins/decodex/skills/decodex/SKILL.md b/plugins/decodex/skills/decodex/SKILL.md index f43d81be0..55fcfb48f 100644 --- a/plugins/decodex/skills/decodex/SKILL.md +++ b/plugins/decodex/skills/decodex/SKILL.md @@ -77,6 +77,10 @@ boundary without making the user learn the commands. - Decodex-native research/design behavior belongs to `apps/decodex/src/research_design.rs` and `docs/spec/loop-runtime.md`; external research artifacts are supporting evidence only for Decodex runtime semantics. +- Harness-improvement recommendations from `decodex evidence` are advisory runtime + feedback. Treat them as candidates for an explicit accepted improvement path; do not + auto-edit prompts, skills, validators, issue templates, or loop policies solely + because a private outcome record suggested them. - Operator lane-control capabilities belong to `docs/spec/lane-control.md`, with the low-level app-server method boundary in `docs/spec/app-server.md`. - Operator procedures belong to `docs/runbook/`.