XY-857: Add harness-improvement telemetry and eval feedback loop

apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json

@@ -0,0 +1,93 @@
+{
+  "schema": "decodex.harness_improvement_eval/1",
+  "record_version": 1,
+  "project_id": "decodex",
+  "issue_id": "issue-857-fixture",
+  "issue_identifier": "XY-857-EVAL",
+  "run_id": "xy-857-eval-run",
+  "attempt_number": 1,
+  "decision_contract": {
+    "schema": "decodex.decision_contract/1",
+    "record_version": 1,
+    "contract_id": "incomplete-contract-eval",
+    "status": "needs_human_decision",
+    "source_intent": {
+      "summary": "Arrange a research result whose generated issue brief omitted the contract authority boundary.",
+      "user_utterance": "push this forward",
+      "source_issue_identifier": "XY-857-EVAL"
+    },
+    "research_provenance": [
+      {
+        "kind": "fixture",
+        "reference": "apps/decodex/fixtures/harness_improvement/incomplete_contract_eval.json",
+        "summary": "Fixture intentionally leaves execution authority underspecified."
+      }
+    ],
+    "research_evidence": [
+      {
+        "claim": "Generated work can be unsafe when the issue brief omits the accepted contract id.",
+        "support": "The lane stopped with uncovered direction instead of guessing the missing authority boundary.",
+        "source_ref": "fixture"
+      }
+    ],
+    "accepted_authority": {
+      "accepted_objectives": [],
+      "non_goals": [
+        "Do not queue generated issues when promotion authority is missing."
+      ],
+      "constraints": [
+        "Keep private research evidence in runtime SQLite."
+      ],
+      "assumptions": [],
+      "objections": [
+        "The generated issue lacks a contract id and conflict-domain field."
+      ],
+      "stop_conditions": [
+        "Stop for human decision when generated issue authority is missing."
+      ]
+    },
+    "execution_readiness": {
+      "summary": "Not ready for issue shaping because the generated issue template omitted the accepted contract boundary.",
+      "ready_for_issue_shaping": false,
+      "missing_decisions": [
+        "Decide how generated issues must cite accepted Decision Contract provenance before queueing."
+      ],
+      "validation_expectations": [
+        "A harness eval should emit an underspecified_decision_contract recommendation."
+      ],
+      "risk_notes": [
+        "A future agent could implement from latent evidence if the issue template omits the authority field."
+      ],
+      "proposed_issue_summaries": [],
+      "conflict_domains": [],
+      "queue_intent": []
+    },
+    "links": {
+      "generated_issue_ids": [],
+      "generated_issue_identifiers": [],
+      "execution_program_node_ids": []
+    },
+    "evidence_boundary": {
+      "private_evidence_refs": [],
+      "public_projection_refs": [],
+      "public_summary": "Incomplete contract fixture for harness-improvement evaluation."
+    }
+  },
+  "private_events": [
+    {
+      "event_type": "loop_guardrail_checkpoint",
+      "payload": {
+        "schema": "decodex.loop_guardrail_checkpoint/1",
+        "reason": "uncovered_direction",
+        "consecutive_count": 3,
+        "threshold": 3,
+        "source_error_class": "uncovered_direction"
+      }
+    }
+  ],
+  "expected_candidate": {
+    "kind": "underspecified_decision_contract",
+    "reason_code": "missing_decisions",
+    "target": "decision_contract:incomplete-contract-eval"
+  }
+}

apps/decodex/src/execution_program.rs

@@ -38,6 +38,21 @@ pub(crate) enum ExecutionProgramNodeStage {
 	/// Review, PR, delivery, or handoff work.
 	Handoff,
 }
+impl ExecutionProgramNodeStage {
+	/// Stable machine-readable stage name.
+	pub(crate) fn as_str(self) -> &'static str {
+		match self {
+			Self::Research => "research",
+			Self::Design => "design",
+			Self::Spec => "spec",
+			Self::Schema => "schema",
+			Self::Runtime => "runtime",
+			Self::Plugin => "plugin",
+			Self::Eval => "eval",
+			Self::Handoff => "handoff",
+		}
+	}
+}
 
 /// Queue intent for one internal Execution Program node.
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)]
@@ -59,6 +74,19 @@ pub(crate) enum ExecutionQueueIntent {
 	Canceled,
 }
 impl ExecutionQueueIntent {
+	/// Stable machine-readable queue-intent name.
+	pub(crate) fn as_str(self) -> &'static str {
+		match self {
+			Self::NotReady => "not_ready",
+			Self::ReadyToQueue => "ready_to_queue",
+			Self::Queued => "queued",
+			Self::Active => "active",
+			Self::Paused => "paused",
+			Self::Done => "done",
+			Self::Canceled => "canceled",
+		}
+	}
+
 	fn is_terminal(self) -> bool {
 		matches!(self, Self::Done | Self::Canceled)
 	}
@@ -82,7 +110,8 @@ pub(crate) enum ExecutionConflictDomainKind {
 	ReviewSurface,
 }
 impl ExecutionConflictDomainKind {
-	fn as_str(self) -> &'static str {
+	/// Stable machine-readable conflict-domain class name.
+	pub(crate) fn as_str(self) -> &'static str {
 		match self {
 			Self::File => "file",
 			Self::Module => "module",
@@ -159,6 +188,11 @@ impl ExecutionConflictDomain {
 		&self.key
 	}
 
+	/// Stable conflict-domain kind.
+	pub(crate) fn kind(&self) -> ExecutionConflictDomainKind {
+		self.kind
+	}
+
 	fn validate(&self) -> Result<()> {
 		validate_required("execution program conflict_domain.key", &self.key)
 	}
@@ -284,6 +318,11 @@ impl ExecutionLinearIssueMapping {
 		&self.issue_identifier
 	}
 
+	/// Linear issue id used by tracker APIs.
+	pub(crate) fn issue_id(&self) -> &str {
+		&self.issue_id
+	}
+
 	/// Tracker workflow state for the mapped issue.
 	pub(crate) fn issue_state(&self) -> &str {
 		&self.issue_state
@@ -439,11 +478,21 @@ impl ExecutionProgramNode {
 		&self.node_id
 	}
 
+	/// Node execution stage.
+	pub(crate) fn stage(&self) -> ExecutionProgramNodeStage {
+		self.stage
+	}
+
 	/// Node queue intent.
 	pub(crate) fn queue_intent(&self) -> ExecutionQueueIntent {
 		self.queue_intent
 	}
 
+	/// Conflict domains occupied by this node.
+	pub(crate) fn conflict_domains(&self) -> &[ExecutionConflictDomain] {
+		&self.conflict_domains
+	}
+
 	/// Linked normal Linear issue, when the node is executable.
 	pub(crate) fn linear_issue(&self) -> Option<&ExecutionLinearIssueMapping> {
 		self.linear_issue.as_ref()

apps/decodex/src/loop_contract.rs

@@ -535,6 +535,10 @@ pub(crate) struct DecisionContractLinks {
 }
 #[allow(dead_code)]
 impl DecisionContractLinks {
+	pub(crate) fn generated_issue_ids(&self) -> &[String] {
+		&self.generated_issue_ids
+	}
+
 	pub(crate) fn generated_issue_identifiers(&self) -> &[String] {
 		&self.generated_issue_identifiers
 	}

apps/decodex/src/orchestrator.rs

@@ -1,4 +1,9 @@
 mod lane_control;
+mod harness_improvement {
+	use crate::orchestrator::{IssueRunPlan, Result, Serialize, StateStore, Value, records, state};
+
+	include!("orchestrator/harness_improvement.rs");
+}
 
 pub(crate) use lane_control::{
 	DEFAULT_STEER_RESULT_WAIT_TIMEOUT, LaneInspectRequest, LaneInterruptRequest, interrupt_lane,
@@ -36,6 +41,12 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use crate::{agent, default_branch_sync, git_credentials, maintenance, state};
 #[rustfmt::skip]
 use crate::{agent::{ACTIVE_RUN_IDLE_TIMEOUT, AppServerCapabilityPreflightFailure, AppServerDynamicToolFailure, AppServerHomePreflightFailure, AppServerPhaseGoalFailure, AppServerProcessEnv, AppServerRunRequest, AppServerRunResult, AppServerTransportFailure, AppServerTurnFailure, ISSUE_DELIVERY_CLOSEOUT_COMPLETE_TOOL_NAME, ISSUE_LABEL_ADD_TOOL_NAME, ISSUE_PROGRESS_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_CHECKPOINT_TOOL_NAME, ISSUE_REVIEW_HANDOFF_TOOL_NAME, ISSUE_REVIEW_REPAIR_COMPLETE_TOOL_NAME, ISSUE_TERMINAL_FINALIZE_TOOL_NAME, ISSUE_TRANSITION_TOOL_NAME, DecodexRunContext, DecodexToolBridge, PhaseGoalController, PhaseGoalKind, PhaseGoalSpec, PhaseGoalTransition, ReviewExecutionMode, ReviewHandoffContext, ReviewHandoffWritebackFailed, ReviewPolicyStopReason, ReviewPolicyStopRequested, RunCompletionDisposition, TrackerToolBridge, TurnContinuationGuard}, config::{InternalReviewMode, ServiceConfig}, execution_program::{ExecutionProgramOperatorSummary, ExecutionProgramReadinessContext, ExecutionWorkflowPolicy}, git_credentials::GitCredentialSource, github, prelude::{Result, eyre}, state::{ChildAgentActivityBucket, ChildAgentActivitySummary, CodexAccountActivitySummary, ExecutionProgramRecord, LoopGuardrailCheckpoint, LoopGuardrailCheckpointInput, ProjectRegistration, ProjectRunStatus, ProtocolActivitySummary, RUN_OPERATION_AGENT_RUN, RUN_OPERATION_APP_SERVER_PREFLIGHT, RUN_OPERATION_GIT_CREDENTIALS, RUN_OPERATION_IDLE, RUN_OPERATION_RECONCILIATION, RUN_OPERATION_REPO_GATE, RUN_OPERATION_REVIEW_WRITEBACK, RUN_OPERATION_WAITING_EXTERNAL, ReviewHandoffMarker, ReviewOrchestrationMarker, RunActivityMarker, RunAttempt, StateStore, WorktreeMapping}, tracker::{IssueTracker, TrackerComment, TrackerIssue, linear::LinearClient, records}, workflow::{WorkflowDocument, WorkflowExecution}, worktree::{WorktreeManager, WorktreeSpec}};
+use harness_improvement::{
+	HarnessImprovementCandidateSummary, HarnessOutcomeKind,
+	harness_improvement_candidates_from_private_events, record_harness_outcome_best_effort,
+};
+#[cfg(test)]
+use harness_improvement::{HarnessOutcomeRecordInput, record_harness_outcome_for_issue_run};
 
 include!("orchestrator/types.rs");
 

apps/decodex/src/orchestrator/agent_evidence.rs

@@ -1,7 +1,5 @@
 use std::collections::{self, BTreeMap};
 
-use state::PrivateExecutionEvent;
-
 const AGENT_HANDOFF_INDEX_SCHEMA: &str = "decodex.agent_handoff_index/1";
 const AGENT_BLOCKER_SNAPSHOT_SCHEMA: &str = "decodex.blocker_snapshot/1";
 const AGENT_RUN_CAPSULE_SCHEMA: &str = "decodex.run_capsule/1";
@@ -261,6 +259,7 @@ struct PrivateEvidenceReadback {
 	event_count: usize,
 	latest_event_type: Option<String>,
 	latest_event_at: Option<String>,
+	improvement_candidates: Vec<HarnessImprovementCandidateSummary>,
 	events: Vec<PrivateEvidenceReadbackEvent>,
 	warnings: Vec<String>,
 }
@@ -648,6 +647,7 @@ fn build_private_evidence_readback(
 		event_count: events.len(),
 		latest_event_type: latest_event.map(|event| event.event_type().to_owned()),
 		latest_event_at: latest_event.map(|event| event.recorded_at().to_owned()),
+		improvement_candidates: harness_improvement_candidates_from_private_events(&events),
 		events: events
 			.iter()
 			.map(|event| private_evidence_readback_event(event, request.include_payload))
@@ -719,12 +719,12 @@ fn resolve_private_evidence_target(
 }
 
 fn private_evidence_direct_lookup_issue_id(
-	events: &[PrivateExecutionEvent],
+	events: &[state::PrivateExecutionEvent],
 	selector: &str,
 ) -> Result<Option<String>> {
 	let issue_ids = events
 		.iter()
-		.map(PrivateExecutionEvent::issue_id)
+		.map(state::PrivateExecutionEvent::issue_id)
 		.collect::<collections::BTreeSet<_>>();
 
 	if issue_ids.is_empty() {
@@ -767,7 +767,7 @@ fn private_evidence_run_matches_issue(
 }
 
 fn private_evidence_readback_event(
-	event: &PrivateExecutionEvent,
+	event: &state::PrivateExecutionEvent,
 	include_payload: bool,
 ) -> PrivateEvidenceReadbackEvent {
 	PrivateEvidenceReadbackEvent {
@@ -894,6 +894,10 @@ fn render_private_evidence_readback(readback: &PrivateEvidenceReadback) -> Strin
 	output.push_str(&format!("evidence_ref: {}\n", readback.evidence_ref));
 	output.push_str(&format!("payload_mode: {}\n", readback.payload_mode));
 	output.push_str(&format!("event_count: {}\n", readback.event_count));
+	output.push_str(&format!(
+		"improvement_candidate_count: {}\n",
+		readback.improvement_candidates.len()
+	));
 	output.push_str(&format!(
 		"latest_event_type: {}\n",
 		readback.latest_event_type.as_deref().unwrap_or("none")
@@ -907,6 +911,23 @@ fn render_private_evidence_readback(readback: &PrivateEvidenceReadback) -> Strin
 		output.push_str(&format!("warnings: {}\n", readback.warnings.join(", ")));
 	}
 
+	output.push_str("\nImprovement Candidates\n");
+
+	if readback.improvement_candidates.is_empty() {
+		output.push_str("- none\n");
+	} else {
+		for candidate in &readback.improvement_candidates {
+			output.push_str(&format!(
+				"- kind: {}\n  reason_code: {}\n  target: {}\n  source_event_count: {}\n  recommendation: {}\n",
+				candidate.kind,
+				candidate.reason_code,
+				candidate.target,
+				candidate.source_event_count,
+				candidate.recommendation
+			));
+		}
+	}
+
 	output.push_str("\nEvents\n");
 
 	if readback.events.is_empty() {