From 1db18a1760a11622c51e18d059cb0fbbc22217b4 Mon Sep 17 00:00:00 2001 From: Shibo Sheng Date: Tue, 11 Aug 2026 17:38:28 +0800 Subject: [PATCH 01/11] feat(agent-org): add authoritative team lifecycle Establish durable Starting materialization receipts and a centralized quiescence boundary so Working teams enter Idle only from committed facts. Keep Run View pure-read, bound the global watchdog, and stop frontend polling for hidden or non-pollable teams behind one fail-closed rollout gate. Verification: - cargo clippy --all-targets -- -D warnings - Packaged Tauri Command+5 foreground, hidden, and restore lifecycle passed - Run View refresh left the isolated database byte-identical Pre-commit hook ran. Total eslint: 0, total circular: 0 --- .../coordination/agent_inbox/store_write.rs | 2 +- .../agent_org_plan_approvals/store.rs | 4 +- .../coordination/agent_org_runs/helpers.rs | 46 +- .../agent_org_runs/materialization.rs | 378 +++++++++ .../core/coordination/agent_org_runs/mod.rs | 107 ++- .../coordination/agent_org_runs/progress.rs | 2 +- .../{finality.rs => quiescence.rs} | 335 +++++--- .../coordination/agent_org_runs/rollout.rs | 40 + .../core/coordination/agent_org_runs/store.rs | 779 +++++++++++++----- .../core/coordination/agent_org_runs/tests.rs | 450 ++++++++-- .../core/coordination/agent_org_tasks/mod.rs | 6 +- .../agent_org_tasks/store/read.rs | 2 +- .../coordination/agent_org_tasks/tests.rs | 6 +- .../core/coordination/agent_org_watchdog.rs | 14 +- .../coordination/agent_org_watchdog/budget.rs | 49 +- .../agent_org_watchdog/inspect.rs | 54 +- .../agent_org_watchdog/recover.rs | 86 +- .../coordination/agent_org_watchdog/tests.rs | 79 +- .../src/core/definitions/commands.rs | 3 +- .../src/core/session/launch/launch_helpers.rs | 17 +- .../src/core/session/launch/launch_org.rs | 427 +++++----- .../src/core/session/launch/launch_tests.rs | 6 +- .../agent-core/src/core/session/launch/mod.rs | 656 +++++++++++++-- .../agent-core/src/core/session/scheduler.rs | 73 +- .../agent-core/src/core/session/turn/entry.rs | 1 + .../core/session/turn/event_handler/mod.rs | 85 +- .../core/session/turn/processor/execute.rs | 2 + .../turn/processor/inbox_drain/guard.rs | 2 +- .../src/core/session/turn/processor/mod.rs | 48 +- .../agent-core/src/core/tools/call_context.rs | 2 +- .../orchestration/agent_org/inbox_repair.rs | 6 +- .../orchestration/agent_org/run_complete.rs | 6 +- .../agent_org/send_message/persistence.rs | 2 +- .../agent_org/send_message/tests.rs | 6 +- .../orchestration/agent_org/task_list_get.rs | 12 +- .../orchestration/agent_org/task_tests.rs | 44 +- .../tools/impls/orchestration/member_idle.rs | 6 +- .../subagent_handler/persistence.rs | 10 +- .../foundation/bus/event_pipeline_bridge.rs | 21 +- .../src/foundation/session_bridge.rs | 48 +- src-tauri/crates/agent-core/src/lifecycle.rs | 30 +- .../src/state/commands/session/identity.rs | 18 + .../commands/session/message/org_wake.rs | 13 +- .../state/commands/session/message/send.rs | 96 ++- .../state/commands/session/message/tests.rs | 78 +- .../commands/session/org_tasks/group_chat.rs | 31 +- .../session/org_tasks/intervention.rs | 4 + .../commands/session/org_tasks/lifecycle.rs | 6 +- .../session/org_tasks/plan_approval.rs | 22 +- .../commands/session/org_tasks/run_view.rs | 33 +- .../state/commands/session/org_tasks/tests.rs | 100 ++- .../src/state/commands/session/persistence.rs | 388 +-------- .../crates/agent-core/src/state/unified.rs | 34 +- .../src/agent_core_bridge.rs | 25 + .../session-persistence/src/turn_intents.rs | 120 ++- .../cli/session_runner/finalize.rs | 4 +- src-tauri/src/agent_sessions/cli/types.rs | 2 +- .../event_pipeline/agent_core_bridge.rs | 4 +- src-tauri/src/api/agent/test/agent_org.rs | 30 +- src-tauri/src/lib.rs | 42 +- src-tauri/src/setup/hooks.rs | 11 + src/api/tauri/agent/orgTasks.ts | 12 +- src/api/tauri/agent/session.ts | 2 + src/config/agentOrgRedesign.ts | 15 + .../components/agentOrgRunViewStore.test.ts | 121 ++- .../components/agentOrgRunViewStore.ts | 58 +- .../components/useAgentOrgRunView.ts | 2 + src/i18n/locales/en/sessions.json | 6 +- src/i18n/locales/zh/sessions.json | 6 +- webpack.config.js | 3 + 70 files changed, 3759 insertions(+), 1479 deletions(-) create mode 100644 src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs rename src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/{finality.rs => quiescence.rs} (69%) create mode 100644 src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs create mode 100644 src/config/agentOrgRedesign.ts diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs index 41225e0dc8..bef2d4a65c 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs @@ -28,7 +28,7 @@ impl AgentInboxStore { /// Atomically persist a recovery/outbox message only while its Agent Org /// run is still Running. This shares the sessions writer lock and an - /// IMMEDIATE transaction with run finality, so a queued watchdog action + /// IMMEDIATE transaction with Team Quiescence, so a queued watchdog action /// cannot insert a new unread row after pause or terminal transition. pub(crate) fn insert_if_run_running( params: InsertInboxParams, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs index e8cbaae534..c78a03b3f6 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs @@ -640,7 +640,7 @@ impl AgentOrgPlanApprovalStore { OR EXISTS ( SELECT 1 FROM agent_org_runs run WHERE run.id=approval.org_run_id - AND run.status IN ('completed','failed','cancelled','abandoned') + AND run.status IN ('failed','archived') ) )", ) @@ -667,7 +667,7 @@ impl AgentOrgPlanApprovalStore { OR EXISTS ( SELECT 1 FROM agent_org_runs run WHERE run.id=agent_org_plan_approvals.org_run_id - AND run.status IN ('completed','failed','cancelled','abandoned') + AND run.status IN ('failed','archived') ) )", params![ diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs index aa266e7e16..a3fbd9bb73 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs @@ -49,14 +49,18 @@ pub(super) fn load_by_id(run_id: &str) -> SqliteResult org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE id = ?1 LIMIT 1", @@ -78,14 +82,18 @@ pub(super) fn load_by_root_session( org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id = ?1 ORDER BY created_at DESC @@ -121,14 +129,18 @@ pub(super) fn row_to_run(row: &rusqlite::Row<'_>) -> SqliteResult(8)? != 0, + work_item_id: row.get(9)?, + project_slug: row.get(10)?, + routine_fire_id: row.get(11)?, + summary: row.get(12)?, + last_error: row.get(13)?, + failure_json: row.get(14)?, + last_activity_outcome: row.get(15)?, + created_at: row.get(16)?, + updated_at: row.get(17)?, + idled_at: row.get(18)?, }) } @@ -204,15 +216,19 @@ pub(super) fn insert_run(conn: &Connection, run: &AgentOrgRunRecord) -> SqliteRe org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", + idled_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", params![ &run.id, &run.org_id, @@ -221,14 +237,18 @@ pub(super) fn insert_run(conn: &Connection, run: &AgentOrgRunRecord) -> SqliteRe run.org_snapshot_json.as_deref(), run.entry_mode.as_str(), run.status.as_str(), + run.activation_generation, + i64::from(run.has_initial_work), run.work_item_id.as_deref(), run.project_slug.as_deref(), run.routine_fire_id.as_deref(), run.summary.as_deref(), run.last_error.as_deref(), + run.failure_json.as_deref(), + run.last_activity_outcome.as_deref(), &run.created_at, &run.updated_at, - run.completed_at.as_deref(), + run.idled_at.as_deref(), ], )?; Ok(()) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs new file mode 100644 index 0000000000..18c517334a --- /dev/null +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs @@ -0,0 +1,378 @@ +//! Durable certificates for the one-time construction of an Agent Org Team. +//! +//! These rows describe stable identities, not live Provider runtimes. A +//! restart retries the same `(member_id, agent_id, session_id)` intent and can +//! therefore never mint a second identity for the same Team generation. + +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgMaterializationAuthority { + Starting, + Formal, + UserDirected, +} + +impl AgentOrgMaterializationAuthority { + pub fn as_str(self) -> &'static str { + match self { + Self::Starting => "starting", + Self::Formal => "formal", + Self::UserDirected => "user_directed", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "starting" => Self::Starting, + "formal" => Self::Formal, + "user_directed" => Self::UserDirected, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgMaterializationStatus { + Pending, + Succeeded, + Failed, +} + +impl AgentOrgMaterializationStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Succeeded => "succeeded", + Self::Failed => "failed", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "pending" => Self::Pending, + "succeeded" => Self::Succeeded, + "failed" => Self::Failed, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgMaterializationIntent { + pub org_run_id: String, + pub member_id: String, + pub agent_id: String, + pub generation: i64, + pub session_id: String, + pub authority: AgentOrgMaterializationAuthority, + pub status: AgentOrgMaterializationStatus, + pub error_code: Option, + pub error_json: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateAgentOrgMaterializationIntent { + pub member_id: String, + pub agent_id: String, + pub session_id: String, + pub succeeded: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgInitialInputStatus { + PendingPersistence, + Queued, + Dispatched, +} + +impl AgentOrgInitialInputStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::PendingPersistence => "pending_persistence", + Self::Queued => "queued", + Self::Dispatched => "dispatched", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "pending_persistence" => Self::PendingPersistence, + "queued" => Self::Queued, + "dispatched" => Self::Dispatched, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgInitialInput { + pub org_run_id: String, + pub turn_intent_id: String, + pub message_id: String, + pub content: String, + /// Canonical launch-time attachments/context required to replay this exact + /// accepted input after a crash. The launch owner validates and decodes + /// the versioned JSON; the lifecycle store treats it as opaque evidence. + pub payload_json: String, + pub status: AgentOrgInitialInputStatus, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateAgentOrgInitialInput { + pub turn_intent_id: String, + pub message_id: String, + pub content: String, + pub payload_json: String, +} + +pub(super) fn init_schema(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS agent_org_member_materializations ( + org_run_id TEXT NOT NULL, + member_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + generation INTEGER NOT NULL CHECK(generation >= 1), + session_id TEXT NOT NULL, + authority_class TEXT NOT NULL CHECK(authority_class IN ( + 'starting', 'formal', 'user_directed' + )), + status TEXT NOT NULL CHECK(status IN ( + 'pending', 'succeeded', 'failed' + )), + error_code TEXT, + error_json TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY(org_run_id, member_id, generation), + UNIQUE(org_run_id, session_id), + FOREIGN KEY(org_run_id) REFERENCES agent_org_runs(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_agent_org_materializations_pending + ON agent_org_member_materializations(status, org_run_id, generation); + + CREATE TABLE IF NOT EXISTS agent_org_initial_inputs ( + org_run_id TEXT PRIMARY KEY, + turn_intent_id TEXT NOT NULL, + message_id TEXT NOT NULL, + content TEXT NOT NULL, + payload_json TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ( + 'pending_persistence', 'queued', 'dispatched' + )), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(turn_intent_id), + UNIQUE(message_id), + FOREIGN KEY(org_run_id) REFERENCES agent_org_runs(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_agent_org_initial_inputs_dispatch + ON agent_org_initial_inputs(status, org_run_id);", + ) +} + +pub(super) fn insert_materialization_intent( + conn: &Connection, + org_run_id: &str, + generation: i64, + intent: &CreateAgentOrgMaterializationIntent, + now: &str, +) -> Result<(), String> { + let status = if intent.succeeded { + AgentOrgMaterializationStatus::Succeeded + } else { + AgentOrgMaterializationStatus::Pending + }; + conn.execute( + "INSERT INTO agent_org_member_materializations ( + org_run_id, member_id, agent_id, generation, session_id, + authority_class, status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, 'starting', ?6, ?7, ?7)", + params![ + org_run_id, + &intent.member_id, + &intent.agent_id, + generation, + &intent.session_id, + status.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +pub(super) fn insert_initial_input( + conn: &Connection, + org_run_id: &str, + input: &CreateAgentOrgInitialInput, + now: &str, +) -> Result<(), String> { + conn.execute( + "INSERT INTO agent_org_initial_inputs ( + org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)", + params![ + org_run_id, + &input.turn_intent_id, + &input.message_id, + &input.content, + &input.payload_json, + AgentOrgInitialInputStatus::PendingPersistence.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +pub(super) fn list_materializations_with_connection( + conn: &Connection, + org_run_id: &str, +) -> Result, String> { + let mut statement = conn + .prepare( + "SELECT org_run_id, member_id, agent_id, generation, session_id, + authority_class, status, error_code, error_json, + created_at, updated_at + FROM agent_org_member_materializations + WHERE org_run_id=?1 + ORDER BY member_id ASC", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([org_run_id], row_to_materialization) + .map_err(|error| error.to_string())?; + rows.collect::, _>>() + .map_err(|error| error.to_string()) +} + +pub(super) fn load_initial_input_with_connection( + conn: &Connection, + org_run_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + FROM agent_org_initial_inputs WHERE org_run_id=?1", + [org_run_id], + row_to_initial_input, + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(super) fn load_initial_input_by_turn_with_connection( + conn: &Connection, + turn_intent_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + FROM agent_org_initial_inputs WHERE turn_intent_id=?1", + [turn_intent_id], + row_to_initial_input, + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(super) fn list_recoverable_initial_inputs_with_connection( + conn: &Connection, + limit: usize, +) -> Result, String> { + if limit == 0 { + return Ok(Vec::new()); + } + let limit = i64::try_from(limit) + .map_err(|_| format!("initial input recovery limit is too large: {limit}"))?; + let mut statement = conn + .prepare( + "SELECT initial.org_run_id, initial.turn_intent_id, + initial.message_id, initial.content, initial.payload_json, + initial.status, + initial.created_at, initial.updated_at + FROM agent_org_initial_inputs initial + JOIN agent_org_runs run ON run.id=initial.org_run_id + JOIN session_turn_intents turn + ON turn.org_run_id=initial.org_run_id + AND turn.turn_intent_id=initial.turn_intent_id + WHERE run.status='running' + AND initial.status IN ('queued', 'dispatched') + AND turn.status='queued' + ORDER BY initial.updated_at ASC, initial.org_run_id ASC + LIMIT ?1", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([limit], row_to_initial_input) + .map_err(|error| error.to_string())?; + rows.collect::, _>>() + .map_err(|error| error.to_string()) +} + +fn row_to_materialization( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + let authority_raw: String = row.get(5)?; + let status_raw: String = row.get(6)?; + let authority = AgentOrgMaterializationAuthority::parse(&authority_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 5, + rusqlite::types::Type::Text, + format!("unknown materialization authority: {authority_raw:?}").into(), + ) + })?; + let status = AgentOrgMaterializationStatus::parse(&status_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 6, + rusqlite::types::Type::Text, + format!("unknown materialization status: {status_raw:?}").into(), + ) + })?; + Ok(AgentOrgMaterializationIntent { + org_run_id: row.get(0)?, + member_id: row.get(1)?, + agent_id: row.get(2)?, + generation: row.get(3)?, + session_id: row.get(4)?, + authority, + status, + error_code: row.get(7)?, + error_json: row.get(8)?, + created_at: row.get(9)?, + updated_at: row.get(10)?, + }) +} + +fn row_to_initial_input(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let status_raw: String = row.get(5)?; + let status = AgentOrgInitialInputStatus::parse(&status_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 5, + rusqlite::types::Type::Text, + format!("unknown initial input status: {status_raw:?}").into(), + ) + })?; + Ok(AgentOrgInitialInput { + org_run_id: row.get(0)?, + turn_intent_id: row.get(1)?, + message_id: row.get(2)?, + content: row.get(3)?, + payload_json: row.get(4)?, + status, + created_at: row.get(6)?, + updated_at: row.get(7)?, + }) +} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs index 5e2a0851a5..6ac6ff116f 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs @@ -3,23 +3,33 @@ //! A run records that an Agent Org launched through the normal Rust session //! stack, while the root session remains the transcript source of truth. -mod finality; mod helpers; +mod materialization; mod progress; +mod quiescence; +mod rollout; mod store; mod worker; #[cfg(test)] mod tests; -pub(crate) use finality::guaranteed_current_turn_effects_with_connection; -pub use finality::{ - AgentOrgFinalityAssessment, AgentOrgFinalityBlocker, AgentOrgFinalityDecision, - AgentOrgFinalityFacts, AgentOrgFinalityProjection, AgentOrgFinalitySessionFact, - AgentOrgGuaranteedTurnEffects, +pub use materialization::{ + AgentOrgInitialInput, AgentOrgInitialInputStatus, AgentOrgMaterializationAuthority, + AgentOrgMaterializationIntent, AgentOrgMaterializationStatus, CreateAgentOrgInitialInput, + CreateAgentOrgMaterializationIntent, }; pub(crate) use progress::bump_work_revision_in_tx; pub use progress::AgentOrgRunProgress; +pub(crate) use quiescence::guaranteed_current_turn_effects_with_connection; +pub use quiescence::{ + AgentOrgGuaranteedTurnEffects, AgentOrgQuiescenceAssessment, AgentOrgQuiescenceBlocker, + AgentOrgQuiescenceDecision, AgentOrgQuiescenceFacts, AgentOrgQuiescenceProjection, + AgentOrgQuiescenceSessionFact, +}; +pub use rollout::{ + is_enabled as agent_org_redesign_enabled, require_enabled as require_agent_org_redesign, +}; pub use store::AgentOrgRunStore; pub(crate) use worker::recovery_dispatch_recipient_is_available; pub use worker::{WorkerSessionInfo, WorkerSessionRuntime}; @@ -62,50 +72,40 @@ impl std::fmt::Display for AgentOrgRunEntryMode { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AgentOrgRunStatus { + Starting, Running, - /// User-initiated pause. Non-terminal: the run can be resumed via - /// `AgentOrgRunStore::mark_resumed`. Polling and member switching remain - /// available while paused; the coordinator and members simply stop - /// receiving new dispatch until resumed. + /// Reserved non-terminal user-pause state. PR1 freezes the canonical enum + /// but deliberately does not define Pause/Resume handoff behavior; Paused + /// Teams are not fallback-polled. Paused, - Completed, + Idle, Failed, - Cancelled, - Abandoned, + Archived, } impl AgentOrgRunStatus { pub fn as_str(self) -> &'static str { match self { + Self::Starting => "starting", Self::Running => "running", Self::Paused => "paused", - Self::Completed => "completed", + Self::Idle => "idle", Self::Failed => "failed", - Self::Cancelled => "cancelled", - Self::Abandoned => "abandoned", + Self::Archived => "archived", } } pub fn parse(value: &str) -> Option { match value { + "starting" => Some(Self::Starting), "running" => Some(Self::Running), "paused" => Some(Self::Paused), - "completed" => Some(Self::Completed), + "idle" => Some(Self::Idle), "failed" => Some(Self::Failed), - "cancelled" => Some(Self::Cancelled), - "abandoned" => Some(Self::Abandoned), + "archived" => Some(Self::Archived), _ => None, } } - - /// Whether this status represents a terminal state (no further transitions - /// possible). `Paused` is explicitly non-terminal. - pub fn is_terminal(self) -> bool { - matches!( - self, - Self::Completed | Self::Failed | Self::Cancelled | Self::Abandoned - ) - } } impl std::fmt::Display for AgentOrgRunStatus { @@ -399,14 +399,18 @@ pub struct AgentOrgRunRecord { pub org_snapshot_json: Option, pub entry_mode: AgentOrgRunEntryMode, pub status: AgentOrgRunStatus, + pub activation_generation: i64, + pub has_initial_work: bool, pub work_item_id: Option, pub project_slug: Option, pub routine_fire_id: Option, pub summary: Option, pub last_error: Option, + pub failure_json: Option, + pub last_activity_outcome: Option, pub created_at: String, pub updated_at: String, - pub completed_at: Option, + pub idled_at: Option, } #[derive(Debug, Clone)] @@ -422,6 +426,36 @@ pub struct CreateAgentOrgRunParams { pub routine_fire_id: Option, } +#[derive(Debug, Clone)] +pub struct CreateStartingAgentOrgRunParams { + pub org_id: String, + pub coordinator_agent_id: String, + pub root_session_id: String, + pub org_snapshot: OrgDefinition, + pub entry_mode: AgentOrgRunEntryMode, + pub work_item_id: Option, + pub project_slug: Option, + pub routine_fire_id: Option, + pub materialization_intents: Vec, + pub initial_input: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgStartingFailure { + pub code: String, + pub message: String, +} + +impl AgentOrgStartingFailure { + pub fn new(code: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + message: message.into(), + } + } +} + /// Initialize runtime Agent Org tables in `sessions.db`. pub fn init_schema(conn: &Connection) -> SqliteResult<()> { conn.execute_batch( @@ -432,15 +466,25 @@ pub fn init_schema(conn: &Connection) -> SqliteResult<()> { root_session_id TEXT, org_snapshot_json TEXT, entry_mode TEXT NOT NULL, - status TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ( + 'starting', 'running', 'paused', 'idle', 'failed', 'archived' + )), + activation_generation INTEGER NOT NULL DEFAULT 1 + CHECK(activation_generation >= 1), + has_initial_work INTEGER NOT NULL DEFAULT 0 + CHECK(has_initial_work IN (0, 1)), work_item_id TEXT, project_slug TEXT, routine_fire_id TEXT, summary TEXT, last_error TEXT, + failure_json TEXT, + last_activity_outcome TEXT CHECK(last_activity_outcome IN ( + 'completed', 'failed', 'cancelled' + )), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - completed_at TEXT + idled_at TEXT ); CREATE INDEX IF NOT EXISTS idx_agent_org_runs_org_updated ON agent_org_runs(org_id, updated_at); @@ -451,6 +495,7 @@ pub fn init_schema(conn: &Connection) -> SqliteResult<()> { CREATE INDEX IF NOT EXISTS idx_agent_org_runs_status ON agent_org_runs(status);", )?; + materialization::init_schema(conn)?; progress::init_schema(conn)?; Ok(()) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs index ed25aad591..933b860817 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs @@ -1,7 +1,7 @@ //! Monotonic Agent Org work observation and explicit completion intent. //! //! Timestamps are useful for display but are not a safe concurrency token. -//! This table records a small monotonic revision so finality can prove that a +//! This table records a small monotonic revision so Quiescence can prove that a //! coordinator turn was presented with (and successfully observed) the latest //! durable task mutation before announcing completion. diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs similarity index 69% rename from src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs rename to src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs index bce23aa776..2e70a7d234 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs @@ -1,4 +1,4 @@ -//! Canonical Agent Org finality facts and decision policy. +//! Canonical Agent Org formal-work quiescence facts and decision policy. //! //! Every caller (watchdog inspection, lifecycle reconciliation, completion //! snapshots) must reason from this same typed assessment. This prevents a @@ -16,7 +16,7 @@ use super::{AgentOrgRunStatus, AgentOrgRunStore}; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalitySessionFact { +pub struct AgentOrgQuiescenceSessionFact { pub session_id: String, pub member_id: Option, pub status: SessionStatus, @@ -24,11 +24,12 @@ pub struct AgentOrgFinalitySessionFact { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityFacts { +pub struct AgentOrgQuiescenceFacts { pub run_status: Option, + pub activation_generation: Option, pub root_session_id: Option, pub root_status: Option, - pub worker_sessions: Vec, + pub worker_sessions: Vec, pub task_count: usize, pub unresolved_task_count: usize, pub corrupt_task_count: usize, @@ -38,11 +39,14 @@ pub struct AgentOrgFinalityFacts { pub unread_inbox_count: usize, pub active_intervention_member_ids: Vec, pub in_flight_turn_intent_count: usize, + pub unknown_turn_intent_count: usize, + pub pending_formal_materialization_count: usize, + pub active_recovery_reservation_count: usize, pub pending_plan_approval_count: usize, pub progress: Option, } -impl AgentOrgFinalityFacts { +impl AgentOrgQuiescenceFacts { /// Canonical set of non-quiescent worker member ids for UI/task /// projections. Keeping the status classification here prevents Run View, /// task_list, and the reconciler from growing subtly different ideas of @@ -51,7 +55,7 @@ impl AgentOrgFinalityFacts { let mut member_ids = self .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .map(|session| { session .member_id @@ -67,15 +71,14 @@ impl AgentOrgFinalityFacts { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] -pub enum AgentOrgFinalityDecision { - KeepRunning, - Complete, - Abandon, +pub enum AgentOrgQuiescenceDecision { + KeepWorking, + Quiescent, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case")] -pub enum AgentOrgFinalityBlocker { +pub enum AgentOrgQuiescenceBlocker { RunMissing, RunNotRunning { status: AgentOrgRunStatus, @@ -108,6 +111,15 @@ pub enum AgentOrgFinalityBlocker { InFlightTurnIntents { count: usize, }, + UnknownTurnIntents { + count: usize, + }, + PendingFormalMaterializations { + count: usize, + }, + ActiveRecoveryReservations { + count: usize, + }, PendingPlanApprovals { count: usize, }, @@ -116,7 +128,7 @@ pub enum AgentOrgFinalityBlocker { /// retained facts disagree with the invariants that normally gate that /// transition. This is diagnostic state for repair/audit surfaces, not a /// request to mutate the run back to Running. - TerminalStateInconsistent { + QuietStateInconsistent { status: AgentOrgRunStatus, root_session_missing: bool, active_session_count: usize, @@ -125,29 +137,32 @@ pub enum AgentOrgFinalityBlocker { unread_inbox_count: usize, active_intervention_count: usize, in_flight_turn_intent_count: usize, + unknown_turn_intent_count: usize, + pending_formal_materialization_count: usize, + active_recovery_reservation_count: usize, pending_plan_approval_count: usize, }, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityAssessment { - pub facts: AgentOrgFinalityFacts, - pub decision: AgentOrgFinalityDecision, - pub blockers: Vec, +pub struct AgentOrgQuiescenceAssessment { + pub facts: AgentOrgQuiescenceFacts, + pub decision: AgentOrgQuiescenceDecision, + pub blockers: Vec, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityProjection { - pub decision: AgentOrgFinalityDecision, - pub blockers: Vec, +pub struct AgentOrgQuiescenceProjection { + pub decision: AgentOrgQuiescenceDecision, + pub blockers: Vec, } /// Exact effects that the currently executing coordinator turn will commit /// if (and only if) that turn succeeds. These counts are not caller hints: /// they are revalidated from durable intent/materialization rows inside the -/// same read transaction as the finality snapshot. +/// same read transaction as the quiescence snapshot. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AgentOrgGuaranteedTurnEffects { pub current_coordinator_turn: bool, @@ -155,7 +170,7 @@ pub struct AgentOrgGuaranteedTurnEffects { pub unread_inbox_rows: usize, } -impl AgentOrgFinalityAssessment { +impl AgentOrgQuiescenceAssessment { /// Canonical prospective certificate used by the coordinator inside its /// current turn. It answers one narrow question: "if this coordinator /// turn succeeds now, will the strict reconciler be able to complete?" @@ -164,7 +179,7 @@ impl AgentOrgFinalityAssessment { /// the root session becomes quiescent, and the revision staged into this /// prompt becomes observed. Every worker, task, inbox, approval, /// intervention, corruption, and turn-intent blocker remains unchanged. - pub fn after_successful_coordinator_turn(&self) -> AgentOrgFinalityProjection { + pub fn after_successful_coordinator_turn(&self) -> AgentOrgQuiescenceProjection { self.after_successful_coordinator_turn_with_effects(AgentOrgGuaranteedTurnEffects { current_coordinator_turn: true, ..AgentOrgGuaranteedTurnEffects::default() @@ -174,11 +189,11 @@ impl AgentOrgFinalityAssessment { pub fn after_successful_coordinator_turn_with_effects( &self, effects: AgentOrgGuaranteedTurnEffects, - ) -> AgentOrgFinalityProjection { + ) -> AgentOrgQuiescenceProjection { if self.facts.run_status != Some(AgentOrgRunStatus::Running) || !effects.current_coordinator_turn { - return AgentOrgFinalityProjection { + return AgentOrgQuiescenceProjection { decision: self.decision, blockers: self.blockers.clone(), }; @@ -190,30 +205,30 @@ impl AgentOrgFinalityAssessment { let mut blockers = Vec::new(); for blocker in &self.blockers { match blocker { - AgentOrgFinalityBlocker::SessionsActive { session_ids } => { + AgentOrgQuiescenceBlocker::SessionsActive { session_ids } => { let remaining = session_ids .iter() .filter(|session_id| Some(session_id.as_str()) != root_session_id) .cloned() .collect::>(); if !remaining.is_empty() { - blockers.push(AgentOrgFinalityBlocker::SessionsActive { + blockers.push(AgentOrgQuiescenceBlocker::SessionsActive { session_ids: remaining, }); } } - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } if presented_current_revision => {} - AgentOrgFinalityBlocker::UnreadInbox { count } => { + AgentOrgQuiescenceBlocker::UnreadInbox { count } => { let remaining = count.saturating_sub(effects.unread_inbox_rows); if remaining > 0 { - blockers.push(AgentOrgFinalityBlocker::UnreadInbox { count: remaining }); + blockers.push(AgentOrgQuiescenceBlocker::UnreadInbox { count: remaining }); } } - AgentOrgFinalityBlocker::InFlightTurnIntents { count } => { + AgentOrgQuiescenceBlocker::InFlightTurnIntents { count } => { let remaining = count.saturating_sub(effects.in_flight_turn_intents); if remaining > 0 { - blockers.push(AgentOrgFinalityBlocker::InFlightTurnIntents { + blockers.push(AgentOrgQuiescenceBlocker::InFlightTurnIntents { count: remaining, }); } @@ -221,11 +236,11 @@ impl AgentOrgFinalityAssessment { other => blockers.push(other.clone()), } } - AgentOrgFinalityProjection { + AgentOrgQuiescenceProjection { decision: if blockers.is_empty() { - AgentOrgFinalityDecision::Complete + AgentOrgQuiescenceDecision::Quiescent } else { - AgentOrgFinalityDecision::KeepRunning + AgentOrgQuiescenceDecision::KeepWorking }, blockers, } @@ -311,18 +326,20 @@ pub(crate) fn guaranteed_current_turn_effects_with_connection( pub(super) fn load_and_assess( conn: &Connection, run_id: &str, -) -> Result { - let run_row: Option<(String, Option)> = conn +) -> Result { + let run_row: Option<(String, Option, i64)> = conn .query_row( - "SELECT status, root_session_id FROM agent_org_runs WHERE id=?1", + "SELECT status, root_session_id, activation_generation + FROM agent_org_runs WHERE id=?1", params![run_id], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .optional() .map_err(|err| err.to_string())?; - let Some((run_status_raw, root_session_id)) = run_row else { - return Ok(assess(AgentOrgFinalityFacts { + let Some((run_status_raw, root_session_id, activation_generation)) = run_row else { + return Ok(assess_quiescence(AgentOrgQuiescenceFacts { run_status: None, + activation_generation: None, root_session_id: None, root_status: None, worker_sessions: Vec::new(), @@ -335,6 +352,9 @@ pub(super) fn load_and_assess( unread_inbox_count: 0, active_intervention_member_ids: Vec::new(), in_flight_turn_intent_count: 0, + unknown_turn_intent_count: 0, + pending_formal_materialization_count: 0, + active_recovery_reservation_count: 0, pending_plan_approval_count: 0, progress: None, })); @@ -361,12 +381,12 @@ pub(super) fn load_and_assess( // Use the same cross-transport canonical worker projection as Run View // and recovery. Duplicating the Rust/CLI queries here used to let a stale - // session for the same member block finality even though the UI and + // session for the same member block quiescence even though the UI and // watchdog correctly selected the freshest one. let worker_sessions = AgentOrgRunStore::list_descendant_worker_sessions_with_connection(conn, run_id)? .into_iter() - .map(|session| AgentOrgFinalitySessionFact { + .map(|session| AgentOrgQuiescenceSessionFact { session_id: session.session_id, member_id: session.member_id, status: session.status, @@ -478,6 +498,36 @@ pub(super) fn load_and_assess( |row| row.get(0), ) .map_err(|err| err.to_string())?; + let unknown_turn_intent_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM session_turn_intents + WHERE org_run_id=?1 + AND status NOT IN ( + 'optimistic', 'queued', 'running', 'completed', 'failed', + 'cancelled', 'stale', 'coalesced', 'rejected' + )", + params![run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + let pending_formal_materialization_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_member_materializations + WHERE org_run_id=?1 AND generation=?2 + AND authority_class IN ('starting', 'formal') + AND status<>'succeeded'", + params![run_id, activation_generation], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + let active_recovery_reservation_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_recovery_attempts + WHERE org_run_id=?1 AND reservation_token IS NOT NULL", + params![run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; let pending_plan_approval_count: i64 = conn .query_row( "SELECT COUNT(*) FROM agent_org_plan_approvals @@ -487,8 +537,9 @@ pub(super) fn load_and_assess( ) .map_err(|err| err.to_string())?; - Ok(assess(AgentOrgFinalityFacts { + Ok(assess_quiescence(AgentOrgQuiescenceFacts { run_status: Some(run_status), + activation_generation: Some(activation_generation), root_session_id, root_status, worker_sessions, @@ -504,6 +555,18 @@ pub(super) fn load_and_assess( "in-flight turn intent", in_flight_turn_intent_count, )?, + unknown_turn_intent_count: count_to_usize( + "unknown turn intent", + unknown_turn_intent_count, + )?, + pending_formal_materialization_count: count_to_usize( + "pending formal materialization", + pending_formal_materialization_count, + )?, + active_recovery_reservation_count: count_to_usize( + "active recovery reservation", + active_recovery_reservation_count, + )?, pending_plan_approval_count: count_to_usize( "pending plan approval", pending_plan_approval_count, @@ -512,49 +575,42 @@ pub(super) fn load_and_assess( })) } -pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment { +pub fn assess_quiescence(facts: AgentOrgQuiescenceFacts) -> AgentOrgQuiescenceAssessment { let mut blockers = Vec::new(); let Some(run_status) = facts.run_status else { - blockers.push(AgentOrgFinalityBlocker::RunMissing); - return AgentOrgFinalityAssessment { + blockers.push(AgentOrgQuiescenceBlocker::RunMissing); + return AgentOrgQuiescenceAssessment { + decision: AgentOrgQuiescenceDecision::KeepWorking, facts, - decision: AgentOrgFinalityDecision::KeepRunning, blockers, }; }; - if run_status == AgentOrgRunStatus::Completed { - if let Some(inconsistency) = terminal_state_inconsistency(&facts, run_status) { + if run_status == AgentOrgRunStatus::Idle { + if let Some(inconsistency) = quiet_state_inconsistency(&facts, run_status) { blockers.push(inconsistency); } - return AgentOrgFinalityAssessment { + return AgentOrgQuiescenceAssessment { facts, - decision: AgentOrgFinalityDecision::Complete, - blockers, - }; - } - if run_status == AgentOrgRunStatus::Abandoned { - return AgentOrgFinalityAssessment { - facts, - decision: AgentOrgFinalityDecision::Abandon, + decision: AgentOrgQuiescenceDecision::Quiescent, blockers, }; } if run_status != AgentOrgRunStatus::Running { - blockers.push(AgentOrgFinalityBlocker::RunNotRunning { status: run_status }); - return AgentOrgFinalityAssessment { + blockers.push(AgentOrgQuiescenceBlocker::RunNotRunning { status: run_status }); + return AgentOrgQuiescenceAssessment { facts, - decision: AgentOrgFinalityDecision::KeepRunning, + decision: AgentOrgQuiescenceDecision::KeepWorking, blockers, }; } if facts.root_session_id.is_none() || facts.root_status.is_none() { - blockers.push(AgentOrgFinalityBlocker::RootSessionMissing); + blockers.push(AgentOrgQuiescenceBlocker::RootSessionMissing); } let mut active_session_ids = Vec::new(); if facts .root_status - .is_some_and(|status| !session_is_quiescent_for_completed_run(status)) + .is_some_and(|status| !session_is_quiescent(status)) { if let Some(root_session_id) = facts.root_session_id.as_ref() { active_session_ids.push(root_session_id.clone()); @@ -564,55 +620,71 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment facts .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .map(|session| session.session_id.clone()), ); if !active_session_ids.is_empty() { - blockers.push(AgentOrgFinalityBlocker::SessionsActive { + blockers.push(AgentOrgQuiescenceBlocker::SessionsActive { session_ids: active_session_ids, }); } if facts.unresolved_task_count > 0 { - blockers.push(AgentOrgFinalityBlocker::OpenTasks { + blockers.push(AgentOrgQuiescenceBlocker::OpenTasks { count: facts.unresolved_task_count, }); } if facts.corrupt_task_count > 0 { - blockers.push(AgentOrgFinalityBlocker::CorruptTaskData { + blockers.push(AgentOrgQuiescenceBlocker::CorruptTaskData { count: facts.corrupt_task_count, }); } if facts.unread_inbox_count > 0 { - blockers.push(AgentOrgFinalityBlocker::UnreadInbox { + blockers.push(AgentOrgQuiescenceBlocker::UnreadInbox { count: facts.unread_inbox_count, }); } if !facts.active_intervention_member_ids.is_empty() { - blockers.push(AgentOrgFinalityBlocker::ActiveInterventions { + blockers.push(AgentOrgQuiescenceBlocker::ActiveInterventions { count: facts.active_intervention_member_ids.len(), }); } if facts.in_flight_turn_intent_count > 0 { - blockers.push(AgentOrgFinalityBlocker::InFlightTurnIntents { + blockers.push(AgentOrgQuiescenceBlocker::InFlightTurnIntents { count: facts.in_flight_turn_intent_count, }); } + if facts.unknown_turn_intent_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::UnknownTurnIntents { + count: facts.unknown_turn_intent_count, + }); + } + if facts.pending_formal_materialization_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::PendingFormalMaterializations { + count: facts.pending_formal_materialization_count, + }); + } + if facts.active_recovery_reservation_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { + count: facts.active_recovery_reservation_count, + }); + } if facts.pending_plan_approval_count > 0 { - blockers.push(AgentOrgFinalityBlocker::PendingPlanApprovals { + blockers.push(AgentOrgQuiescenceBlocker::PendingPlanApprovals { count: facts.pending_plan_approval_count, }); } match facts.progress.as_ref() { - None => blockers.push(AgentOrgFinalityBlocker::ProgressStateMissing), + None => blockers.push(AgentOrgQuiescenceBlocker::ProgressStateMissing), Some(progress) => { if facts.task_count == 0 { if !progress.completion_requested { - blockers.push(AgentOrgFinalityBlocker::EmptyTaskBoardRequiresCompletionIntent); + blockers + .push(AgentOrgQuiescenceBlocker::EmptyTaskBoardRequiresCompletionIntent); } else if progress.completion_requested_work_revision != Some(progress.work_revision) { - blockers.push(AgentOrgFinalityBlocker::StaleCompletionIntent { + blockers.push(AgentOrgQuiescenceBlocker::StaleCompletionIntent { requested_work_revision: progress.completion_requested_work_revision, current_work_revision: progress.work_revision, }); @@ -620,7 +692,7 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment } if progress.coordinator_observed_work_revision < Some(progress.work_revision) { blockers.push( - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { observed_work_revision: progress.coordinator_observed_work_revision, current_work_revision: progress.work_revision, }, @@ -629,41 +701,31 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment } } - let coordinator_is_permanently_unavailable = facts.root_status == Some(SessionStatus::Archived); - let every_worker_is_permanently_unavailable = facts - .worker_sessions - .iter() - .all(|session| session.status == SessionStatus::Archived); - let decision = if facts.unresolved_task_count > 0 - && coordinator_is_permanently_unavailable - && every_worker_is_permanently_unavailable - { - AgentOrgFinalityDecision::Abandon - } else if blockers.is_empty() { - AgentOrgFinalityDecision::Complete + let decision = if blockers.is_empty() { + AgentOrgQuiescenceDecision::Quiescent } else { - AgentOrgFinalityDecision::KeepRunning + AgentOrgQuiescenceDecision::KeepWorking }; - AgentOrgFinalityAssessment { + AgentOrgQuiescenceAssessment { facts, decision, blockers, } } -fn terminal_state_inconsistency( - facts: &AgentOrgFinalityFacts, +fn quiet_state_inconsistency( + facts: &AgentOrgQuiescenceFacts, status: AgentOrgRunStatus, -) -> Option { +) -> Option { let root_session_missing = facts.root_session_id.is_none() || facts.root_status.is_none(); let active_session_count = usize::from( facts .root_status - .is_some_and(|session| !session_is_quiescent_for_completed_run(session)), + .is_some_and(|session| !session_is_quiescent(session)), ) + facts .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .count(); let inconsistent = root_session_missing || active_session_count > 0 @@ -672,8 +734,11 @@ fn terminal_state_inconsistency( || facts.unread_inbox_count > 0 || !facts.active_intervention_member_ids.is_empty() || facts.in_flight_turn_intent_count > 0 + || facts.unknown_turn_intent_count > 0 + || facts.pending_formal_materialization_count > 0 + || facts.active_recovery_reservation_count > 0 || facts.pending_plan_approval_count > 0; - inconsistent.then_some(AgentOrgFinalityBlocker::TerminalStateInconsistent { + inconsistent.then_some(AgentOrgQuiescenceBlocker::QuietStateInconsistent { status, root_session_missing, active_session_count, @@ -682,11 +747,14 @@ fn terminal_state_inconsistency( unread_inbox_count: facts.unread_inbox_count, active_intervention_count: facts.active_intervention_member_ids.len(), in_flight_turn_intent_count: facts.in_flight_turn_intent_count, + unknown_turn_intent_count: facts.unknown_turn_intent_count, + pending_formal_materialization_count: facts.pending_formal_materialization_count, + active_recovery_reservation_count: facts.active_recovery_reservation_count, pending_plan_approval_count: facts.pending_plan_approval_count, }) } -pub(super) fn session_is_quiescent_for_completed_run(status: SessionStatus) -> bool { +pub(super) fn session_is_quiescent(status: SessionStatus) -> bool { matches!( status, SessionStatus::Idle @@ -710,12 +778,13 @@ mod tests { fn completed_board_facts( presented_revision: Option, worker_status: SessionStatus, - ) -> AgentOrgFinalityFacts { - AgentOrgFinalityFacts { + ) -> AgentOrgQuiescenceFacts { + AgentOrgQuiescenceFacts { run_status: Some(AgentOrgRunStatus::Running), + activation_generation: Some(1), root_session_id: Some("root".to_string()), root_status: Some(SessionStatus::Running), - worker_sessions: vec![AgentOrgFinalitySessionFact { + worker_sessions: vec![AgentOrgQuiescenceSessionFact { session_id: "worker".to_string(), member_id: Some("member".to_string()), status: worker_status, @@ -729,6 +798,9 @@ mod tests { unread_inbox_count: 0, active_intervention_member_ids: Vec::new(), in_flight_turn_intent_count: 0, + unknown_turn_intent_count: 0, + pending_formal_materialization_count: 0, + active_recovery_reservation_count: 0, pending_plan_approval_count: 0, progress: Some(AgentOrgRunProgress { org_run_id: "run".to_string(), @@ -746,51 +818,57 @@ mod tests { #[test] fn prospective_certificate_allows_current_coordinator_turn_only() { - let assessment = assess(completed_board_facts(Some(2), SessionStatus::Idle)); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::KeepRunning); + let assessment = assess_quiescence(completed_board_facts(Some(2), SessionStatus::Idle)); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::KeepWorking); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(prospective.decision, AgentOrgQuiescenceDecision::Quiescent); assert!(prospective.blockers.is_empty()); } #[test] fn prospective_certificate_rejects_stale_presented_revision() { - let assessment = assess(completed_board_facts(Some(1), SessionStatus::Idle)); + let assessment = assess_quiescence(completed_board_facts(Some(1), SessionStatus::Idle)); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!( + prospective.decision, + AgentOrgQuiescenceDecision::KeepWorking + ); assert!(prospective.blockers.iter().any(|blocker| matches!( blocker, - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } ))); } #[test] fn prospective_certificate_never_hides_active_worker() { - let assessment = assess(completed_board_facts(Some(2), SessionStatus::Running)); + let assessment = assess_quiescence(completed_board_facts(Some(2), SessionStatus::Running)); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!( + prospective.decision, + AgentOrgQuiescenceDecision::KeepWorking + ); assert!(prospective.blockers.iter().any(|blocker| matches!( blocker, - AgentOrgFinalityBlocker::SessionsActive { session_ids } + AgentOrgQuiescenceBlocker::SessionsActive { session_ids } if session_ids == &["worker".to_string()] ))); } #[test] - fn completed_run_stays_terminal_but_reports_inconsistent_retained_facts() { + fn idle_run_stays_quiescent_but_reports_inconsistent_retained_facts() { let mut facts = completed_board_facts(Some(2), SessionStatus::Idle); - facts.run_status = Some(AgentOrgRunStatus::Completed); + facts.run_status = Some(AgentOrgRunStatus::Idle); facts.root_status = Some(SessionStatus::Idle); facts.unresolved_task_count = 1; facts.corrupt_task_count = 1; facts.unread_inbox_count = 2; - let assessment = assess(facts); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::Complete); + let assessment = assess_quiescence(facts); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::Quiescent); assert!(matches!( assessment.blockers.as_slice(), - [AgentOrgFinalityBlocker::TerminalStateInconsistent { - status: AgentOrgRunStatus::Completed, + [AgentOrgQuiescenceBlocker::QuietStateInconsistent { + status: AgentOrgRunStatus::Idle, open_task_count: 1, corrupt_task_count: 1, unread_inbox_count: 2, @@ -798,4 +876,33 @@ mod tests { }] )); } + + #[test] + fn unknown_turn_materialization_and_reservation_fail_closed() { + let mut facts = completed_board_facts(Some(2), SessionStatus::Idle); + facts.root_status = Some(SessionStatus::Idle); + facts + .progress + .as_mut() + .expect("progress") + .coordinator_observed_work_revision = Some(2); + facts.unknown_turn_intent_count = 1; + facts.pending_formal_materialization_count = 1; + facts.active_recovery_reservation_count = 1; + + let assessment = assess_quiescence(facts); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::KeepWorking); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::UnknownTurnIntents { count: 1 } + ))); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::PendingFormalMaterializations { count: 1 } + ))); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { count: 1 } + ))); + } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs new file mode 100644 index 0000000000..9520f6a0ee --- /dev/null +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs @@ -0,0 +1,40 @@ +//! Single internal rollout gate for the long-lived Agent Org redesign. +//! +//! This is deliberately not persisted in Team definitions or exposed to +//! model/tool context. Until the final stack PR changes the default, missing +//! or malformed configuration fails closed. + +const ENABLED_VALUE: &str = "1"; +const ROLLOUT_ENV: &str = "ORGII_AGENT_ORG_REDESIGN"; + +fn configured_enabled(value: Option<&str>, test_build: bool) -> bool { + test_build || value.is_some_and(|value| value.trim() == ENABLED_VALUE) +} + +pub fn is_enabled() -> bool { + let configured = std::env::var(ROLLOUT_ENV).ok(); + configured_enabled(configured.as_deref(), cfg!(test)) +} + +pub fn require_enabled() -> Result<(), String> { + is_enabled().then_some(()).ok_or_else(|| { + "agent_org_redesign_disabled: the long-lived Agent Team lifecycle is not enabled" + .to_string() + }) +} + +#[cfg(test)] +mod tests { + #[test] + fn unit_test_builds_use_the_internal_gate_without_environment_state() { + assert!(super::is_enabled()); + } + + #[test] + fn production_gate_defaults_and_malformed_values_fail_closed() { + assert!(!super::configured_enabled(None, false)); + assert!(!super::configured_enabled(Some("true"), false)); + assert!(!super::configured_enabled(Some("0"), false)); + assert!(super::configured_enabled(Some("1"), false)); + } +} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index c50a58d3d0..b410c81186 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -9,20 +9,26 @@ use crate::definitions::orgs::AgentOrgsStore; use crate::session::SessionStatus; use database::db::{get_connection, with_sessions_writer}; -use super::finality::load_and_assess; use super::helpers::{ context_for_run_record, flatten_members, insert_run, load_by_id, load_by_root_session, parent_session_id_of, row_to_run, validate_entry_mode, validate_status, }; +use super::materialization::{ + insert_initial_input, insert_materialization_intent, list_materializations_with_connection, + list_recoverable_initial_inputs_with_connection, load_initial_input_by_turn_with_connection, + load_initial_input_with_connection, +}; use super::progress::{ ensure_progress_in_conn, load_progress_with_conn, mark_coordinator_observed_revision_with_conn, record_completion_request_in_tx, stage_coordinator_presented_with_conn, }; +use super::quiescence::load_and_assess; use super::worker::{WorkerSessionInfo, WorkerSessionRuntime}; use super::{ - AgentOrgCompletionRequestOutcome, AgentOrgFinalityAssessment, AgentOrgRunContext, - AgentOrgRunProgress, AgentOrgRunRecord, AgentOrgRunStatus, CreateAgentOrgRunParams, - COORDINATOR_MEMBER_ID, + AgentOrgCompletionRequestOutcome, AgentOrgInitialInput, AgentOrgMaterializationIntent, + AgentOrgQuiescenceAssessment, AgentOrgRunContext, AgentOrgRunProgress, AgentOrgRunRecord, + AgentOrgRunStatus, AgentOrgStartingFailure, CreateAgentOrgRunParams, + CreateStartingAgentOrgRunParams, COORDINATOR_MEMBER_ID, }; pub struct AgentOrgRunStore; @@ -62,14 +68,18 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IN ({placeholders}) ORDER BY updated_at DESC, id DESC" @@ -99,14 +109,18 @@ impl AgentOrgRunStore { org_snapshot_json: Some(org_snapshot_json), entry_mode, status, + activation_generation: 1, + has_initial_work: false, work_item_id: params.work_item_id, project_slug: params.project_slug, routine_fire_id: params.routine_fire_id, summary: None, last_error: None, + failure_json: None, + last_activity_outcome: None, created_at: now.clone(), updated_at: now, - completed_at: None, + idled_at: None, }; with_sessions_writer(|| -> Result<(), String> { @@ -122,6 +136,457 @@ impl AgentOrgRunStore { Ok(run) } + /// Create the authoritative Team construction envelope and every stable + /// identity/input intent in one IMMEDIATE transaction. + pub fn create_starting( + params: CreateStartingAgentOrgRunParams, + ) -> Result { + let entry_mode = validate_entry_mode(params.entry_mode.as_str())?; + let org_snapshot_json = serde_json::to_string(¶ms.org_snapshot) + .map_err(|error| format!("failed to serialize Agent Org launch snapshot: {error}"))?; + let now = chrono::Utc::now().to_rfc3339(); + let run = AgentOrgRunRecord { + id: format!("agent-org-run-{}", uuid::Uuid::new_v4()), + org_id: params.org_id, + coordinator_agent_id: params.coordinator_agent_id, + root_session_id: Some(params.root_session_id.clone()), + org_snapshot_json: Some(org_snapshot_json), + entry_mode, + status: AgentOrgRunStatus::Starting, + activation_generation: 1, + has_initial_work: params.initial_input.is_some(), + work_item_id: params.work_item_id, + project_slug: params.project_slug, + routine_fire_id: params.routine_fire_id, + summary: None, + last_error: None, + failure_json: None, + last_activity_outcome: None, + created_at: now.clone(), + updated_at: now.clone(), + idled_at: None, + }; + + let mut member_ids = HashSet::new(); + let mut session_ids = HashSet::new(); + let mut expected_roster = flatten_members(¶ms.org_snapshot.children, None) + .into_iter() + .map(|member| (member.member_id, member.agent_id)) + .collect::>(); + expected_roster.insert( + COORDINATOR_MEMBER_ID.to_string(), + run.coordinator_agent_id.clone(), + ); + for intent in ¶ms.materialization_intents { + if intent.member_id.trim().is_empty() + || intent.agent_id.trim().is_empty() + || intent.session_id.trim().is_empty() + { + return Err("Agent Org materialization intent contains an empty identity".into()); + } + if !member_ids.insert(intent.member_id.clone()) { + return Err(format!( + "duplicate Agent Org materialization member_id: {}", + intent.member_id + )); + } + if !session_ids.insert(intent.session_id.clone()) { + return Err(format!( + "duplicate Agent Org materialization session_id: {}", + intent.session_id + )); + } + if expected_roster.get(&intent.member_id) != Some(&intent.agent_id) { + return Err(format!( + "Agent Org materialization roster does not match the launch snapshot for member {}", + intent.member_id + )); + } + if intent.member_id == COORDINATOR_MEMBER_ID { + if intent.session_id != params.root_session_id || !intent.succeeded { + return Err( + "Agent Org coordinator receipt must certify the canonical root Session" + .into(), + ); + } + } else if intent.succeeded { + return Err(format!( + "Agent Org member {} cannot be pre-certified before materialization", + intent.member_id + )); + } + } + if member_ids.len() != expected_roster.len() + || expected_roster + .keys() + .any(|member_id| !member_ids.contains(member_id)) + { + return Err( + "Agent Org materialization roster does not exactly match the launch snapshot" + .into(), + ); + } + + with_sessions_writer(|| -> Result<(), String> { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let coordinator_identity: Option<(Option, Option, Option)> = + transaction + .query_row( + "SELECT agent_definition_id, org_member_id, parent_session_id + FROM agent_sessions WHERE session_id=?1", + [¶ms.root_session_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + if coordinator_identity + != Some(( + Some(run.coordinator_agent_id.clone()), + Some(COORDINATOR_MEMBER_ID.to_string()), + None, + )) + { + return Err(format!( + "Agent Org coordinator Session identity is missing or mismatched: {}", + params.root_session_id + )); + } + insert_run(&transaction, &run).map_err(|error| error.to_string())?; + for intent in ¶ms.materialization_intents { + insert_materialization_intent( + &transaction, + &run.id, + run.activation_generation, + intent, + &now, + )?; + } + if let Some(input) = params.initial_input.as_ref() { + insert_initial_input(&transaction, &run.id, input, &now)?; + } + ensure_progress_in_conn(&transaction, &run.id)?; + transaction.commit().map_err(|error| error.to_string()) + })?; + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(&run.id); + Ok(run) + } + + pub fn materializations(run_id: &str) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + list_materializations_with_connection(&connection, run_id) + } + + pub fn initial_input(run_id: &str) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + load_initial_input_with_connection(&connection, run_id) + } + + pub fn initial_input_for_turn( + turn_intent_id: &str, + ) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + load_initial_input_by_turn_with_connection(&connection, turn_intent_id) + } + + pub fn recoverable_initial_inputs(limit: usize) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + list_recoverable_initial_inputs_with_connection(&connection, limit) + } + + pub fn load(run_id: &str) -> Result, String> { + load_by_id(run_id).map_err(|error| error.to_string()) + } + + /// Certify one stable member identity after the exact persisted Session + /// row has been read back. A retry of the same receipt is a no-op; a + /// different identity can never satisfy it. + pub fn mark_materialization_succeeded( + run_id: &str, + member_id: &str, + generation: i64, + session_id: &str, + ) -> Result { + with_sessions_writer(|| -> Result { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let receipt: Option<(String, String, String, String)> = transaction + .query_row( + "SELECT materialization.agent_id, materialization.session_id, + run.root_session_id, materialization.status + FROM agent_org_member_materializations materialization + JOIN agent_org_runs run ON run.id=materialization.org_run_id + WHERE materialization.org_run_id=?1 + AND materialization.member_id=?2 + AND materialization.generation=?3 + AND run.status='starting' + AND run.activation_generation=?3", + params![run_id, member_id, generation], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((agent_id, expected_session_id, root_session_id, status)) = receipt else { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(false); + }; + if expected_session_id != session_id { + return Err(format!( + "materialization session mismatch for {run_id}/{member_id}: expected {expected_session_id}, got {session_id}" + )); + } + if status == "succeeded" { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(false); + } + let persisted_identity: Option<(Option, Option, Option)> = + transaction + .query_row( + "SELECT agent_definition_id, org_member_id, parent_session_id + FROM agent_sessions WHERE session_id=?1", + [session_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((persisted_agent_id, persisted_member_id, parent_session_id)) = + persisted_identity + else { + return Err(format!( + "materialized Session {session_id} is missing for {run_id}/{member_id}" + )); + }; + let expected_parent = (member_id != COORDINATOR_MEMBER_ID).then_some(root_session_id); + if persisted_agent_id.as_deref() != Some(agent_id.as_str()) + || persisted_member_id.as_deref() != Some(member_id) + || parent_session_id != expected_parent + { + return Err(format!( + "materialized Session identity mismatch for {run_id}/{member_id}" + )); + } + let changed = transaction + .execute( + "UPDATE agent_org_member_materializations + SET status='succeeded', error_code=NULL, error_json=NULL, + updated_at=?5 + WHERE org_run_id=?1 AND member_id=?2 AND generation=?3 + AND session_id=?4 AND status='pending'", + params![ + run_id, + member_id, + generation, + session_id, + chrono::Utc::now().to_rfc3339() + ], + ) + .map_err(|error| error.to_string())?; + transaction.commit().map_err(|error| error.to_string())?; + Ok(changed == 1) + }) + } + + /// Finish Starting only after the stable roster and initial-input + /// certificate are complete. This is the sole Starting completion owner. + pub fn finish_starting( + run_id: &str, + expected_generation: i64, + ) -> Result { + let status = with_sessions_writer(|| -> Result { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let run: Option<(String, String, i64, bool)> = transaction + .query_row( + "SELECT status, root_session_id, activation_generation, has_initial_work + FROM agent_org_runs WHERE id=?1", + [run_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get::<_, i64>(3)? != 0, + )) + }, + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status_raw, root_session_id, generation, has_initial_work)) = run else { + return Err(format!("agent_org_run_not_found: {run_id}")); + }; + let current = AgentOrgRunStatus::parse(&status_raw) + .ok_or_else(|| format!("unknown Agent Org run status: {status_raw:?}"))?; + if current != AgentOrgRunStatus::Starting { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(current); + } + if generation != expected_generation { + return Err(format!( + "stale Starting generation for {run_id}: expected {expected_generation}, current {generation}" + )); + } + let invalid_materialized_identities: i64 = transaction + .query_row( + "SELECT COUNT(*) + FROM agent_org_member_materializations materialization + LEFT JOIN agent_sessions session + ON session.session_id=materialization.session_id + WHERE materialization.org_run_id=?1 + AND materialization.generation=?2 + AND materialization.status='succeeded' + AND ( + session.session_id IS NULL + OR session.agent_definition_id IS NULL + OR session.agent_definition_id<>materialization.agent_id + OR session.org_member_id IS NULL + OR session.org_member_id<>materialization.member_id + OR ( + materialization.member_id=?3 + AND ( + materialization.session_id<>?4 + OR session.parent_session_id IS NOT NULL + ) + ) + OR ( + materialization.member_id<>?3 + AND ( + materialization.session_id=?4 + OR session.parent_session_id IS NULL + OR session.parent_session_id<>?4 + ) + ) + )", + params![ + run_id, + expected_generation, + COORDINATOR_MEMBER_ID, + &root_session_id + ], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if invalid_materialized_identities != 0 { + return Err(format!( + "materialization_identity_mismatch: {invalid_materialized_identities} certified Session identity row(s) are invalid for {run_id}" + )); + } + let incomplete_materializations: i64 = transaction + .query_row( + "SELECT COUNT(*) FROM agent_org_member_materializations + WHERE org_run_id=?1 AND generation=?2 AND status<>'succeeded'", + params![run_id, expected_generation], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if incomplete_materializations != 0 { + return Err(format!( + "team_not_materialized: {incomplete_materializations} receipt(s) incomplete for {run_id}" + )); + } + let initial_input = load_initial_input_with_connection(&transaction, run_id)?; + if has_initial_work { + let input = initial_input.as_ref().ok_or_else(|| { + format!("initial input certificate missing for Starting run {run_id}") + })?; + let message_exists: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM agent_messages + WHERE id=?1 AND session_id=?2 AND role='user' AND content=?3)", + params![&input.message_id, &root_session_id, &input.content], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + let event_id = format!("user-message-{}", input.message_id); + let event_exists: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM events WHERE id=?1 AND session_id=?2)", + params![event_id, &root_session_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if !message_exists || !event_exists { + return Err(format!( + "initial input is not durably materialized for Starting run {run_id}" + )); + } + crate::foundation::session_bridge::upsert_turn_intent_with_connection( + &transaction, + &root_session_id, + &input.turn_intent_id, + Some(&input.message_id), + Some(run_id), + crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, + crate::foundation::session_bridge::TurnIntentBridgeStatus::Queued, + )?; + transaction + .execute( + "UPDATE agent_org_initial_inputs + SET status='queued', updated_at=?2 + WHERE org_run_id=?1 AND status='pending_persistence'", + params![run_id, chrono::Utc::now().to_rfc3339()], + ) + .map_err(|error| error.to_string())?; + } else if initial_input.is_some() { + return Err(format!( + "unexpected initial input certificate for no-work Starting run {run_id}" + )); + } + + let next = if has_initial_work { + AgentOrgRunStatus::Running + } else { + AgentOrgRunStatus::Idle + }; + let now = chrono::Utc::now().to_rfc3339(); + let changed = transaction + .execute( + "UPDATE agent_org_runs + SET status=?1, updated_at=?2, + idled_at=CASE WHEN ?1='idle' THEN ?2 ELSE NULL END + WHERE id=?3 AND status='starting' + AND activation_generation=?4", + params![next.as_str(), &now, run_id, expected_generation], + ) + .map_err(|error| error.to_string())?; + if changed != 1 { + return Err(format!("Starting transition lost for run {run_id}")); + } + transaction.commit().map_err(|error| error.to_string())?; + Ok(next) + })?; + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); + Ok(status) + } + + pub fn mark_initial_input_dispatched( + run_id: &str, + turn_intent_id: &str, + ) -> Result { + with_sessions_writer(|| { + let connection = get_connection().map_err(|error| error.to_string())?; + let changed = connection + .execute( + "UPDATE agent_org_initial_inputs + SET status='dispatched', updated_at=?3 + WHERE org_run_id=?1 AND turn_intent_id=?2 + AND status IN ('queued', 'dispatched')", + params![run_id, turn_intent_id, chrono::Utc::now().to_rfc3339()], + ) + .map_err(|error| error.to_string())?; + Ok(changed == 1) + }) + } + + pub fn list_starting_runs(limit: usize) -> Result, String> { + Self::list_runs_by_status(AgentOrgRunStatus::Starting, limit) + } + /// Pause a running run. Only transitions `running → paused`; already /// non-running runs are left unchanged and return `Ok(false)` (idempotent). pub fn mark_paused(run_id: &str) -> Result { @@ -148,42 +613,14 @@ impl AgentOrgRunStore { Ok(changed) } - /// Called once at app startup to pause every org run that was `running` - /// when the previous process exited. The member sessions will have been - /// marked `abandoned` by `mark_stale_running_sessions_abandoned`, but the - /// org run itself should remain accessible and resumable — not auto-terminated - /// by `reconcile_run_finality`. Transitioning to `paused` achieves this: - /// `reconcile_run_finality` is a no-op for non-`running` runs, and the - /// frontend's `TERMINAL_RUN_STATUSES` set excludes `paused`, so the overview - /// panel, member switcher, and task board stay visible. - /// - /// Returns the number of runs transitioned. - pub fn mark_all_running_as_paused_on_startup() -> Result { - let paused = validate_status(AgentOrgRunStatus::Paused.as_str())?; - let running = validate_status(AgentOrgRunStatus::Running.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - with_sessions_writer(|| -> Result { - let conn = get_connection().map_err(|err| err.to_string())?; - let rows_changed = conn - .execute( - "UPDATE agent_org_runs - SET status = ?1, - updated_at = ?2 - WHERE status = ?3", - params![paused.as_str(), now, running.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(rows_changed) - }) - } - /// Apply the normal failed-member task disposition after crash recovery - /// has converted stranded Running sessions to Abandoned, but before the - /// parent runs are paused. Tasks with an eligible peer return to the pool; - /// sole-member work stays owned and pending for an explicit retry. + /// has converted stranded Running sessions to Abandoned. Tasks with an + /// eligible peer return to the pool; sole-member work stays owned and + /// pending for an explicit retry. The Team remains Running and is then + /// reconciled from its Quiescence facts. pub fn requeue_abandoned_member_tasks_on_startup() -> Result { let mut changed = 0usize; - for run in Self::list_running_runs(usize::MAX)? { + for run in Self::list_running_runs(100)? { for worker in Self::list_descendant_worker_sessions(&run.id)? { if worker.status != SessionStatus::Abandoned { continue; @@ -198,24 +635,6 @@ impl AgentOrgRunStore { Ok(changed) } - /// Complete already-resolved runs before the generic startup pause sweep. - /// - /// A previous process may have left a run `running` only because an - /// orphaned turn intent incorrectly looked queued. Startup reconciliation - /// closes those intents. Run the canonical atomic finality check for every - /// Running run, including an empty board with an explicit completion - /// intent; only runs that still have blockers fall through to - /// `mark_all_running_as_paused_on_startup`. - pub fn reconcile_resolved_running_runs_on_startup() -> Result { - let mut completed = 0usize; - for run in Self::list_running_runs(usize::MAX)? { - if Self::reconcile_run_finality(&run.id)? == Some(AgentOrgRunStatus::Completed) { - completed += 1; - } - } - Ok(completed) - } - /// Resume a paused run. Only transitions `paused → running`; already /// non-paused runs are left unchanged and return `Ok(false)` (idempotent). pub fn mark_resumed(run_id: &str) -> Result { @@ -242,67 +661,45 @@ impl AgentOrgRunStore { Ok(changed) } - /// Establish the durable fence for a user-requested hierarchy deletion. - /// - /// `paused` remains resumable, so deletion must not use it as the final - /// stop signal. Moving a live run to `cancelled` prevents resume and wake - /// paths from starting new work while the caller drains Rust runtimes. - pub(crate) fn cancel_for_delete_with_connection( - conn: &Connection, + pub fn fail_starting( run_id: &str, + expected_generation: i64, + failure: &AgentOrgStartingFailure, ) -> Result { + let failure_json = serde_json::to_string(failure) + .map_err(|error| format!("failed to serialize Starting failure: {error}"))?; let now = chrono::Utc::now().to_rfc3339(); - let changed = conn - .execute( - "UPDATE agent_org_runs - SET status='cancelled', - updated_at=?2, - completed_at=COALESCE(completed_at, ?2) - WHERE id=?1 - AND status IN ('running', 'paused')", - params![run_id, &now], - ) - .map_err(|err| err.to_string())? - > 0; - conn.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], - ) - .map_err(|err| err.to_string())?; - Ok(changed) - } - - pub fn mark_failed(run_id: &str, error_message: &str) -> Result<(), String> { - let status = validate_status(AgentOrgRunStatus::Failed.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - with_sessions_writer(|| -> Result<(), String> { + let changed = with_sessions_writer(|| -> Result { let mut conn = get_connection().map_err(|err| err.to_string())?; let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) .map_err(|err| err.to_string())?; - tx.execute( - "UPDATE agent_org_runs - SET status = ?1, + let changed = tx + .execute( + "UPDATE agent_org_runs + SET status = 'failed', last_error = ?2, - updated_at = ?3, - completed_at = ?3 - WHERE id = ?4", - params![status.as_str(), error_message, now, run_id], - ) - .map_err(|err| err.to_string())?; - tx.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], - ) - .map_err(|err| err.to_string())?; - tx.commit().map_err(|err| err.to_string()) + failure_json = ?3, + last_activity_outcome = 'failed', + updated_at = ?4 + WHERE id = ?1 AND status='starting' + AND activation_generation=?5", + params![ + run_id, + &failure.message, + &failure_json, + &now, + expected_generation + ], + ) + .map_err(|err| err.to_string())?; + tx.commit().map_err(|err| err.to_string())?; + Ok(changed == 1) })?; - crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); - Ok(()) + if changed { + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); + } + Ok(changed) } pub fn progress(run_id: &str) -> Result, String> { @@ -363,7 +760,7 @@ impl AgentOrgRunStore { } /// Persist a coordinator-only completion request without forcing the run - /// terminal. Finality still waits for delivery, approvals, interventions, + /// terminal. Quiescence still waits for delivery, approvals, interventions, /// sessions, and work-observation invariants to become safe. pub fn request_completion( run_id: &str, @@ -423,7 +820,7 @@ impl AgentOrgRunStore { Ok(outcome) } - pub fn assess_run_finality(run_id: &str) -> Result { + pub fn assess_run_quiescence(run_id: &str) -> Result { let mut conn = get_connection().map_err(|err| err.to_string())?; let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Deferred) @@ -433,72 +830,70 @@ impl AgentOrgRunStore { Ok(assessment) } - pub fn reconcile_run_finality(run_id: &str) -> Result, String> { - // Finality and every task mutation share the sessions writer lock. The - // canonical typed facts are re-read inside this IMMEDIATE transaction; - // no analyzer snapshot is trusted across the lock boundary. - let (status, changed) = - with_sessions_writer(|| -> Result<(Option, bool), String> { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let assessment = load_and_assess(&tx, run_id)?; - let Some(current_status) = assessment.facts.run_status else { - tx.commit().map_err(|err| err.to_string())?; - return Ok((None, false)); - }; - let next_status = match assessment.decision { - super::AgentOrgFinalityDecision::Complete => AgentOrgRunStatus::Completed, - super::AgentOrgFinalityDecision::Abandon => AgentOrgRunStatus::Abandoned, - super::AgentOrgFinalityDecision::KeepRunning => { - tx.commit().map_err(|err| err.to_string())?; - return Ok((Some(current_status), false)); - } - }; - let now = chrono::Utc::now().to_rfc3339(); - let completion_summary = assessment + /// Atomically commit the only automatic lifecycle transition owned by + /// PR 1. Both snapshot certificates are required so a stale finalizer or + /// watchdog pass cannot idle a newer activation or newer work graph. + pub fn try_transition_working_to_idle( + run_id: &str, + expected_generation: i64, + expected_work_revision: i64, + ) -> Result { + let changed = with_sessions_writer(|| -> Result { + let mut conn = get_connection().map_err(|err| err.to_string())?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|err| err.to_string())?; + let assessment = load_and_assess(&tx, run_id)?; + if assessment.facts.run_status != Some(AgentOrgRunStatus::Running) + || assessment.facts.activation_generation != Some(expected_generation) + || assessment .facts .progress .as_ref() - .and_then(|progress| progress.completion_summary.as_deref()); - let changed = tx - .execute( - "UPDATE agent_org_runs - SET status=?1, - summary=COALESCE(?2, summary), - updated_at=?3, - completed_at=?3 - WHERE id=?4 AND status=?5", - params![ - next_status.as_str(), - completion_summary, - &now, - run_id, - AgentOrgRunStatus::Running.as_str(), - ], - ) - .map_err(|err| err.to_string())?; - if changed != 1 { - tx.commit().map_err(|err| err.to_string())?; - return Ok((Self::get_run_status(run_id)?, false)); - } - // Terminal status and cancellation of an otherwise stranded plan - // approval are one atomic state transition. - tx.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], + .map(|progress| progress.work_revision) + != Some(expected_work_revision) + || assessment.decision != super::AgentOrgQuiescenceDecision::Quiescent + { + tx.commit().map_err(|err| err.to_string())?; + return Ok(false); + } + let now = chrono::Utc::now().to_rfc3339(); + let completion_summary = assessment + .facts + .progress + .as_ref() + .and_then(|progress| progress.completion_summary.as_deref()); + let changed = tx + .execute( + "UPDATE agent_org_runs + SET status='idle', + summary=COALESCE(?1, summary), + last_activity_outcome='completed', + updated_at=?2, + idled_at=?2 + WHERE id=?3 AND status='running' + AND activation_generation=?4 + AND EXISTS ( + SELECT 1 FROM agent_org_run_progress progress + WHERE progress.org_run_id=agent_org_runs.id + AND progress.work_revision=?5 + )", + params![ + completion_summary, + &now, + run_id, + expected_generation, + expected_work_revision, + ], ) .map_err(|err| err.to_string())?; - tx.commit().map_err(|err| err.to_string())?; - Ok((Some(next_status), true)) - })?; + tx.commit().map_err(|err| err.to_string())?; + Ok(changed == 1) + })?; if changed { crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); } - Ok(status) + Ok(changed) } /// Resolve the org-run context for an arbitrary session — works for @@ -617,14 +1012,18 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IS NOT NULL ORDER BY updated_at DESC @@ -641,11 +1040,21 @@ impl AgentOrgRunStore { Ok(out) } - /// List runs currently in `running` status, newest-updated first. - /// SQL-side status filter avoids loading terminal runs. Callers that must - /// inspect every running run (the watchdog) pass `usize::MAX`, which is - /// safely clamped to SQLite's `i64` limit. + /// List runs currently in `running` status, oldest-updated first. Periodic + /// callers must pass their explicit bounded batch size. pub fn list_running_runs(limit: usize) -> Result, String> { + Self::list_runs_by_status(AgentOrgRunStatus::Running, limit) + } + + fn list_runs_by_status( + status: AgentOrgRunStatus, + limit: usize, + ) -> Result, String> { + if limit == 0 { + return Ok(Vec::new()); + } + let bounded_limit = i64::try_from(limit) + .map_err(|_| format!("Agent Org run list limit is too large: {limit}"))?; let conn = get_connection().map_err(|err| err.to_string())?; let mut stmt = conn .prepare( @@ -656,29 +1065,27 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IS NOT NULL AND status = ?1 - ORDER BY updated_at DESC + ORDER BY updated_at ASC, id ASC LIMIT ?2", ) .map_err(|err| err.to_string())?; let rows = stmt - .query_map( - params![ - AgentOrgRunStatus::Running.as_str(), - i64::try_from(limit).unwrap_or(i64::MAX) - ], - row_to_run, - ) + .query_map(params![status.as_str(), bounded_limit], row_to_run) .map_err(|err| err.to_string())?; let mut out = Vec::new(); for row in rows { @@ -708,14 +1115,14 @@ impl AgentOrgRunStore { Ok(status_raw.as_deref().and_then(AgentOrgRunStatus::parse)) } - /// Read the canonical finality facts and decision from an existing + /// Read the canonical quiescence facts and decision from an existing /// connection or read transaction. Run View and task-list projections use /// this to keep all of their independently-shaped rows on one SQLite /// snapshot instead of opening a fresh connection for each block. - pub(crate) fn finality_assessment_with_connection( + pub(crate) fn quiescence_assessment_with_connection( conn: &Connection, run_id: &str, - ) -> Result { + ) -> Result { load_and_assess(conn, run_id) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs index def031c102..1071dc5bc5 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs @@ -13,11 +13,72 @@ fn enum_values_round_trip() { AgentOrgRunEntryMode::parse(AgentOrgRunEntryMode::StandaloneSession.as_str()), Some(AgentOrgRunEntryMode::StandaloneSession) ); - assert_eq!( - AgentOrgRunStatus::parse(AgentOrgRunStatus::Running.as_str()), - Some(AgentOrgRunStatus::Running) - ); - assert_eq!(AgentOrgRunStatus::parse("idle"), None); + for status in [ + AgentOrgRunStatus::Starting, + AgentOrgRunStatus::Running, + AgentOrgRunStatus::Paused, + AgentOrgRunStatus::Idle, + AgentOrgRunStatus::Failed, + AgentOrgRunStatus::Archived, + ] { + assert_eq!(AgentOrgRunStatus::parse(status.as_str()), Some(status)); + } + for retired in ["completed", "cancelled", "abandoned", "unknown"] { + assert_eq!(AgentOrgRunStatus::parse(retired), None); + } +} + +#[test] +fn canonical_schema_snapshot_contains_only_the_long_lived_run_states() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_runtime_schemas(); + let conn = database::db::get_connection().expect("test sqlite connection"); + let run_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_runs'", + [], + |row| row.get(0), + ) + .expect("canonical Agent Org run DDL"); + let status_ddl = run_ddl + .split_once("status TEXT") + .and_then(|(_, tail)| tail.split_once("activation_generation")) + .map(|(status_ddl, _)| status_ddl) + .expect("isolated run-status CHECK"); + for status in [ + "starting", "running", "paused", "idle", "failed", "archived", + ] { + assert!( + status_ddl.contains(&format!("'{status}'")), + "DDL: {status_ddl}" + ); + } + for retired in ["'abandoned'", "'completed'", "'cancelled'"] { + assert!(!status_ddl.contains(retired), "DDL: {status_ddl}"); + } + + let materialization_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_member_materializations'", + [], + |row| row.get(0), + ) + .expect("canonical materialization receipt DDL"); + assert!(materialization_ddl.contains("PRIMARY KEY(org_run_id, member_id, generation)")); + assert!(materialization_ddl.contains("UNIQUE(org_run_id, session_id)")); + + let initial_input_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_initial_inputs'", + [], + |row| row.get(0), + ) + .expect("canonical initial-input DDL"); + assert!(initial_input_ddl.contains("UNIQUE(turn_intent_id)")); + assert!(initial_input_ddl.contains("UNIQUE(message_id)")); } /// Build an `AgentOrgsStore` pre-loaded with a single org definition. @@ -50,6 +111,35 @@ fn sample_org() -> OrgDefinition { } } +fn test_upsert_turn_intent_with_connection( + conn: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: crate::foundation::session_bridge::TurnIntentBridgeSource, + status: crate::foundation::session_bridge::TurnIntentBridgeStatus, +) -> Result<(), String> { + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "INSERT OR IGNORE INTO session_turn_intents ( + session_id, turn_intent_id, client_message_id, org_run_id, + source, status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)", + params![ + session_id, + turn_intent_id, + client_message_id, + org_run_id, + source.as_str(), + status.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + fn ensure_runtime_schemas() { let conn = database::db::get_connection().expect("test sqlite connection"); crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); @@ -76,9 +166,16 @@ fn ensure_runtime_schemas() { created_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (session_id, turn_intent_id) + ); + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL );", ) .expect("cli session schema"); + crate::foundation::session_bridge::register_upsert_turn_intent_with_connection( + test_upsert_turn_intent_with_connection, + ); } fn create_run_for_root(org: &OrgDefinition, root_session_id: &str) -> AgentOrgRunRecord { @@ -97,6 +194,237 @@ fn create_run_for_root(org: &OrgDefinition, root_session_id: &str) -> AgentOrgRu .expect("create run") } +/// Exercise the production two-step protocol used by lifecycle owners: read a +/// pure quiescence certificate, then present its exact generation and work +/// revision to the atomic CAS transition. Keeping this helper in tests makes +/// old completion scenarios validate the new protocol instead of recreating the +/// removed one-shot reconciler. +fn reconcile_run_to_idle_for_test(run_id: &str) -> Result { + let assessment = AgentOrgRunStore::assess_run_quiescence(run_id)?; + if assessment.decision == AgentOrgQuiescenceDecision::Quiescent { + let generation = assessment + .facts + .activation_generation + .ok_or_else(|| "missing activation generation".to_string())?; + let work_revision = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + .ok_or_else(|| "missing work revision".to_string())?; + AgentOrgRunStore::try_transition_working_to_idle(run_id, generation, work_revision)?; + } + load_by_id(run_id) + .map_err(|err| err.to_string())? + .map(|run| run.status) + .ok_or_else(|| format!("agent_org_run_not_found: {run_id}")) +} + +fn create_starting_fixture(has_initial_work: bool) -> AgentOrgRunRecord { + ensure_runtime_schemas(); + let org = sample_org(); + upsert_session_row_for_member( + "starting-root", + None, + Some("agent-coord"), + Some(COORDINATOR_MEMBER_ID), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::create_starting(CreateStartingAgentOrgRunParams { + org_id: org.id.clone(), + coordinator_agent_id: org.agent_id.clone(), + root_session_id: "starting-root".to_string(), + org_snapshot: org, + entry_mode: AgentOrgRunEntryMode::StandaloneSession, + work_item_id: None, + project_slug: None, + routine_fire_id: None, + materialization_intents: vec![ + CreateAgentOrgMaterializationIntent { + member_id: COORDINATOR_MEMBER_ID.to_string(), + agent_id: "agent-coord".to_string(), + session_id: "starting-root".to_string(), + succeeded: true, + }, + CreateAgentOrgMaterializationIntent { + member_id: "member-w1".to_string(), + agent_id: "agent-w1".to_string(), + session_id: "starting-member-w1".to_string(), + succeeded: false, + }, + ], + initial_input: has_initial_work.then(|| CreateAgentOrgInitialInput { + turn_intent_id: "starting-turn".to_string(), + message_id: "starting-message".to_string(), + content: "Start the work".to_string(), + payload_json: serde_json::json!({ + "version": 1, + "images": ["image-a"], + "ideContext": null, + "subAgentIds": [], + }) + .to_string(), + }), + }) + .expect("create Starting fixture") +} + +#[test] +fn starting_creation_commits_exact_roster_and_initial_input_receipts() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(true); + + assert_eq!(run.status, AgentOrgRunStatus::Starting); + assert_eq!(run.activation_generation, 1); + assert!(run.has_initial_work); + let receipts = AgentOrgRunStore::materializations(&run.id).expect("load receipts"); + assert_eq!(receipts.len(), 2); + assert_eq!(receipts[0].session_id, "starting-root"); + assert_eq!(receipts[0].status, AgentOrgMaterializationStatus::Succeeded); + assert_eq!(receipts[1].session_id, "starting-member-w1"); + assert_eq!(receipts[1].status, AgentOrgMaterializationStatus::Pending); + let input = AgentOrgRunStore::initial_input(&run.id) + .expect("load initial input") + .expect("initial input exists"); + assert_eq!(input.turn_intent_id, "starting-turn"); + assert!(input.payload_json.contains("image-a")); +} + +#[test] +fn starting_finish_requires_exact_member_and_input_durability_then_is_idempotent() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(true); + assert!(AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("pending member must block Starting") + .contains("receipt(s) incomplete")); + + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + assert!(AgentOrgRunStore::mark_materialization_succeeded( + &run.id, + "member-w1", + 1, + "starting-member-w1", + ) + .expect("certify stable member")); + assert!(!AgentOrgRunStore::mark_materialization_succeeded( + &run.id, + "member-w1", + 1, + "starting-member-w1", + ) + .expect("retry same receipt")); + assert!(AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("missing initial EventStore row must block Starting") + .contains("not durably materialized")); + + crate::session::persistence::save_user_msg_with_id( + "starting-message", + "starting-root", + "Start the work", + ) + .expect("persist transcript input"); + database::db::get_connection() + .expect("db") + .execute( + "INSERT INTO events (id, session_id) VALUES (?1, ?2)", + params!["user-message-starting-message", "starting-root"], + ) + .expect("persist EventStore proof"); + + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("finish Starting"), + AgentOrgRunStatus::Running + ); + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("idempotent finish"), + AgentOrgRunStatus::Running + ); + let conn = database::db::get_connection().expect("db"); + let member_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_sessions WHERE session_id='starting-member-w1'", + [], + |row| row.get(0), + ) + .expect("count stable member identity"); + assert_eq!(member_count, 1); + let turn_status: String = conn + .query_row( + "SELECT status FROM session_turn_intents + WHERE session_id='starting-root' AND turn_intent_id='starting-turn'", + [], + |row| row.get(0), + ) + .expect("load durable initial Turn Intent"); + assert_eq!(turn_status, "queued"); +} + +#[test] +fn starting_without_initial_work_finishes_idle() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(false); + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::mark_materialization_succeeded(&run.id, "member-w1", 1, "starting-member-w1") + .expect("certify stable member"); + + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("finish no-work Starting"), + AgentOrgRunStatus::Idle + ); + assert!(load_by_id(&run.id) + .expect("load run") + .expect("run exists") + .idled_at + .is_some()); +} + +#[test] +fn starting_finish_revalidates_every_certified_session_identity() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(false); + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::mark_materialization_succeeded(&run.id, "member-w1", 1, "starting-member-w1") + .expect("certify stable member"); + database::db::get_connection() + .expect("db") + .execute( + "UPDATE agent_sessions + SET parent_session_id='wrong-root' + WHERE session_id='starting-member-w1'", + [], + ) + .expect("corrupt certified identity after receipt"); + + let error = AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("a stale receipt must not authorize Starting completion"); + assert!(error.starts_with("materialization_identity_mismatch:")); + assert_eq!( + AgentOrgRunStore::load(&run.id) + .expect("load Starting run") + .expect("run exists") + .status, + AgentOrgRunStatus::Starting + ); +} + #[test] fn delete_by_id_cascades_all_run_owned_state_and_plan_artifact() { let sandbox = test_helpers::test_env::sandbox(); @@ -253,7 +581,7 @@ fn delete_by_id_cascades_all_run_owned_state_and_plan_artifact() { } #[test] -fn delete_by_id_preserves_nested_run_intents_and_finality_isolation() { +fn delete_by_id_preserves_nested_run_intents_and_quiescence_isolation() { let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); let outer = create_run_for_root(&org, "outer-root"); @@ -301,15 +629,15 @@ fn delete_by_id_preserves_nested_run_intents_and_finality_isolation() { .expect("seed independently owned intents"); let outer_assessment = - AgentOrgRunStore::assess_run_finality(&outer.id).expect("assess outer run finality"); + AgentOrgRunStore::assess_run_quiescence(&outer.id).expect("assess outer run quiescence"); assert_eq!( outer_assessment.facts.in_flight_turn_intent_count, 1, - "nested run work must not block outer run finality" + "nested run work must not block outer run quiescence" ); assert_eq!(outer_assessment.facts.worker_sessions.len(), 1); assert_eq!( outer_assessment.facts.worker_sessions[0].session_id, "outer-worker", - "a Running worker owned by a nested run must not block outer finality" + "a Running worker owned by a nested run must not block outer quiescence" ); AgentOrgRunStore::delete_by_id(&outer.id).expect("delete outer run"); @@ -361,7 +689,7 @@ fn recursive_session_queries_terminate_on_parent_cycle() { descendants.len() <= 3, "cycle must not duplicate descendants" ); - AgentOrgRunStore::assess_run_finality(&run.id).expect("cyclic finality scan terminates"); + AgentOrgRunStore::assess_run_quiescence(&run.id).expect("cyclic quiescence scan terminates"); let conn = database::db::get_connection().expect("test sqlite connection"); let now = chrono::Utc::now().to_rfc3339(); @@ -756,7 +1084,7 @@ fn find_worker_session_by_member_id_picks_most_recent_when_multi_instance() { } #[test] -fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_finality() { +fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_quiescence() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -837,16 +1165,16 @@ fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_f mark_coordinator_observed_current_work(&run.id); stamp_coordinator_terminal_turn("coord-root-cross-transport"); - let assessment = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess finality"); + let assessment = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess quiescence"); assert_eq!(assessment.facts.worker_sessions.len(), 1); assert_eq!( assessment.facts.worker_sessions[0].session_id, "rust-worker-current" ); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::Quiescent); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile finality"), - Some(AgentOrgRunStatus::Completed), + reconcile_run_to_idle_for_test(&run.id).expect("transition to idle"), + AgentOrgRunStatus::Idle, "the stale CLI Running row must not keep the run falsely active" ); } @@ -923,7 +1251,7 @@ fn coordinator_observation_records_only_the_exact_presented_revision() { } #[test] -fn reconcile_run_finality_completes_run_when_all_tasks_completed() { +fn quiescence_transitions_run_to_idle_when_all_tasks_completed() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -985,8 +1313,8 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { ) .expect("advance pending turn intent"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile pending intent"), - Some(AgentOrgRunStatus::Running), + reconcile_run_to_idle_for_test(&run.id).expect("reconcile pending intent"), + AgentOrgRunStatus::Running, "a {pending_status} turn intent must keep the run open" ); } @@ -1004,12 +1332,12 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { ) .expect("set terminal turn intent"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile terminal intent"), - Some(AgentOrgRunStatus::Completed), + reconcile_run_to_idle_for_test(&run.id).expect("reconcile terminal intent"), + AgentOrgRunStatus::Idle, "a {terminal_status} turn intent must not keep the run open" ); conn.execute( - "UPDATE agent_org_runs SET status='running', completed_at=NULL WHERE id=?1", + "UPDATE agent_org_runs SET status='running', idled_at=NULL WHERE id=?1", params![&run.id], ) .expect("reset run for next terminal status"); @@ -1037,13 +1365,12 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { 1 ); assert_eq!( - AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .expect("startup reconcile ok"), - 1 + reconcile_run_to_idle_for_test(&run.id).expect("explicit lifecycle reconcile ok"), + AgentOrgRunStatus::Idle ); let reloaded = load_by_id(&run.id).expect("load run").expect("run exists"); - assert_eq!(reloaded.status, AgentOrgRunStatus::Completed); - assert!(reloaded.completed_at.is_some()); + assert_eq!(reloaded.status, AgentOrgRunStatus::Idle); + assert!(reloaded.idled_at.is_some()); let legacy_cleared_at: Option = conn .query_row( "SELECT cleared_at FROM agent_member_interventions @@ -1056,7 +1383,7 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { } #[test] -fn reconcile_completes_normal_idle_run_only_after_inbox_is_drained() { +fn quiescence_idles_run_only_after_inbox_is_drained() { use crate::coordination::agent_inbox::{ AgentInboxStore, AgentMessage, InsertInboxParams, SYSTEM_SENDER_ID, }; @@ -1113,20 +1440,20 @@ fn reconcile_completes_normal_idle_run_only_after_inbox_is_drained() { stamp_coordinator_terminal_turn("coord-root-idle-complete"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).unwrap(), - Some(AgentOrgRunStatus::Running), - "unread completion facts must be delivered before finality" + reconcile_run_to_idle_for_test(&run.id).unwrap(), + AgentOrgRunStatus::Running, + "unread completion facts must be delivered before quiescence" ); AgentInboxStore::mark_many_read(&[row.id]).unwrap(); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).unwrap(), - Some(AgentOrgRunStatus::Completed), - "normal successful members settle to Idle and must still allow run completion" + reconcile_run_to_idle_for_test(&run.id).unwrap(), + AgentOrgRunStatus::Idle, + "normal successful members settle to Idle and must allow the Team to become Idle" ); } #[test] -fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { +fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_quiescence() { use crate::coordination::agent_inbox::{ AgentInboxDeliveryResolutionKind, AgentInboxStore, AgentMessage, ResolveInboxDeliveryParams, }; @@ -1188,9 +1515,9 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { .expect("seed historical orphan row"); let inbox_id = conn.last_insert_rowid(); - let before = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess before repair"); + let before = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess before repair"); assert_eq!(before.facts.unread_inbox_count, 1); - assert_eq!(before.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!(before.decision, AgentOrgQuiescenceDecision::KeepWorking); AgentInboxStore::resolve_delivery(ResolveInboxDeliveryParams { inbox_id, @@ -1203,12 +1530,12 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { }) .expect("resolve undeliverable delivery"); - let after = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess after repair"); + let after = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess after repair"); assert_eq!(after.facts.unread_inbox_count, 0); - assert_eq!(after.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(after.decision, AgentOrgQuiescenceDecision::Quiescent); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile repaired run"), - Some(AgentOrgRunStatus::Completed) + reconcile_run_to_idle_for_test(&run.id).expect("transition repaired run"), + AgentOrgRunStatus::Idle ); let evidence = AgentInboxStore::get_by_id_for_run(&run.id, inbox_id) .unwrap() @@ -1220,7 +1547,7 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { } #[test] -fn startup_reconcile_completes_empty_board_with_explicit_completion_intent() { +fn explicit_lifecycle_reconcile_idles_empty_board_with_completion_intent() { let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); let run = create_run_for_root(&org, "coord-root-empty-complete"); @@ -1235,21 +1562,20 @@ fn startup_reconcile_completes_empty_board_with_explicit_completion_intent() { .expect("record explicit empty-board completion intent"); assert_eq!( - AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .expect("startup reconcile empty board"), - 1 + reconcile_run_to_idle_for_test(&run.id).expect("reconcile empty board"), + AgentOrgRunStatus::Idle ); assert_eq!( load_by_id(&run.id) .expect("load run") .expect("run exists") .status, - AgentOrgRunStatus::Completed + AgentOrgRunStatus::Idle ); } #[test] -fn reconcile_run_finality_abandons_run_with_open_work_only_after_all_sessions_archived() { +fn archived_sessions_with_open_work_do_not_auto_archive_the_run() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -1313,11 +1639,11 @@ fn reconcile_run_finality_abandons_run_with_open_work_only_after_all_sessions_ar }) .expect("create open task"); - let status = AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile ok"); - assert_eq!(status, Some(AgentOrgRunStatus::Abandoned)); + let status = reconcile_run_to_idle_for_test(&run.id).expect("reconcile ok"); + assert_eq!(status, AgentOrgRunStatus::Running); let reloaded = load_by_id(&run.id).expect("load run").expect("run exists"); - assert_eq!(reloaded.status, AgentOrgRunStatus::Abandoned); - assert!(reloaded.completed_at.is_some()); + assert_eq!(reloaded.status, AgentOrgRunStatus::Running); + assert!(reloaded.idled_at.is_none()); } #[test] @@ -1358,32 +1684,32 @@ fn failed_or_cancelled_sessions_do_not_abandon_recoverable_open_work() { .expect("create recoverable task"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile"), - Some(AgentOrgRunStatus::Running) + reconcile_run_to_idle_for_test(&run.id).expect("reconcile"), + AgentOrgRunStatus::Running ); } #[test] -fn reconcile_and_task_create_have_one_serializable_outcome() { +fn idle_cas_and_task_create_have_one_serializable_outcome() { use std::sync::{Arc, Barrier}; use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); - let run = create_run_for_root(&org, "coord-root-finality-race"); + let run = create_run_for_root(&org, "coord-root-quiescence-race"); upsert_session_row_full( - "coord-root-finality-race", + "coord-root-quiescence-race", None, Some("agent-coord"), SessionStatus::Completed.as_str(), ); upsert_session(&UnifiedSessionRecord { - session_id: "worker-finality-race".to_string(), + session_id: "worker-quiescence-race".to_string(), name: "worker".to_string(), status: SessionStatus::Completed.as_str().to_string(), session_type: crate::core::session::persistence::session_type::ORG_MEMBER.to_string(), - parent_session_id: Some("coord-root-finality-race".to_string()), + parent_session_id: Some("coord-root-quiescence-race".to_string()), agent_definition_id: Some("agent-w1".to_string()), org_member_id: Some("member-w1".to_string()), created_at: chrono::Utc::now().to_rfc3339(), @@ -1405,14 +1731,14 @@ fn reconcile_and_task_create_have_one_serializable_outcome() { }) .unwrap(); mark_coordinator_observed_current_work(&run.id); - stamp_coordinator_terminal_turn("coord-root-finality-race"); + stamp_coordinator_terminal_turn("coord-root-quiescence-race"); let barrier = Arc::new(Barrier::new(2)); let reconcile_barrier = Arc::clone(&barrier); let reconcile_run_id = run.id.clone(); let reconcile = std::thread::spawn(move || { reconcile_barrier.wait(); - AgentOrgRunStore::reconcile_run_finality(&reconcile_run_id) + reconcile_run_to_idle_for_test(&reconcile_run_id) }); let create_barrier = Arc::clone(&barrier); let create_run_id = run.id.clone(); @@ -1435,17 +1761,17 @@ fn reconcile_and_task_create_have_one_serializable_outcome() { }) }); - let status = reconcile.join().unwrap().unwrap().unwrap(); + let status = reconcile.join().unwrap().unwrap(); let created = create.join().unwrap(); match (status, created) { - (AgentOrgRunStatus::Completed, Err(error)) => { + (AgentOrgRunStatus::Idle, Err(error)) => { assert!(error.contains("agent_org_run_not_mutable"), "got {error}"); } // The create committed first. Reconcile then sees recoverable open // work and correctly leaves the Run Running; this is the other valid // serial order. Abandoning here would lose a newly-created task. (AgentOrgRunStatus::Running, Ok(task)) => assert_eq!(task.id, "racing-task"), - (status, result) => panic!("non-serializable finality result: {status:?}, {result:?}"), + (status, result) => panic!("non-serializable quiescence result: {status:?}, {result:?}"), } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs index 5c275ada9a..6f8e309917 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs @@ -39,10 +39,10 @@ pub const TASK_METADATA_REQUIRED_ROLE: &str = "required_role"; pub const TASK_METADATA_OUTPUT: &str = "output"; pub const TASK_METADATA_EXECUTION_MODE: &str = "execution_mode"; -/// SQL predicate shared by finality and watchdog repair discovery. +/// SQL predicate shared by Quiescence and watchdog repair discovery. /// /// Historical/manual SQLite rows can bypass the typed write boundary. Keep -/// this predicate in one place so the finality count and the watchdog's +/// this predicate in one place so the Quiescence count and the watchdog's /// concrete repair identities cannot disagree about whether a row is safe to /// deserialize. The numeric values are interpolated from the same payload /// constants used by new writes. @@ -514,7 +514,7 @@ impl TaskStatus { } } - /// `completed` is treated as resolved for dependency and finality checks. + /// `completed` is treated as resolved for dependency and Quiescence checks. pub fn is_resolved(&self) -> bool { matches!(self, TaskStatus::Completed) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs index b9b05c54dd..a3016d3156 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs @@ -93,7 +93,7 @@ impl AgentOrgTaskStore { } /// Internal projection for a caller that has already run the shared - /// finality/corruption assessment in the same SQLite read snapshot. + /// Quiescence/corruption assessment in the same SQLite read snapshot. /// Keeping this separate avoids evaluating the expensive JSON integrity /// predicate twice per watchdog tick while the public wrapper remains /// fail-closed for every other caller. diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs index 047e5b0b16..1ec49c5754 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs @@ -153,7 +153,7 @@ fn task_mutations_require_running_parent_run() { )) .expect("running run permits create"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id='guarded-run'", + "UPDATE agent_org_runs SET status='archived' WHERE id='guarded-run'", [], ) .unwrap(); @@ -694,7 +694,7 @@ fn store_rejects_malformed_reserved_dispatch_metadata() { .expect("classify historical producer"); assert!( classified, - "historical oversized producer must block finality" + "historical oversized producer must block Quiescence" ); let timezone_less_metadata = serde_json::json!({ @@ -726,7 +726,7 @@ fn store_rejects_malformed_reserved_dispatch_metadata() { |row| row.get(0), ) .expect("classify historical timezone-less output"); - assert!(classified, "timezone-less output must block finality"); + assert!(classified, "timezone-less output must block Quiescence"); } #[test] diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs index f2a3b2dd00..94ba20f85e 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs @@ -12,8 +12,8 @@ //! without explicit repair: tasks owned by dead members, stale //! `in_progress` work, and ready ownerless tasks awaiting explicit //! coordinator assignment (issue #272 E1). -//! - **Reconcile the run** when every task is resolved and every worker -//! is terminal. +//! - **Reconcile Team Quiescence** when every formal Task, Inbox delivery, +//! Turn Intent, recovery reservation, and relevant Session is settled. //! //! Failed members are rate-limited by a per-`(run, member)` rewake budget //! (three attempts with 1/5/15-minute backoff) that resets on the next @@ -49,7 +49,7 @@ pub(crate) use reservation::{ }; use std::collections::{BTreeSet, HashMap, HashSet}; -use std::time::Duration; +use std::time::{Duration, Instant}; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use database::db::{get_connection, with_sessions_writer}; @@ -61,9 +61,9 @@ use crate::coordination::agent_inbox::{ }; use crate::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore; use crate::coordination::agent_org_runs::{ - recovery_dispatch_recipient_is_available, AgentOrgFinalityBlocker, AgentOrgFinalityDecision, - AgentOrgRunRecord, AgentOrgRunStatus, AgentOrgRunStore, WorkerSessionRuntime, - COORDINATOR_MEMBER_ID, + recovery_dispatch_recipient_is_available, AgentOrgQuiescenceBlocker, + AgentOrgQuiescenceDecision, AgentOrgRunRecord, AgentOrgRunStatus, AgentOrgRunStore, + WorkerSessionRuntime, COORDINATOR_MEMBER_ID, }; use crate::coordination::agent_org_tasks::{self, Task, TaskStatus}; use crate::core::session::SessionStatus; @@ -71,6 +71,8 @@ use crate::tools::impls::orchestration::inbox_wake::AppHandleInboxWakeHook; use crate::tools::impls::orchestration::org_send_message::InboxWakeHook; const WATCHDOG_INTERVAL_SECS: u64 = 60; +const WATCHDOG_MAX_RUNS: usize = 100; +const WATCHDOG_SCAN_BUDGET: Duration = Duration::from_millis(250); const RECOVERY_DELAYS_SECS: [i64; 3] = [60, 5 * 60, 15 * 60]; const PENDING_MATERIALIZATION_GRACE_SECS: i64 = 2 * 60; const MEMBER_REWAKE: &str = "member_rewake"; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs index 49ad41c198..b57782a6a4 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs @@ -21,33 +21,7 @@ pub fn init_schema(conn: &Connection) -> rusqlite::Result<()> { ); CREATE INDEX IF NOT EXISTS idx_agent_org_recovery_attempts_run ON agent_org_recovery_attempts(org_run_id);", - )?; - // Existing databases predate dispatch reservations. Keeping the token in - // the same row lets a failed/coalesced scheduler request refund only its - // own provisional attempt without undoing a newer recovery fingerprint. - ensure_recovery_attempt_column(conn, "reservation_token", "TEXT")?; - Ok(()) -} - -fn ensure_recovery_attempt_column( - conn: &Connection, - column_name: &str, - column_definition: &str, -) -> rusqlite::Result<()> { - let mut stmt = conn.prepare("PRAGMA table_info(agent_org_recovery_attempts)")?; - let columns = stmt.query_map([], |row| row.get::<_, String>(1))?; - for column in columns { - if column? == column_name { - return Ok(()); - } - } - conn.execute( - &format!( - "ALTER TABLE agent_org_recovery_attempts ADD COLUMN {column_name} {column_definition}" - ), - [], - )?; - Ok(()) + ) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -305,24 +279,3 @@ pub(super) fn member_rewake_fingerprint_from_unread( .map(|unread| format!("unread:{unread}")) .unwrap_or_else(|| format!("status:{}", status.as_str())) } - -/// Drop budget entries whose run is no longer running so the -/// process-global maps cannot grow unbounded over the app lifetime -/// (issue #272 E6). Paused runs also lose their entries; resuming one -/// intentionally grants a fresh set of recovery attempts. -pub(super) fn prune_recovery_budgets() -> Result<(), String> { - with_sessions_writer(|| { - let conn = get_connection().map_err(|err| err.to_string())?; - conn.execute( - "DELETE FROM agent_org_recovery_attempts - WHERE NOT EXISTS ( - SELECT 1 FROM agent_org_runs run - WHERE run.id = agent_org_recovery_attempts.org_run_id - AND run.status = ?1 - )", - params![AgentOrgRunStatus::Running.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(()) - }) -} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs index 36b29ad67d..d841e37bee 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs @@ -783,7 +783,8 @@ pub(super) fn inspect_stalled_run_with_connection( return Ok(StallRecoveryPlan::default()); } - let finality_assessment = AgentOrgRunStore::finality_assessment_with_connection(conn, run_id)?; + let quiescence_assessment = + AgentOrgRunStore::quiescence_assessment_with_connection(conn, run_id)?; let unread_counts = AgentInboxStore::unread_counts_by_recipient_with_connection(conn, run_id)?; let unread_fingerprints_by_member = unread_fingerprints_by_member(&unread_counts); let (coordinator_unread, coordinator_unread_wake_member_ids) = @@ -804,8 +805,8 @@ pub(super) fn inspect_stalled_run_with_connection( let coordinator_unread_suppresses_notice = coordinator_unread && !coordinator_unread_is_unavailable; - if finality_assessment.facts.corrupt_task_count > 0 { - let count = finality_assessment.facts.corrupt_task_count; + if quiescence_assessment.facts.corrupt_task_count > 0 { + let count = quiescence_assessment.facts.corrupt_task_count; let mut reasons = vec![format!( "The Agent Org task board has {count} persisted integrity or run-limit violation(s). The watchdog refused to guess task state or declare completion. Use task_list to identify bounded diagnostics. Ordinary task tools intentionally cannot rewrite malformed rows; cancel/delete this run or use a trusted maintenance path to repair the database before continuing." )]; @@ -816,7 +817,7 @@ pub(super) fn inspect_stalled_run_with_connection( &mut repair_facts, ); let has_new_notice = !coordinator_unread_suppresses_notice; - let work_revision = finality_assessment + let work_revision = quiescence_assessment .facts .progress .as_ref() @@ -831,7 +832,7 @@ pub(super) fn inspect_stalled_run_with_connection( .then(|| { recovery_repair_fingerprint(&repair_facts).ok_or_else(|| { format!( - "finality reported {count} corrupt task row(s), but no corrupt identity was found" + "quiescence reported {count} corrupt task row(s), but no corrupt identity was found" ) }) }) @@ -851,7 +852,7 @@ pub(super) fn inspect_stalled_run_with_connection( agent_org_tasks::AgentOrgTaskStore::list_operational_after_validated_with_connection( conn, run_id, )?; - let task_snapshot_work_revision = finality_assessment + let task_snapshot_work_revision = quiescence_assessment .facts .progress .as_ref() @@ -1254,9 +1255,9 @@ pub(super) fn inspect_stalled_run_with_connection( needs_repair.push(ready_unassigned_repair_reason(task)); } - for blocker in &finality_assessment.blockers { + for blocker in &quiescence_assessment.blockers { match blocker { - AgentOrgFinalityBlocker::EmptyTaskBoardRequiresCompletionIntent => { + AgentOrgQuiescenceBlocker::EmptyTaskBoardRequiresCompletionIntent => { repair_facts.push(RecoveryRepairFact::marker( "empty_board_requires_completion_intent", )); @@ -1265,7 +1266,7 @@ pub(super) fn inspect_stalled_run_with_connection( .to_string(), ); } - AgentOrgFinalityBlocker::StaleCompletionIntent { + AgentOrgQuiescenceBlocker::StaleCompletionIntent { requested_work_revision, current_work_revision, } => { @@ -1280,7 +1281,7 @@ pub(super) fn inspect_stalled_run_with_connection( "the previous completion request observed work revision {requested_work_revision:?}, but the board is now revision {current_work_revision}. Re-inspect the current task board and call org_run_complete again only if it is still finished." )); } - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { observed_work_revision, current_work_revision, } if tasks.iter().all(|task| task.status.is_resolved()) => { @@ -1295,36 +1296,39 @@ pub(super) fn inspect_stalled_run_with_connection( "all durable tasks are resolved, but the coordinator has only observed work revision {observed_work_revision:?}; the current revision is {current_work_revision}. Refresh task_list and produce the final user-facing synthesis." )); } - AgentOrgFinalityBlocker::CorruptTaskData { count } => { + AgentOrgQuiescenceBlocker::CorruptTaskData { count } => { repair_facts.extend(corrupt_task_repair_facts(conn, run_id)?); needs_repair.push(format!( "{count} task row(s) contain invalid persisted JSON. Do not declare completion; inspect and repair the task records." )); } - AgentOrgFinalityBlocker::ProgressStateMissing => { + AgentOrgQuiescenceBlocker::ProgressStateMissing => { repair_facts.push(RecoveryRepairFact::marker("missing_run_progress")); needs_repair.push( "the run is missing its durable work-revision record. Do not declare completion until the state is repaired." .to_string(), ); } - AgentOrgFinalityBlocker::RootSessionMissing => { + AgentOrgQuiescenceBlocker::RootSessionMissing => { repair_facts.push(RecoveryRepairFact::marker("missing_coordinator_session")); needs_repair.push( "the run has no materialized coordinator session, so final completion cannot be safely presented." .to_string(), ); } - AgentOrgFinalityBlocker::RunMissing - | AgentOrgFinalityBlocker::RunNotRunning { .. } - | AgentOrgFinalityBlocker::SessionsActive { .. } - | AgentOrgFinalityBlocker::OpenTasks { .. } - | AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } - | AgentOrgFinalityBlocker::UnreadInbox { .. } - | AgentOrgFinalityBlocker::ActiveInterventions { .. } - | AgentOrgFinalityBlocker::InFlightTurnIntents { .. } - | AgentOrgFinalityBlocker::PendingPlanApprovals { .. } - | AgentOrgFinalityBlocker::TerminalStateInconsistent { .. } => {} + AgentOrgQuiescenceBlocker::RunMissing + | AgentOrgQuiescenceBlocker::RunNotRunning { .. } + | AgentOrgQuiescenceBlocker::SessionsActive { .. } + | AgentOrgQuiescenceBlocker::OpenTasks { .. } + | AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } + | AgentOrgQuiescenceBlocker::UnreadInbox { .. } + | AgentOrgQuiescenceBlocker::ActiveInterventions { .. } + | AgentOrgQuiescenceBlocker::InFlightTurnIntents { .. } + | AgentOrgQuiescenceBlocker::UnknownTurnIntents { .. } + | AgentOrgQuiescenceBlocker::PendingFormalMaterializations { .. } + | AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { .. } + | AgentOrgQuiescenceBlocker::PendingPlanApprovals { .. } + | AgentOrgQuiescenceBlocker::QuietStateInconsistent { .. } => {} } } @@ -1339,8 +1343,8 @@ pub(super) fn inspect_stalled_run_with_connection( .and_then(|_| recovery_repair_fingerprint(&repair_facts)); let terminal_candidate = matches!( - finality_assessment.decision, - AgentOrgFinalityDecision::Complete | AgentOrgFinalityDecision::Abandon + quiescence_assessment.decision, + AgentOrgQuiescenceDecision::Quiescent ); let has_coordinator_repair = coordinator_repair_reason.is_some(); let coordinator_repair_active = !needs_repair.is_empty(); diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index 3f6d6db36f..f53d467700 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -3,8 +3,7 @@ //! [`super::inspect::inspect_stalled_run`]. use super::budget::{ - budget_disposition_with_connection, prune_recovery_budgets, record_attempt_with_connection, - BudgetDisposition, + budget_disposition_with_connection, record_attempt_with_connection, BudgetDisposition, }; use super::inspect::{ inspect_stalled_run_with_connection, pending_materialization_disposition, @@ -14,31 +13,10 @@ use super::inspect::{ use super::*; pub fn spawn(app_handle: AppHandle) { + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return; + } tauri::async_runtime::spawn(async move { - match tokio::task::spawn_blocking(|| { - AgentOrgPlanApprovalStore::repair_latest_plan_artifacts() - }) - .await - { - Ok(Ok(report)) => { - if report.repaired > 0 || report.failed > 0 { - tracing::info!( - inspected = report.inspected, - repaired = report.repaired, - failed = report.failed, - "[agent_org_watchdog] reconciled durable plan artifacts at startup" - ); - } - } - Ok(Err(err)) => tracing::warn!( - error = %err, - "[agent_org_watchdog] startup plan artifact reconciliation failed" - ), - Err(err) => tracing::warn!( - error = %err, - "[agent_org_watchdog] startup plan artifact worker failed" - ), - } let mut interval = tokio::time::interval(Duration::from_secs(WATCHDOG_INTERVAL_SECS)); // A slow scan must not be "repaid" with back-to-back burst // ticks afterwards; the next scheduled tick is enough. @@ -60,40 +38,32 @@ pub fn spawn(app_handle: AppHandle) { } fn recover_all_stalled_runs(app_handle: AppHandle) -> Result<(), String> { - let runs = AgentOrgRunStore::list_running_runs(usize::MAX)?; - run_best_effort_cleanup("prune recovery budgets", prune_recovery_budgets); - run_best_effort_cleanup("clear expired member interventions", || { - crate::coordination::agent_member_interventions::AgentMemberInterventionStore::clear_expired_and_legacy() - .map(|_| ()) - }); - run_best_effort_cleanup("cancel stale plan approvals", || { - AgentOrgPlanApprovalStore::cancel_pending_for_terminal_or_missing_runs().map(|_| ()) - }); - recover_listed_runs(app_handle, runs, recover_stalled_run) + let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; + let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS)?; + recover_listed_runs_until(app_handle, runs, deadline, recover_stalled_run) } -/// Auxiliary cleanup is useful but cannot be a global recovery gate. One bad -/// row must not prevent healthy runs from being inspected during this tick. -pub(super) fn run_best_effort_cleanup( - label: &'static str, - cleanup: impl FnOnce() -> Result<(), String>, -) { - if let Err(err) = cleanup() { - tracing::warn!( - cleanup = label, - error = %err, - "[agent_org_watchdog] maintenance failed; continuing run scan" - ); - } +#[cfg(test)] +pub(super) fn recover_listed_runs( + handle: H, + runs: Vec, + recover: impl FnMut(H, &str) -> Result, +) -> Result<(), String> { + let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; + recover_listed_runs_until(handle, runs, deadline, recover) } -pub(super) fn recover_listed_runs( +fn recover_listed_runs_until( handle: H, runs: Vec, + deadline: Instant, mut recover: impl FnMut(H, &str) -> Result, ) -> Result<(), String> { let mut failed_run_ids = Vec::new(); for run in runs { + if Instant::now() >= deadline { + break; + } if let Err(err) = recover(handle.clone(), &run.id) { tracing::warn!( run_id = %run.id, @@ -136,9 +106,19 @@ fn execute_stall_recovery_plan( // coordinator root session is still open), fall through and deliver // the wakes so pending inbox rows still reach their recipients. if plan.terminal_candidate { - let reconciled = AgentOrgRunStore::reconcile_run_finality(run_id)?; - if reconciled.is_some_and(|status| status != AgentOrgRunStatus::Running) { - return Ok(plan); + let assessment = AgentOrgRunStore::assess_run_quiescence(run_id)?; + if let (Some(generation), Some(work_revision)) = ( + assessment.facts.activation_generation, + assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision), + ) { + if AgentOrgRunStore::try_transition_working_to_idle(run_id, generation, work_revision)? + { + return Ok(plan); + } } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs index 4a777bbf83..b5a2b1f3b2 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs @@ -2,7 +2,7 @@ use super::budget::{ budget_disposition, coordinator_notice_allowed, rewake_budget_exhausted, BudgetDisposition, }; use super::inspect::is_wakeable_status; -use super::recover::{recover_listed_runs, run_best_effort_cleanup}; +use super::recover::recover_listed_runs; use super::*; use crate::coordination::agent_org_runs::{AgentOrgRunEntryMode, AgentOrgRunRecord}; @@ -16,14 +16,18 @@ fn fake_run(id: &str) -> AgentOrgRunRecord { org_snapshot_json: None, entry_mode: AgentOrgRunEntryMode::StandaloneSession, status: AgentOrgRunStatus::Running, + activation_generation: 1, + has_initial_work: true, work_item_id: None, project_slug: None, routine_fire_id: None, summary: None, + failure_json: None, last_error: None, + last_activity_outcome: None, created_at: now.clone(), updated_at: now, - completed_at: None, + idled_at: None, } } @@ -116,8 +120,75 @@ fn one_failed_run_does_not_skip_later_runs() { } #[test] -fn maintenance_failure_is_best_effort() { - run_best_effort_cleanup("injected", || Err("failure".to_string())); +fn watchdog_constants_match_the_single_bounded_design() { + assert_eq!(WATCHDOG_INTERVAL_SECS, 60); + assert_eq!(WATCHDOG_MAX_RUNS, 100); + assert_eq!(WATCHDOG_SCAN_BUDGET, Duration::from_millis(250)); +} + +#[test] +fn shared_scan_deadline_is_checked_at_each_team_boundary() { + let first = fake_run("run-slow"); + let second = fake_run("run-after-budget"); + let mut inspected = Vec::new(); + + recover_listed_runs((), vec![first, second], |(), run_id| { + inspected.push(run_id.to_string()); + if run_id == "run-slow" { + std::thread::sleep(Duration::from_millis(275)); + } + Ok(()) + }) + .expect("budget expiry is a bounded stop, not an error"); + + assert_eq!(inspected, vec!["run-slow"]); +} + +#[test] +fn running_query_is_limited_and_never_visits_quiet_states() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + let now = Utc::now().to_rfc3339(); + for index in 0..105 { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'watchdog-org', 'coordinator', ?2, 'standalone_session', + 'running', ?3, ?3)", + params![ + format!("running-{index:03}"), + format!("root-running-{index:03}"), + &now + ], + ) + .expect("seed Working run"); + } + for status in ["starting", "paused", "idle", "failed", "archived"] { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'watchdog-org', 'coordinator', ?2, 'standalone_session', + ?3, ?4, ?4)", + params![ + format!("quiet-{status}"), + format!("root-quiet-{status}"), + status, + &now + ], + ) + .expect("seed quiet run"); + } + + let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS).expect("bounded query"); + assert_eq!(runs.len(), WATCHDOG_MAX_RUNS); + assert!(runs + .iter() + .all(|run| run.status == AgentOrgRunStatus::Running)); + assert_eq!(runs.first().map(|run| run.id.as_str()), Some("running-000")); + assert_eq!(runs.last().map(|run| run.id.as_str()), Some("running-099")); } #[test] diff --git a/src-tauri/crates/agent-core/src/core/definitions/commands.rs b/src-tauri/crates/agent-core/src/core/definitions/commands.rs index ec4b458dc8..0b5ab93091 100644 --- a/src-tauri/crates/agent-core/src/core/definitions/commands.rs +++ b/src-tauri/crates/agent-core/src/core/definitions/commands.rs @@ -128,9 +128,10 @@ pub struct InboxRunSummary { #[tauri::command] pub async fn agent_org_run_list(limit: Option) -> Result, String> { use crate::core::coordination::agent_org_runs::AgentOrgRunStore; + crate::core::coordination::agent_org_runs::require_agent_org_redesign()?; const MAX_LIMIT: usize = 200; let effective_limit = limit.map(|n| n.min(MAX_LIMIT)).unwrap_or(MAX_LIMIT); - // This command backs a read-only Inbox list. Finality reconciliation is a + // This command backs a read-only Inbox list. Quiescence reconciliation is a // lifecycle/watchdog responsibility: doing it here used to turn one UI // refresh into as many as 200 global writer-lock + IMMEDIATE // transactions. Keep the read off the async command executor as well. diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs index 122ec142be..b39871376d 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use core_types::key_source::KeySource; -use crate::coordination::agent_org_runs::AgentOrgRunStore; +use crate::coordination::agent_org_runs::{AgentOrgRunStore, AgentOrgStartingFailure}; use crate::definitions::orgs::{OrgMember, OrgMemberRuntimeConfig}; use crate::session::turn::streaming::{ broadcast_agent_error_structured, classify_streaming_error_message, StreamingError, @@ -30,7 +30,16 @@ pub(super) async fn handle_background_launch_failure( ) { tracing::warn!("{}", message); if let Some(run_id) = agent_org_run_id { - if let Err(mark_err) = AgentOrgRunStore::mark_failed(run_id, message) { + let failure_result = AgentOrgRunStore::load(run_id).and_then(|run| { + let run = run.ok_or_else(|| format!("Agent Org run not found: {run_id}"))?; + AgentOrgRunStore::fail_starting( + run_id, + run.activation_generation, + &AgentOrgStartingFailure::new("starting_convergence_failed", message), + ) + .map(|_| ()) + }); + if let Err(mark_err) = failure_result { tracing::warn!( run_id = %run_id, error = %mark_err, @@ -179,10 +188,6 @@ pub(super) fn member_runtime_account_id( .or_else(|| fallback.clone()) } -pub(super) fn member_runtime_tier(config: Option<&OrgMemberRuntimeConfig>) -> Option { - config.and_then(|cfg| clean_runtime_value(cfg.tier.as_ref())) -} - pub(super) fn member_runtime_key_source( config: Option<&OrgMemberRuntimeConfig>, fallback: &KeySource, diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs index 3d621a898d..50c7e4560e 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs @@ -1,12 +1,17 @@ -//! Org-member materialization for the agent run launch service. +//! Receipt-driven Agent Org Starting convergence. //! -//! Handles spawning background tasks that create member sessions for an -//! Agent Org run, covering both Rust-native and CLI agent members. +//! The stable roster is committed before any Session row is created. Every +//! retry therefore targets the exact same member/session identity instead of +//! minting a replacement after a crash. + +use std::collections::HashMap; use core_types::key_source::KeySource; -use crate::coordination::agent_org_runs::AgentOrgRunStore; -use crate::definitions::orgs::{parse_cli_agent_org_reference, OrgDefinition}; +use crate::coordination::agent_org_runs::{ + AgentOrgMaterializationStatus, AgentOrgRunStore, AgentOrgStartingFailure, COORDINATOR_MEMBER_ID, +}; +use crate::definitions::orgs::{is_cli_agent_org_reference, OrgDefinition, OrgMember}; use crate::session::persistence::{ self as session_persistence, session_type, UnifiedSessionRecord, }; @@ -15,56 +20,46 @@ use crate::state::AgentAppState; use super::launch_helpers::{ flatten_org_members, member_runtime_account_id, member_runtime_key_source, - member_runtime_model, member_runtime_native_harness_type, member_runtime_tier, + member_runtime_model, member_runtime_native_harness_type, }; -#[allow(clippy::too_many_arguments)] -pub(super) fn spawn_agent_org_member_materialization( - org_run_id: String, - org: OrgDefinition, - root_session_id: String, - root_session_name: String, - workspace_path: String, - model: Option, - account_id: Option, - key_source: Option, - agent_exec_mode: Option, - native_harness_type: Option, - work_item_id: Option, - project_slug: Option, -) { - tokio::spawn(async move { - if let Err(err) = materialize_org_member_sessions( - &org_run_id, - &org, - &root_session_id, - &root_session_name, - &workspace_path, - model, - account_id, - key_source, - agent_exec_mode, - native_harness_type, - work_item_id, - project_slug, - ) - .await - { - tracing::warn!( - run_id = %org_run_id, - root_session_id = %root_session_id, - error = %err, - "[session_launch] failed to materialize Agent Org member sessions in background" - ); - if let Err(mark_err) = AgentOrgRunStore::mark_failed(&org_run_id, &err) { - tracing::warn!( - run_id = %org_run_id, - error = %mark_err, - "[session_launch] failed to mark Agent Org run failed after member materialization error" - ); - } +#[derive(Debug)] +pub(super) struct AgentOrgMaterializationError { + failure: AgentOrgStartingFailure, + retryable: bool, +} + +impl AgentOrgMaterializationError { + fn permanent(code: &'static str, message: impl Into) -> Self { + Self { + failure: AgentOrgStartingFailure::new(code, message), + retryable: false, } - }); + } + + fn retryable(message: impl Into) -> Self { + Self { + failure: AgentOrgStartingFailure::new( + "materialization_temporarily_unavailable", + message, + ), + retryable: true, + } + } + + pub(super) fn is_retryable(&self) -> bool { + self.retryable + } + + pub(super) fn failure(&self) -> &AgentOrgStartingFailure { + &self.failure + } +} + +impl std::fmt::Display for AgentOrgMaterializationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.failure.message) + } } #[allow(clippy::too_many_arguments)] @@ -81,22 +76,7 @@ pub(super) async fn materialize_org_member_sessions( native_harness_type: Option, work_item_id: Option, project_slug: Option, -) -> Result, String> { - let flattened_members = flatten_org_members(&org.children); - if flattened_members.is_empty() { - return Ok(Vec::new()); - } - - let mut rust_members = Vec::new(); - let mut cli_members = Vec::new(); - for member in flattened_members { - if parse_cli_agent_org_reference(&member.agent_id).is_some() { - cli_members.push(member); - } else { - rust_members.push(member); - } - } - +) -> Result, AgentOrgMaterializationError> { let workspace_path = workspace_path.to_string(); let root_session_id = root_session_id.to_string(); let org_name = org.name.clone(); @@ -105,197 +85,165 @@ pub(super) async fn materialize_org_member_sessions( .as_deref() .filter(|value| !value.trim().is_empty()) { - Some(raw) => KeySource::parse(raw).ok_or_else(|| format!("Unknown key_source: {raw:?}"))?, + Some(raw) => KeySource::parse(raw).ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "invalid_member_runtime_config", + format!("Unknown key_source: {raw:?}"), + ) + })?, None => KeySource::default(), }; let native_harness_type = native_harness_type .filter(|value| !value.trim().is_empty()) .map(|raw| { core_types::providers::NativeHarnessType::parse(&raw) - .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) + .ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "invalid_member_runtime_config", + format!("Unknown native_harness_type: {raw:?}"), + ) + }) .map(|parsed| parsed.as_str().to_string()) }) .transpose()?; let agent_exec_mode = agent_exec_mode.filter(|mode| !mode.trim().is_empty()); - let org_run_id = org_run_id.to_string(); - let mut created_session_ids = Vec::with_capacity(rust_members.len() + cli_members.len()); - let mut created_rust_session_ids = Vec::new(); - let mut created_cli_session_ids = Vec::new(); - - if !rust_members.is_empty() { - let rust_workspace_path = workspace_path.clone(); - let rust_root_session_id = root_session_id.clone(); - let rust_org_name = org_name.clone(); - let rust_model = model.clone(); - let rust_account_id = account_id.clone(); - let rust_key_source = key_source; - let rust_agent_exec_mode = agent_exec_mode.clone(); - let rust_native_harness_type = native_harness_type.clone(); - let rust_work_item_id = work_item_id.clone(); - let rust_project_slug = project_slug.clone(); - let rust_org_run_id = org_run_id.clone(); - created_rust_session_ids = tokio::task::spawn_blocking(move || { - let now = chrono::Utc::now().to_rfc3339(); - let mut created_session_ids: Vec = Vec::with_capacity(rust_members.len()); - let has_workspace_path = !rust_workspace_path.is_empty(); + let members = flatten_org_members(&org.children) + .into_iter() + .map(|member| (member.id.clone(), member)) + .collect::>(); + let receipts = AgentOrgRunStore::materializations(org_run_id) + .map_err(AgentOrgMaterializationError::retryable)?; + let mut materialized_session_ids = Vec::new(); + for receipt in receipts { + if receipt.member_id == COORDINATOR_MEMBER_ID { + continue; + } + if receipt.status == AgentOrgMaterializationStatus::Succeeded { + materialized_session_ids.push(receipt.session_id); + continue; + } + let member = members.get(&receipt.member_id).ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!( + "materialization receipt references missing canonical member {}", + receipt.member_id + ), + ) + })?; + if member.agent_id != receipt.agent_id { + return Err(AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!( + "materialization receipt agent mismatch for member {}", + receipt.member_id + ), + )); + } + if is_cli_agent_org_reference(&member.agent_id) { + return Err(AgentOrgMaterializationError::permanent( + "unsupported_member_runtime", + format!( + "CLI Agent Org member {} cannot be materialized on the canonical lifecycle path", + member.id + ), + )); + } - for member in rust_members { - let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( - Some(&member.agent_id), - has_workspace_path, - ); - let session_id = format!("{}{}", prefix, uuid::Uuid::new_v4()); - let member_config = member.runtime_config.as_ref(); - let member_model = member_runtime_model(member_config, &rust_model); - let member_account_id = member_runtime_account_id(member_config, &rust_account_id); - let member_key_source = member_runtime_key_source(member_config, &rust_key_source) - .map_err(|err| format!("invalid runtime config for member '{}': {}", member.name, err))?; - let member_native_harness_type = - member_runtime_native_harness_type(member_config, &rust_native_harness_type) - .map_err(|err| format!("invalid runtime config for member '{}': {}", member.name, err))?; - let session = UnifiedSessionRecord { - session_id: session_id.clone(), - name: format!("{} · {}", member.name, member.role), - status: crate::session::SessionStatus::Idle.as_str().to_string(), - model: member_model, - account_id: member_account_id, - workspace_path: Some(rust_workspace_path.clone()), - org_id: Some(project_management::projects::types::PERSONAL_ORG_ID.to_string()), - user_input: None, - total_tokens: 0, - created_at: now.clone(), - updated_at: now.clone(), - session_type: session_type::ORG_MEMBER.to_string(), - work_item_id: rust_work_item_id.clone(), - // Same rule as the launch resolver: a work-item-linked - // session is a Project session. Members inherit it so the - // PM tools aren't policy-denied for the team doing the work. - product_mode: rust_work_item_id - .as_ref() - .map(|_| "project".to_string()), - agent_role: Some(member.role.clone()), - project_slug: rust_project_slug.clone(), - agent_definition_id: Some(member.agent_id.clone()), - org_member_id: Some(member.id.clone()), - parent_session_id: Some(rust_root_session_id.clone()), - key_source: member_key_source, - agent_exec_mode: rust_agent_exec_mode.clone(), - native_harness_type: member_native_harness_type, - ..Default::default() - }; - if let Err(err) = session_persistence::upsert_session(&session) { - for created_session_id in &created_session_ids { - if let Err(cleanup_err) = session_persistence::delete_session(created_session_id) - { - tracing::warn!( - session_id = %created_session_id, - error = %cleanup_err, - "[session_launch] failed to clean up materialized Agent Org member session" - ); - } - } - return Err(format!( - "failed to materialize Agent Org member '{}' for run '{}': {}", - member.name, rust_org_run_id, err + let session_id = receipt.session_id.clone(); + let member = member.clone(); + let workspace_path = workspace_path.clone(); + let root_session_id = root_session_id.clone(); + let model = model.clone(); + let account_id = account_id.clone(); + let agent_exec_mode = agent_exec_mode.clone(); + let native_harness_type = native_harness_type.clone(); + let work_item_id = work_item_id.clone(); + let project_slug = project_slug.clone(); + let member_config = member.runtime_config.as_ref(); + let member_model = member_runtime_model(member_config, &model); + let member_account_id = member_runtime_account_id(member_config, &account_id); + let member_key_source = + member_runtime_key_source(member_config, &key_source).map_err(|error| { + AgentOrgMaterializationError::permanent("invalid_member_runtime_config", error) + })?; + let member_native_harness_type = + member_runtime_native_harness_type(member_config, &native_harness_type).map_err( + |error| { + AgentOrgMaterializationError::permanent("invalid_member_runtime_config", error) + }, + )?; + let persisted_session_id = tokio::task::spawn_blocking(move || { + if let Some(existing) = session_persistence::get_session(&session_id) + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))? + { + let identity_matches = existing.agent_definition_id.as_deref() + == Some(member.agent_id.as_str()) + && existing.org_member_id.as_deref() == Some(member.id.as_str()) + && existing.parent_session_id.as_deref() == Some(root_session_id.as_str()); + if !identity_matches { + return Err(AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!("stable materialization Session identity mismatch: {session_id}"), )); } - created_session_ids.push(session_id); + return Ok(session_id); } - - tracing::info!( - run_id = %rust_org_run_id, - org_name = %rust_org_name, - member_sessions = created_session_ids.len(), - "[session_launch] materialized Rust Agent Org member sessions" - ); - Ok(created_session_ids) - }) - .await - .map_err(|err| err.to_string())??; - created_session_ids.extend(created_rust_session_ids.iter().cloned()); - } - - for member in cli_members { - let cli_agent_type = parse_cli_agent_org_reference(&member.agent_id) - .ok_or_else(|| format!("invalid CLI Agent Org reference: {}", member.agent_id))? - .as_str() - .to_string(); - let member_config = member.runtime_config.as_ref(); - let member_key_source = member_runtime_key_source(member_config, &key_source)?; - let outcome = crate::foundation::session_bridge::launch_cli_agent( - crate::foundation::session_bridge::CliLaunchParams { - name: Some(format!("{} · {}", member.name, member.role)), - cli_agent_type, - model: member_runtime_model(member_config, &model), - tier: member_runtime_tier(member_config), - account_id: member_runtime_account_id(member_config, &account_id), - repo_path: Some(workspace_path.clone()).filter(|path| !path.is_empty()), - branch: None, - worktree_path: None, - worktree_base_ref: None, - hosted_token: None, - isolate: false, - background: true, - key_source: Some(member_key_source.as_ref().to_string()), - additional_directories: None, - parent_session_id: Some(root_session_id.clone()), - org_member_id: Some(member.id.clone()), - org_id: project_management::projects::types::PERSONAL_ORG_ID.to_string(), - project_id: None, - project_name: None, - project_slug: project_slug.clone(), + let now = chrono::Utc::now().to_rfc3339(); + let session = UnifiedSessionRecord { + session_id: session_id.clone(), + name: format!("{} · {}", member.name, member.role), + status: crate::session::SessionStatus::Idle.as_str().to_string(), + model: member_model, + account_id: member_account_id, + workspace_path: Some(workspace_path), + org_id: Some(project_management::projects::types::PERSONAL_ORG_ID.to_string()), + user_input: None, + total_tokens: 0, + created_at: now.clone(), + updated_at: now, + session_type: session_type::ORG_MEMBER.to_string(), work_item_id: work_item_id.clone(), - agent_role: None, product_mode: work_item_id.as_ref().map(|_| "project".to_string()), - durable_run_id: None, - user_input: String::new(), - ide_context: None, - mode: agent_exec_mode.clone(), - images: None, - }, + agent_role: Some(member.role), + project_slug, + agent_definition_id: Some(member.agent_id), + org_member_id: Some(member.id), + parent_session_id: Some(root_session_id), + key_source: member_key_source, + agent_exec_mode, + native_harness_type: member_native_harness_type, + ..Default::default() + }; + session_persistence::upsert_session(&session) + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))?; + Ok::<_, AgentOrgMaterializationError>(session_id) + }) + .await + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))??; + AgentOrgRunStore::mark_materialization_succeeded( + org_run_id, + &receipt.member_id, + receipt.generation, + &persisted_session_id, ) - .await; - match outcome { - Ok(outcome) => { - created_session_ids.push(outcome.session_id.clone()); - created_cli_session_ids.push(outcome.session_id); - } - Err(err) => { - for session_id in &created_rust_session_ids { - if let Err(cleanup_err) = session_persistence::delete_session(session_id) { - tracing::warn!( - session_id = %session_id, - error = %cleanup_err, - "[session_launch] failed to clean up Rust Agent Org member session after CLI materialization failure" - ); - } - } - for session_id in &created_cli_session_ids { - if let Err(cleanup_err) = - crate::foundation::session_bridge::delete_cli_session(session_id) - { - tracing::warn!( - session_id = %session_id, - error = %cleanup_err, - "[session_launch] failed to clean up CLI Agent Org member session" - ); - } - } - return Err(format!( - "failed to materialize CLI Agent Org member '{}' for run '{}': {}", - member.name, org_run_id, err - )); + .map_err(|error| { + if error.contains("identity mismatch") || error.contains("Session identity") { + AgentOrgMaterializationError::permanent("materialization_identity_mismatch", error) + } else { + AgentOrgMaterializationError::retryable(error) } - } + })?; + materialized_session_ids.push(persisted_session_id); } tracing::info!( run_id = %org_run_id, org_name = %org_name, - member_sessions = created_session_ids.len(), - "[session_launch] materialized Agent Org member sessions" + member_sessions = materialized_session_ids.len(), + "[session_launch] converged Agent Org member materialization receipts" ); - Ok(created_session_ids) + Ok(materialized_session_ids) } #[allow(clippy::too_many_arguments)] @@ -313,11 +261,11 @@ pub(super) async fn send_initial_turn( agent_definition_id: Option, sub_agent_ids: Vec, intent_org_run_id: Option, - durable_run_id: Option, + client_message_id: Option, + turn_intent_id: Option, source: crate::foundation::session_bridge::TurnIntentBridgeSource, ) -> Result<(), String> { if sub_agent_ids.is_empty() { - let client_message_id = durable_run_id.clone(); crate::state::commands::session::message::send_message_impl( state, session_id.to_string(), @@ -335,7 +283,7 @@ pub(super) async fn send_initial_turn( false, false, client_message_id, - durable_run_id, + turn_intent_id, None, intent_org_run_id, source, @@ -357,7 +305,6 @@ pub(super) async fn send_initial_turn( .await?; crate::init::init_session(state, launch_spec).await?; - let client_message_id = durable_run_id.clone(); crate::state::commands::session::message::send_message_impl( state, session_id.to_string(), @@ -375,7 +322,7 @@ pub(super) async fn send_initial_turn( false, false, client_message_id, - durable_run_id, + turn_intent_id, None, intent_org_run_id, crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs index 08813c2dd2..00c608615c 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs @@ -1,7 +1,7 @@ use super::launch_helpers::{ apply_member_launch_overrides_to_snapshot, member_runtime_account_id, member_runtime_key_source, member_runtime_model, member_runtime_native_harness_type, - member_runtime_tier, validate_launch_agent_definitions, + validate_launch_agent_definitions, }; use crate::coordination::agent_org_runs::COORDINATOR_MEMBER_ID; use crate::definitions::builtin::SDE_AGENT_ID; @@ -145,10 +145,6 @@ fn member_runtime_resolution_prefers_member_config_then_falls_back() { member_runtime_account_id(Some(&config), &fallback_account).as_deref(), Some("member-account") ); - assert_eq!( - member_runtime_tier(Some(&config)).as_deref(), - Some("premium") - ); assert_eq!( member_runtime_key_source(Some(&config), &KeySource::OwnKey).expect("key source"), KeySource::HostedKey diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index 3f9b5d514d..72d642932b 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -17,8 +17,8 @@ use std::collections::HashMap; use tauri::Manager; use crate::coordination::agent_org_runs::{ - AgentOrgRunEntryMode, AgentOrgRunStatus, AgentOrgRunStore, CreateAgentOrgRunParams, - COORDINATOR_MEMBER_ID, + AgentOrgRunEntryMode, AgentOrgRunStore, CreateAgentOrgInitialInput, + CreateAgentOrgMaterializationIntent, CreateStartingAgentOrgRunParams, COORDINATOR_MEMBER_ID, }; use crate::definitions::orgs::{AgentOrgsStore, OrgMemberLaunchOverride}; use crate::init::launch_spec::AgentLaunchSpec; @@ -28,12 +28,13 @@ use crate::state::AgentAppState; use project_management::projects::types as project_types; use launch_helpers::{ - apply_member_launch_overrides_to_snapshot, derive_name, handle_background_launch_failure, - provenance_fields, provenance_lock_reason, validate_launch_agent_definitions, + apply_member_launch_overrides_to_snapshot, derive_name, flatten_org_members, + handle_background_launch_failure, provenance_fields, provenance_lock_reason, + validate_launch_agent_definitions, }; use launch_org::{ - cleanup_session_after_org_run_create_failure, send_initial_turn, - spawn_agent_org_member_materialization, + cleanup_session_after_org_run_create_failure, materialize_org_member_sessions, + send_initial_turn, }; use launch_workspace::{ acquire_work_item_execution_lock, prepare_rust_agent_workspace_for_launch, @@ -42,6 +43,36 @@ use launch_workspace::{ pub(crate) const MAX_AUTO_NAME_LEN: usize = 80; +const AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION: u8 = 1; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct AgentOrgInitialInputPayload { + version: u8, + images: Option>, + ide_context: Option, + sub_agent_ids: Vec, +} + +fn decode_agent_org_initial_input_payload( + input: &crate::coordination::agent_org_runs::AgentOrgInitialInput, +) -> Result { + let payload: AgentOrgInitialInputPayload = + serde_json::from_str(&input.payload_json).map_err(|error| { + format!( + "invalid Starting initial input payload for {}: {error}", + input.org_run_id + ) + })?; + if payload.version != AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION { + return Err(format!( + "unsupported Starting initial input payload version {} for {}", + payload.version, input.org_run_id + )); + } + Ok(payload) +} + #[derive(Debug, Clone)] pub(crate) struct AgentRunLaunchRequest { /// Stable WorkItemRun id used for deterministic Session and turn ids. @@ -405,6 +436,9 @@ pub(crate) async fn launch_rust_agent_run( org_store: Option<&AgentOrgsStore>, request: AgentRunLaunchRequest, ) -> Result { + if matches!(&request.target, AgentRunTarget::AgentOrg { .. }) { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; + } let (workspace_path, branch, isolate, existing_worktree_path, additional_directories) = match &request.workspace { WorkspaceLaunchTarget::LocalWorkspace { @@ -530,6 +564,91 @@ pub(crate) async fn launch_rust_agent_run( .as_ref() .map(|entry| entry.branch.clone()); + let has_initial_content = !request.content.trim().is_empty(); + let org_session_key = agent_org_id.as_ref().map(|_| { + request + .durable_run_id + .clone() + .unwrap_or_else(|| format!("agent-org-{}", uuid::Uuid::new_v4())) + }); + let expected_root_session_id = org_session_key.as_ref().map(|key| { + let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( + agent_definition_id.as_deref(), + !workspace_path.is_empty(), + ); + format!("{prefix}{key}") + }); + let initial_turn_intent_id = + has_initial_content.then(|| format!("agent-org-initial-turn-{}", uuid::Uuid::new_v4())); + let initial_message_id = + has_initial_content.then(|| format!("agent-org-initial-message-{}", uuid::Uuid::new_v4())); + let initial_input_payload_json = has_initial_content + .then(|| { + serde_json::to_string(&AgentOrgInitialInputPayload { + version: AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION, + images: request.images.clone(), + ide_context: request.ide_context.clone(), + sub_agent_ids: request.sub_agent_ids.clone(), + }) + .map_err(|error| format!("serialize Agent Org initial input: {error}")) + }) + .transpose()?; + + // Build the construction envelope before any Team lifecycle row exists. + // The already-persisted coordinator is certified in the same transaction + // as Starting and the remaining stable member identities. + let starting_params = match ( + agent_org_id.as_ref(), + coordinator_agent_id.as_ref(), + effective_org_definition.as_ref(), + expected_root_session_id.as_ref(), + ) { + (Some(org_id), Some(coordinator_id), Some(org_snapshot), Some(root_session_id)) => { + let mut materialization_intents = vec![CreateAgentOrgMaterializationIntent { + member_id: COORDINATOR_MEMBER_ID.to_string(), + agent_id: coordinator_id.clone(), + session_id: root_session_id.clone(), + succeeded: true, + }]; + for member in flatten_org_members(&org_snapshot.children) { + let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( + Some(&member.agent_id), + !workspace_path.is_empty(), + ); + materialization_intents.push(CreateAgentOrgMaterializationIntent { + member_id: member.id, + agent_id: member.agent_id, + session_id: format!("{prefix}{}", uuid::Uuid::new_v4()), + succeeded: false, + }); + } + Some(CreateStartingAgentOrgRunParams { + org_id: org_id.clone(), + coordinator_agent_id: coordinator_id.clone(), + root_session_id: root_session_id.clone(), + org_snapshot: org_snapshot.clone(), + entry_mode: AgentOrgRunEntryMode::StandaloneSession, + work_item_id: work_item_id.clone(), + project_slug: project_slug.clone(), + routine_fire_id: routine_fire_id.clone(), + materialization_intents, + initial_input: initial_turn_intent_id.as_ref().map(|turn_intent_id| { + CreateAgentOrgInitialInput { + turn_intent_id: turn_intent_id.clone(), + message_id: initial_message_id + .clone() + .expect("initial message id accompanies initial turn"), + content: request.content.clone(), + payload_json: initial_input_payload_json + .clone() + .expect("initial payload accompanies initial turn"), + } + }), + }) + } + _ => None, + }; + let create_result = crate::state::commands::session::create::create_session_impl( None, workspace_path.clone(), @@ -549,7 +668,9 @@ pub(crate) async fn launch_rust_agent_run( request.product_mode.clone(), request.resources.native_harness_type.clone(), request.parent_session_id.clone(), - request.durable_run_id.clone(), + org_session_key + .clone() + .or_else(|| request.durable_run_id.clone()), ) .await?; @@ -558,11 +679,34 @@ pub(crate) async fn launch_rust_agent_run( .and_then(|value| value.as_str()) .ok_or("create_session_impl did not return sessionId")? .to_string(); + if let Some(expected_session_id) = expected_root_session_id.as_deref() { + if session_id != expected_session_id { + return Err(format!( + "coordinator identity mismatch: expected {expected_session_id}, got {session_id}" + )); + } + } let resolved_product_mode = create_result .get("productMode") .and_then(|value| value.as_str()) .map(str::to_string); + if starting_params.is_some() { + persistence::update_org_member_id(&session_id, COORDINATOR_MEMBER_ID) + .map_err(|err| format!("failed to persist coordinator member_id: {err}"))?; + } + + let starting_run = match starting_params { + Some(params) => match AgentOrgRunStore::create_starting(params) { + Ok(run) => Some(run), + Err(error) => { + cleanup_session_after_org_run_create_failure(session_id.clone()).await; + return Err(error); + } + }, + None => None, + }; + if let (Some(project_slug_value), Some(work_item_id_value)) = (project_slug.as_deref(), work_item_id.as_deref()) { @@ -580,61 +724,14 @@ pub(crate) async fn launch_rust_agent_run( } } - let agent_org_run_id = match (agent_org_id.as_ref(), coordinator_agent_id.as_ref()) { - (Some(org_id), Some(coordinator_id)) => { - let org_snapshot = effective_org_definition - .as_ref() - .ok_or("Agent Org launch is missing resolved org definition")? - .clone(); - let run = AgentOrgRunStore::create(CreateAgentOrgRunParams { - org_id: org_id.clone(), - coordinator_agent_id: coordinator_id.clone(), - root_session_id: Some(session_id.clone()), - org_snapshot, - entry_mode: AgentOrgRunEntryMode::StandaloneSession, - status: AgentOrgRunStatus::Running, - work_item_id: work_item_id.clone(), - project_slug: project_slug.clone(), - routine_fire_id, - }); - match run { - Ok(record) => { - persistence::update_org_member_id(&session_id, COORDINATOR_MEMBER_ID) - .map_err(|err| format!("failed to persist coordinator member_id: {err}"))?; - if let Some(org) = effective_org_definition.as_ref() { - spawn_agent_org_member_materialization( - record.id.clone(), - org.clone(), - session_id.clone(), - name.clone(), - workspace_path.clone(), - request.resources.model.clone(), - request.resources.account_id.clone(), - request.resources.key_source.clone(), - request.mode.clone(), - request.resources.native_harness_type.clone(), - work_item_id.clone(), - project_slug.clone(), - ); - } - if apply_member_overrides_for_future { - if let Some(store) = org_store { - store.apply_member_launch_overrides(org_id, &member_overrides)?; - } - } - Some(record.id) - } - Err(err) => { - cleanup_session_after_org_run_create_failure(session_id.clone()).await; - return Err(err); - } - } + let agent_org_run_id = starting_run.as_ref().map(|run| run.id.clone()); + if starting_run.is_some() && apply_member_overrides_for_future { + if let (Some(store), Some(org_id)) = (org_store, agent_org_id.as_ref()) { + store.apply_member_launch_overrides(org_id, &member_overrides)?; } - _ => None, - }; + } let created_at = chrono::Utc::now().to_rfc3339(); - let has_initial_content = !request.content.trim().is_empty(); let native_harness_type_for_send = request .resources .native_harness_type @@ -662,10 +759,20 @@ pub(crate) async fn launch_rust_agent_run( let sub_agent_ids_for_send = request.sub_agent_ids.clone(); let agent_definition_id_for_send = agent_definition_id.clone(); let agent_org_run_id_for_background = agent_org_run_id.clone(); - let durable_run_id_for_background = request.durable_run_id.clone(); let project_slug_for_background = project_slug.clone(); let work_item_id_for_background = work_item_id.clone(); let app_handle_for_background = state.app_handle.clone(); + let request_key_source_for_background = request.resources.key_source.clone(); + let request_native_harness_for_background = request.resources.native_harness_type.clone(); + let org_for_background = effective_org_definition + .clone() + .expect("Agent Org launch has a validated snapshot"); + let starting_generation = starting_run + .as_ref() + .map(|run| run.activation_generation) + .expect("Agent Org launch has a Starting generation"); + let initial_turn_intent_id_for_background = initial_turn_intent_id.clone(); + let initial_message_id_for_background = initial_message_id.clone(); tokio::spawn(async move { let prepared_workspace = match prepare_rust_agent_workspace_for_launch( @@ -699,14 +806,165 @@ pub(crate) async fn launch_rust_agent_run( } }; - if !has_initial_content { - return; - } - let workspace_path_for_send = prepared_workspace .worktree_path .clone() .unwrap_or_else(|| workspace_path_for_background.clone()); + write_agent_session_marker( + &workspace_path_for_send, + &session_id_for_background, + agent_definition_id_for_send.as_deref(), + None, + project_slug_for_background.as_deref(), + Some(project_management::projects::types::PERSONAL_ORG_ID), + ); + + let run_id = agent_org_run_id_for_background + .as_deref() + .expect("Agent Org background launch has a run id"); + if let Err(err) = materialize_org_member_sessions( + run_id, + &org_for_background, + &session_id_for_background, + &name, + &workspace_path_for_send, + model_for_send.clone(), + account_id_for_send.clone(), + request_key_source_for_background.clone(), + mode_for_send.clone(), + request_native_harness_for_background.clone(), + work_item_id_for_background.clone(), + project_slug_for_background.clone(), + ) + .await + { + if err.is_retryable() { + tracing::warn!( + run_id = %run_id, + error = %err, + "[session_launch] retryable Starting materialization deferred" + ); + return; + } + let message = format!( + "[session_launch] member materialization failed for {}: {}", + session_id_for_background, err + ); + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &message, + "[session_launch] failed to mark Starting materialization failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + + if let Some(turn_intent_id) = initial_turn_intent_id_for_background.as_deref() { + let input = match AgentOrgRunStore::initial_input(run_id) { + Ok(Some(input)) => input, + Ok(None) => { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + "Starting initial input receipt is missing", + "[session_launch] failed to mark missing Starting input", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + Err(error) => { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &error, + "[session_launch] failed to mark Starting input lookup failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + }; + let session_id_for_persistence = session_id_for_background.clone(); + let input_for_persistence = input.clone(); + let transcript_result = tokio::task::spawn_blocking(move || { + persistence::save_user_msg_with_id( + &input_for_persistence.message_id, + &session_id_for_persistence, + &input_for_persistence.content, + ) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string()) + .and_then(|result| result); + let event_result = app_handle_for_background + .as_ref() + .ok_or_else(|| "App handle is unavailable during Starting".to_string()) + .and_then(|handle| { + crate::bus::event_pipeline_bridge::persist_user_message_event( + handle, + &session_id_for_background, + &input.message_id, + &input.content, + None, + images_for_send.as_deref(), + crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, + turn_intent_id, + ) + }); + if let Err(error) = transcript_result.and(event_result) { + tracing::warn!( + run_id = %run_id, + error = %error, + "[session_launch] retryable Starting input persistence deferred" + ); + return; + } + } + + if let Err(error) = AgentOrgRunStore::finish_starting(run_id, starting_generation) { + if error.starts_with("materialization_identity_mismatch:") + || error.contains("initial input certificate missing") + || error.contains("unexpected initial input certificate") + { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &format!("Starting convergence failed: {error}"), + "[session_launch] failed to mark Starting convergence failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + } else { + tracing::warn!( + run_id = %run_id, + error = %error, + "[session_launch] retryable Starting convergence deferred" + ); + } + return; + } + + if !has_initial_content { + return; + } + // Title generation runs concurrently — it must not delay the // first turn. See `spawn_session_title_generation`. spawn_session_title_generation( @@ -732,7 +990,8 @@ pub(crate) async fn launch_rust_agent_run( agent_definition_id_for_send, sub_agent_ids_for_send, agent_org_run_id_for_background.clone(), - durable_run_id_for_background, + initial_message_id_for_background.clone(), + initial_turn_intent_id_for_background.clone(), crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, ) .await; @@ -753,6 +1012,16 @@ pub(crate) async fn launch_rust_agent_run( "[session_launch] failed to mark session failed after first-message error", ) .await; + } else if let Some(turn_intent_id) = initial_turn_intent_id_for_background.as_deref() { + if let Err(error) = + AgentOrgRunStore::mark_initial_input_dispatched(run_id, turn_intent_id) + { + tracing::warn!( + run_id, + error = %error, + "[session_launch] initial input was accepted but dispatch receipt update failed" + ); + } } }); @@ -883,6 +1152,7 @@ pub(crate) async fn launch_rust_agent_run( agent_definition_id_for_send, sub_agent_ids_for_send, agent_org_run_id_for_send.clone(), + durable_run_id_for_send.clone(), durable_run_id_for_send, crate::foundation::session_bridge::TurnIntentBridgeSource::UserSubmit, ) @@ -945,3 +1215,259 @@ pub(crate) async fn launch_rust_agent_run( product_mode: resolved_product_mode, }) } + +const AGENT_ORG_STARTUP_RECOVERY_LIMIT: usize = 100; + +/// Run the one-shot Starting/initial-input recovery owner after app state and +/// the EventStore bridge are ready. This is intentionally separate from the +/// periodic Working watchdog. +pub fn spawn_agent_org_startup_recovery(state: AgentAppState) { + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return; + } + tauri::async_runtime::spawn(async move { + if let Err(error) = recover_agent_org_starting_runs(&state).await { + tracing::warn!(error = %error, "[agent-org-startup] Starting recovery failed"); + } + if let Err(error) = recover_agent_org_initial_dispatches(&state).await { + tracing::warn!(error = %error, "[agent-org-startup] initial dispatch recovery failed"); + } + }); +} + +async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), String> { + let runs = tokio::task::spawn_blocking(|| { + AgentOrgRunStore::list_starting_runs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) + }) + .await + .map_err(|error| error.to_string())??; + + for run in runs { + let Some(root_session_id) = run.root_session_id.as_deref() else { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_coordinator_identity", + "Starting run has no coordinator Session identity", + ), + )?; + continue; + }; + let root = persistence::get_session(root_session_id) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("Starting coordinator Session is missing: {root_session_id}")); + let root = match root { + Ok(root) => root, + Err(message) => { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_coordinator_identity", + message, + ), + )?; + continue; + } + }; + let Some(snapshot_raw) = run.org_snapshot_json.as_deref() else { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_launch_snapshot", + format!("Starting run {} has no launch snapshot", run.id), + ), + )?; + continue; + }; + let snapshot: crate::definitions::orgs::OrgDefinition = + match serde_json::from_str(snapshot_raw) { + Ok(snapshot) => snapshot, + Err(error) => { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "invalid_launch_snapshot", + error.to_string(), + ), + )?; + continue; + } + }; + let workspace_path = root + .worktree_path + .clone() + .or_else(|| root.workspace_path.clone()) + .unwrap_or_default(); + if let Err(error) = materialize_org_member_sessions( + &run.id, + &snapshot, + root_session_id, + &root.name, + &workspace_path, + root.model.clone(), + root.account_id.clone(), + Some(root.key_source.as_ref().to_string()), + root.agent_exec_mode.clone(), + root.native_harness_type.clone(), + root.work_item_id.clone(), + root.project_slug.clone(), + ) + .await + { + if !error.is_retryable() { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + error.failure(), + )?; + continue; + } + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] materialization retry deferred"); + continue; + } + + let initial_input = match AgentOrgRunStore::initial_input(&run.id) { + Ok(input) => input, + Err(error) => { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input lookup retry deferred"); + continue; + } + }; + if let Some(input) = initial_input { + if let Err(error) = persist_starting_initial_input(state, root_session_id, &input).await + { + if error.starts_with("invalid Starting initial input payload") + || error.starts_with("unsupported Starting initial input payload version") + { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "invalid_initial_input_payload", + error, + ), + )?; + } else { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input persistence retry deferred"); + } + continue; + } + } + if let Err(error) = AgentOrgRunStore::finish_starting(&run.id, run.activation_generation) { + if error.starts_with("materialization_identity_mismatch:") + || error.contains("initial input certificate missing") + || error.contains("unexpected initial input certificate") + { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "starting_certificate_invalid", + error, + ), + )?; + } else { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] Starting transition retry deferred"); + } + } + } + Ok(()) +} + +async fn recover_agent_org_initial_dispatches(state: &AgentAppState) -> Result<(), String> { + let inputs = tokio::task::spawn_blocking(|| { + AgentOrgRunStore::recoverable_initial_inputs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) + }) + .await + .map_err(|error| error.to_string())??; + + for input in inputs { + let Some(run) = AgentOrgRunStore::load(&input.org_run_id)? else { + continue; + }; + let Some(root_session_id) = run.root_session_id.as_deref() else { + continue; + }; + let Some(root) = + persistence::get_session(root_session_id).map_err(|error| error.to_string())? + else { + continue; + }; + persistence::update_status(root_session_id, crate::session::SessionStatus::Idle) + .map_err(|error| error.to_string())?; + let workspace_path = root + .worktree_path + .clone() + .or(root.workspace_path.clone()) + .unwrap_or_default(); + let native_harness_type = root + .native_harness_type + .as_deref() + .map(|raw| { + core_types::providers::NativeHarnessType::parse(raw) + .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) + }) + .transpose()?; + let payload = decode_agent_org_initial_input_payload(&input)?; + send_initial_turn( + state, + root_session_id, + input.content.clone(), + root.model, + root.account_id, + workspace_path, + native_harness_type, + root.agent_exec_mode, + payload.images, + payload.ide_context, + Some(run.coordinator_agent_id), + payload.sub_agent_ids, + Some(run.id.clone()), + Some(input.message_id.clone()), + Some(input.turn_intent_id.clone()), + crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, + ) + .await?; + AgentOrgRunStore::mark_initial_input_dispatched(&run.id, &input.turn_intent_id)?; + } + Ok(()) +} + +async fn persist_starting_initial_input( + state: &AgentAppState, + session_id: &str, + input: &crate::coordination::agent_org_runs::AgentOrgInitialInput, +) -> Result<(), String> { + let payload = decode_agent_org_initial_input_payload(input)?; + let session_id_owned = session_id.to_string(); + let input_owned = input.clone(); + tokio::task::spawn_blocking(move || { + persistence::save_user_msg_with_id( + &input_owned.message_id, + &session_id_owned, + &input_owned.content, + ) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string())??; + let handle = state + .app_handle + .as_ref() + .ok_or_else(|| "App handle is unavailable during Starting recovery".to_string())?; + crate::bus::event_pipeline_bridge::persist_user_message_event( + handle, + session_id, + &input.message_id, + &input.content, + None, + payload.images.as_deref(), + crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, + &input.turn_intent_id, + ) +} diff --git a/src-tauri/crates/agent-core/src/core/session/scheduler.rs b/src-tauri/crates/agent-core/src/core/session/scheduler.rs index 29e6fed79a..75a4ab51f6 100644 --- a/src-tauri/crates/agent-core/src/core/session/scheduler.rs +++ b/src-tauri/crates/agent-core/src/core/session/scheduler.rs @@ -100,7 +100,7 @@ pub struct ScheduledMessage { /// user-message persistence (resume with empty content). pub turn_intent_id: String, /// Durable Agent Org run that owns this turn, when any. The scheduler - /// uses this only after the intent reaches a terminal state so finality + /// uses this only after the intent reaches a terminal state so Quiescence /// is rechecked after (not before) the current intent stops blocking it. pub org_run_id: Option, /// The user content to process. @@ -500,8 +500,22 @@ impl WorkerTask { if let Some(run_id) = org_run_id { let reconcile_run_id = run_id.clone(); match tokio::task::spawn_blocking(move || { - crate::coordination::agent_org_runs::AgentOrgRunStore::reconcile_run_finality( + let assessment = crate::coordination::agent_org_runs::AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + crate::coordination::agent_org_runs::AgentOrgRunStore::try_transition_working_to_idle( &reconcile_run_id, + generation, + work_revision, ) }) .await @@ -510,12 +524,12 @@ impl WorkerTask { Ok(Err(error)) => warn!( run_id = %run_id, error = %error, - "[scheduler] post-intent Agent Org finality reconcile failed" + "[scheduler] post-intent Agent Org quiescence reconcile failed" ), Err(error) => warn!( run_id = %run_id, error = %error, - "[scheduler] post-intent Agent Org finality reconcile task failed" + "[scheduler] post-intent Agent Org quiescence reconcile task failed" ), } } @@ -527,16 +541,23 @@ impl WorkerTask { "[scheduler] Message {} failed for session {}: {}", msg.message_id, self.session_id, err ); - // Lifecycle: running → failed. Cancelled turns walk - // here too (the executor returns Err on user stop); a - // future commit can distinguish via the cancel_flag - // probe if we need a separate `cancelled` bucket on - // the round renderer. - crate::foundation::session_bridge::update_turn_intent_status( - &self.session_id, - &turn_intent_id, - crate::foundation::session_bridge::TurnIntentBridgeStatus::Failed, - ); + let assistant_persistence_failed = + should_keep_agent_org_intent_in_flight(org_run_id.as_deref(), err); + if assistant_persistence_failed { + warn!( + session_id = %self.session_id, + turn_intent_id = %turn_intent_id, + "[scheduler] keeping Agent Org turn in-flight because final assistant persistence failed" + ); + } else { + // Lifecycle: running → failed. Cancelled turns walk + // here too (the executor returns Err on user stop). + crate::foundation::session_bridge::update_turn_intent_status( + &self.session_id, + &turn_intent_id, + crate::foundation::session_bridge::TurnIntentBridgeStatus::Failed, + ); + } // Turn-only: an `agent:error` renders as a chat bubble. // Maintenance jobs report failures through their own // channel (e.g. the manual-compact command's reply). @@ -578,11 +599,35 @@ impl WorkerTask { } } +fn should_keep_agent_org_intent_in_flight(org_run_id: Option<&str>, error: &str) -> bool { + org_run_id.is_some() + && error.starts_with( + crate::core::session::turn::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX, + ) +} + #[cfg(test)] mod tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; + #[test] + fn assistant_persistence_failure_keeps_only_agent_org_intent_in_flight() { + let error = format!( + "{} disk full", + crate::core::session::turn::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX + ); + assert!(should_keep_agent_org_intent_in_flight( + Some("run-1"), + &error + )); + assert!(!should_keep_agent_org_intent_in_flight(None, &error)); + assert!(!should_keep_agent_org_intent_in_flight( + Some("run-1"), + "provider failed" + )); + } + #[tokio::test] async fn invalidated_pending_message_is_skipped() { let scheduler = DialogScheduler::new("session-a", 8); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs index e265b20972..c82ed01eba 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs @@ -162,6 +162,7 @@ pub async fn process_message( .as_ref() .and_then(|ctx| ctx.repo_path.clone()), agent_org_task_lifecycle: None, + require_durable_assistant_event: false, }; let policy = Arc::clone(&runtime.policy); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs index fc7b33cb98..b7ea16a225 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs @@ -52,6 +52,9 @@ use core_types::session_event::SessionEvent; use super::super::persistence as unified_persistence; +pub(crate) const AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX: &str = + "agent_org_assistant_persistence_failed:"; + fn tool_result_is_error(result: &str) -> bool { if result.starts_with("Error") { return true; @@ -123,6 +126,10 @@ pub struct EventHandlerConfig { /// Agent Org worker identity used by the bounded task-lifecycle stop gate. /// Coordinators and non-org sessions leave this unset. pub agent_org_task_lifecycle: Option, + + /// Agent Org work-capable turns may not become terminal until their + /// assistant EventStore rows are durably committed. + pub require_durable_assistant_event: bool, } /// Durable identity needed to verify that an Agent Org worker did not end a @@ -170,6 +177,7 @@ pub struct UnifiedEventHandler { /// A second miss is reported durably to the coordinator by `MemberIdle` /// rather than looping the provider. agent_org_lifecycle_correction_emitted: AtomicBool, + assistant_persistence_error: Mutex>, } /// Accumulated state for one streaming `create_plan` call. @@ -242,6 +250,7 @@ impl UnifiedEventHandler { plan_draft_streams: Mutex::new(std::collections::HashMap::new()), last_context_tokens: std::sync::atomic::AtomicI64::new(0), agent_org_lifecycle_correction_emitted: AtomicBool::new(false), + assistant_persistence_error: Mutex::new(None), } } @@ -279,7 +288,7 @@ impl UnifiedEventHandler { sessions.insert(session_id.to_string()); } self.track_retractable_segment(session_id, &event.id); - self.push_to_store(session_id, event.clone()); + self.push_to_store_durable_assistant(session_id, event.clone()); broadcast_event( "agent:streaming_complete", serde_json::json!({ @@ -310,6 +319,38 @@ impl UnifiedEventHandler { self.agent_called.load(Ordering::Relaxed) } + pub fn take_assistant_persistence_error(&self) -> Option { + self.assistant_persistence_error + .lock() + .ok() + .and_then(|mut error| error.take()) + } + + fn record_assistant_persistence_error(&self, error: String) { + if let Ok(mut slot) = self.assistant_persistence_error.lock() { + if slot.is_none() { + *slot = Some(error); + } + } + } + + fn push_to_store_durable_assistant(&self, session_id: &str, event: SessionEvent) { + if self.is_cancelled() || !self.is_current_turn_generation() { + return; + } + if self.config.require_durable_assistant_event { + if let Err(error) = event_pipeline_bridge::persist_events( + "agent-org-assistant-final", + session_id, + std::slice::from_ref(&event), + 5, + ) { + self.record_assistant_persistence_error(error); + } + } + self.push_to_store(session_id, event); + } + /// Push a SessionEvent into the session's EventStore so frontend /// subscribers receive it via `es:changed`. Silently no-op when the /// handler was constructed without an app handle (tests / non-Tauri @@ -743,6 +784,11 @@ impl TurnEventHandler for UnifiedEventHandler { "[unified_handler] Failed to persist assistant iteration: {}", err ); + if self.config.require_durable_assistant_event { + self.record_assistant_persistence_error(format!( + "assistant transcript persistence failed: {err}" + )); + } } let has_active_message_stream = self @@ -761,7 +807,7 @@ impl TurnEventHandler for UnifiedEventHandler { ) { let mut event = event_factory::build_assistant_message_event(session_id, text); attach_turn_id(&mut event, self.config.turn_id.as_deref()); - self.push_to_store(session_id, event); + self.push_to_store_durable_assistant(session_id, event); } } @@ -877,7 +923,7 @@ impl TurnEventHandler for UnifiedEventHandler { // steering queue and is about to be presented to the model. // Never leave its durable intent queued merely because the // transcript write failed: that would block Agent Org - // finality forever. Failed is terminal and truthfully records + // Quiescence forever. Failed is terminal and truthfully records // that durable persistence did not complete. crate::foundation::session_bridge::update_turn_intent_status( session_id, @@ -1140,6 +1186,39 @@ mod tests { assert!(should_push_assistant_event(false, false, true)); } + #[test] + fn agent_org_assistant_persistence_failure_is_retained_for_turn_owner() { + let handler = UnifiedEventHandler::new(EventHandlerConfig { + require_durable_assistant_event: true, + ..Default::default() + }); + let event = super::event_factory::build_assistant_message_event( + "agent-org-session", + "durable final answer", + ); + + handler.push_to_store_durable_assistant("agent-org-session", event); + + let error = handler + .take_assistant_persistence_error() + .expect("unregistered durable EventStore bridge must fail closed"); + assert!(error.contains("event pipeline persistence is not registered")); + assert!(handler.take_assistant_persistence_error().is_none()); + } + + #[test] + fn generic_assistant_event_does_not_require_synchronous_eventstore_commit() { + let handler = UnifiedEventHandler::new(EventHandlerConfig::default()); + let event = super::event_factory::build_assistant_message_event( + "generic-session", + "ordinary answer", + ); + + handler.push_to_store_durable_assistant("generic-session", event); + + assert!(handler.take_assistant_persistence_error().is_none()); + } + #[test] fn retractable_segments_drained_once_on_retry() { let handler = UnifiedEventHandler::new(EventHandlerConfig::default()); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs index 2a854a3fde..744d0c2d52 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs @@ -101,6 +101,8 @@ impl UnifiedMessageProcessor { let mut event_handler_config = self.event_handler_config.clone(); event_handler_config.turn_id = Some(turn_id.to_string()); + event_handler_config.require_durable_assistant_event = + self.runtime.agent_org_context.is_some(); event_handler_config.agent_org_task_lifecycle = self .runtime .agent_org_context diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs index 1ea018c45a..feb04ce8ff 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs @@ -74,7 +74,7 @@ impl DrainGuard { /// Exact source rows this turn will acknowledge only after successful /// provider execution. Threaded into tool-call context so prospective - /// finality can project this turn's guaranteed commit without treating + /// Quiescence can project this turn's guaranteed commit without treating /// unrelated unread mail as consumed. pub fn pending_ids(&self) -> &[i64] { &self.pending_ids diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs index f1f07f30d3..e051fef593 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs @@ -546,13 +546,39 @@ impl UnifiedMessageProcessor { // ("text content is empty") on the very next request. let should_save_user_msg = !(context.is_resume && content.is_empty()); if should_save_user_msg { - let message_id = tokio::task::block_in_place(|| { - unified_persistence::save_user_msg(session_id, content, context.images.as_deref()) - }) + let initial_input = tokio::task::block_in_place(|| { + crate::coordination::agent_org_runs::AgentOrgRunStore::initial_input_for_turn( + &context.turn_intent_id, + ) + })?; + let message_id = if let Some(input) = initial_input.as_ref() { + if input.content != content { + return Err(format!( + "Starting input content mismatch for turn {}", + context.turn_intent_id + )); + } + tokio::task::block_in_place(|| { + unified_persistence::save_user_msg_with_id( + &input.message_id, + session_id, + content, + ) + }) + .map(|(message_id, _inserted)| message_id) + } else { + tokio::task::block_in_place(|| { + unified_persistence::save_user_msg( + session_id, + content, + context.images.as_deref(), + ) + }) + } .map_err(|err| format!("Failed to save user message: {}", err))?; if let Some(handle) = self.app_handle.as_ref() { - if let Err(err) = tokio::task::block_in_place(|| { + let event_result = tokio::task::block_in_place(|| { crate::bus::event_pipeline_bridge::persist_user_message_event( handle, session_id, @@ -563,7 +589,13 @@ impl UnifiedMessageProcessor { crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, context.turn_intent_id.as_str(), ) - }) { + }); + if let Err(err) = event_result { + if initial_input.is_some() { + return Err(format!( + "Failed to persist Starting user-message event: {err}" + )); + } tracing::warn!( session_id, error = %err, @@ -926,6 +958,12 @@ impl UnifiedMessageProcessor { // Flush any pending streaming content before completing the turn. handler.flush_streaming(session_id); + if let Some(error) = handler.take_assistant_persistence_error() { + return Err(format!( + "{} {error}", + super::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX + )); + } // Update nag-reminder counter based on whether manage_todo was called // during this turn. Reset to 0 on any todo call; increment otherwise. diff --git a/src-tauri/crates/agent-core/src/core/tools/call_context.rs b/src-tauri/crates/agent-core/src/core/tools/call_context.rs index c6ee85adb9..f08cab7938 100644 --- a/src-tauri/crates/agent-core/src/core/tools/call_context.rs +++ b/src-tauri/crates/agent-core/src/core/tools/call_context.rs @@ -29,7 +29,7 @@ //! //! - `turn_intent_id` and `projected_inbox_ids`: identify the exact durable //! turn and Inbox batch whose effects become committed only after this turn -//! succeeds. Agent Org finality uses them to build a prospective +//! succeeds. Agent Org Quiescence uses them to build a prospective //! completion certificate without guessing or subtracting unrelated work. //! //! ## Defaults diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs index c3aada258b..70612e31db 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs @@ -179,7 +179,7 @@ async fn resolve( "outcome": resolution.resolution_kind.as_str(), "org_run_id": run_id, "delivery_resolution": resolution, - "guidance": "The original Inbox row remains durable and unread as audit evidence, but no longer blocks delivery/finality. Re-inspect task_list and the replacement work before requesting completion." + "guidance": "The original Inbox row remains durable and unread as audit evidence, but no longer blocks delivery/Quiescence. Re-inspect task_list and the replacement work before requesting completion." })) .map_err(|err| { ToolError::ExecutionFailed(format!( @@ -377,10 +377,10 @@ mod tests { let fixture = fixture(); let conn = get_connection().expect("test sqlite connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id=?1", + "UPDATE agent_org_runs SET status='archived' WHERE id=?1", params![&fixture.run_id], ) - .expect("complete run"); + .expect("archive run"); let error = OrgInboxRepairTool::new(fixture.coordinator) .execute_text( json!({ diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs index 9015290f8c..3755b396cd 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs @@ -21,7 +21,7 @@ pub struct OrgRunCompleteParams { /// Coordinator-only explicit completion intent. /// /// This does not force a terminal state. It records a durable request at the -/// current work revision; the canonical finality reconciler still waits for a +/// current work revision; the canonical Quiescence reconciler still waits for a /// successful coordinator turn, resolved tasks, drained inbox, settled /// approvals/interventions, and no in-flight turns. pub struct OrgRunCompleteTool { @@ -41,7 +41,7 @@ impl Tool for OrgRunCompleteTool { } fn description(&self) -> &str { - "Request safe completion of the current Agent Org run. Coordinator-only. Records a durable summary at the current work revision; it never bypasses open tasks or finality checks. Use it when task_list says an empty task board requires explicit completion intent." + "Request safe completion of the current Agent Org run. Coordinator-only. Records a durable summary at the current work revision; it never bypasses open tasks or Quiescence checks. Use it when task_list says an empty task board requires explicit completion intent." } fn category(&self) -> &str { @@ -80,7 +80,7 @@ impl Tool for OrgRunCompleteTool { "outcome": "recorded", "org_run_id": run_id, "work_revision": progress.work_revision, - "guidance": "Completion was requested durably. Finish this coordinator turn normally; the canonical finality reconciler will close the run only after every remaining delivery and lifecycle blocker has settled." + "guidance": "Completion was requested durably. Finish this coordinator turn normally; the canonical Quiescence reconciler will move the Team to Idle only after every remaining delivery and lifecycle blocker has settled." }), AgentOrgCompletionRequestOutcome::OpenTasks { unresolved_task_ids, diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs index a2c7a2d8fa..517003592b 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs @@ -130,7 +130,7 @@ pub(super) fn persist_ordinary_message_if_running( "reason": "run_not_running", "org_run_id": run_id, "run_status": run_status, - "guidance": "The Agent Org run is paused or terminal, so this message was not persisted. Resume a paused run before sending new work; terminal runs cannot be reopened.", + "guidance": "The Agent Org Team is not Running, so this formal peer message was not persisted. Starting, Paused, Idle, Failed, and Archived Teams do not accept this mutation in PR1.", })) .map_err(|err| ToolError::ExecutionFailed(err.to_string()))?; tx.commit() diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs index 9f6cb4d14e..2579d7723a 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs @@ -375,14 +375,14 @@ async fn plain_message_to_worker_without_task_returns_guidance_and_does_not_wake } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -async fn ordinary_message_does_not_create_unread_work_after_run_is_terminal() { +async fn ordinary_message_does_not_create_unread_work_after_run_is_archived() { let _sandbox = init_inbox_schema(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id='run-1'", + "UPDATE agent_org_runs SET status='archived' WHERE id='run-1'", [], ) - .expect("complete run"); + .expect("archive run"); let wake = Arc::new(RecordingWakeHook::default()); let tool = OrgSendMessageTool::with_hooks( context(), diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs index 5c3bc6b4d7..9563b33c2c 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs @@ -9,7 +9,7 @@ use serde_json::{json, Value}; use crate::coordination::agent_org_payload_limits::validate_task_identifier; use crate::coordination::agent_org_runs::{ - guaranteed_current_turn_effects_with_connection, AgentOrgFinalityDecision, AgentOrgRunStore, + guaranteed_current_turn_effects_with_connection, AgentOrgQuiescenceDecision, AgentOrgRunStore, }; use crate::coordination::agent_org_tasks::AgentOrgTaskStore; use crate::tools::names as tool_names; @@ -137,7 +137,7 @@ impl Tool for TaskListTool { .map_err(ToolError::InvalidParams)?; } - // Task summaries and run finality facts must describe the same + // Task summaries and Team Quiescence facts must describe the same // database moment. Use one deferred read transaction and project only // bounded columns; routine task_list calls never deserialize full // descriptions, raw metadata, or output content for the entire board. @@ -158,7 +158,7 @@ impl Tool for TaskListTool { let tx = conn .transaction_with_behavior(TransactionBehavior::Deferred) .map_err(|err| err.to_string())?; - let completion = AgentOrgRunStore::finality_assessment_with_connection(&tx, &run_id)?; + let completion = AgentOrgRunStore::quiescence_assessment_with_connection(&tx, &run_id)?; let guaranteed_turn_effects = guaranteed_current_turn_effects_with_connection( &tx, &run_id, @@ -203,7 +203,7 @@ impl Tool for TaskListTool { completion.after_successful_coordinator_turn_with_effects(guaranteed_turn_effects); let completion_ready = matches!( completion_after_turn.decision, - AgentOrgFinalityDecision::Complete + AgentOrgQuiescenceDecision::Quiescent ); let body = json!({ "tasks": page.tasks.iter().map(compact_task_summary_to_json).collect::>(), @@ -236,8 +236,8 @@ impl Tool for TaskListTool { "unread_inbox_count": completion.facts.unread_inbox_count, "pending_plan_approval_count": completion.facts.pending_plan_approval_count, "completion_ready": completion_ready, - "finality_decision": completion.decision, - "current_finality_blockers": &completion.blockers, + "quiescence_decision": completion.decision, + "current_quiescence_blockers": &completion.blockers, "completion_blockers": &completion_after_turn.blockers, }, "org_run_id": self.ctx.org_context.run_id, diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs index 63b1510775..281ab235f2 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs @@ -226,13 +226,7 @@ fn task_tools_sandbox() -> test_env::SandboxGuard { crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) .expect("agent sessions schema"); crate::session::persistence::init(&conn).expect("unified session schema"); - crate::coordination::agent_inbox::init_schema(&conn).expect("agent inbox schema"); - crate::coordination::agent_org_runs::init_schema(&conn).expect("agent org runs schema"); - crate::coordination::agent_member_interventions::init_schema(&conn) - .expect("member intervention schema"); - crate::coordination::agent_org_tasks::init_schema(&conn).expect("agent team tasks schema"); - crate::coordination::agent_org_plan_approvals::init_schema(&conn) - .expect("agent org plan approval schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("canonical Agent Org schemas"); conn.execute_batch( "CREATE TABLE IF NOT EXISTS code_sessions ( session_id TEXT PRIMARY KEY, @@ -2836,7 +2830,7 @@ async fn task_list_defaults_to_fifty_compact_rows() { assert!(value["page"]["next_cursor"].is_string()); } -fn seed_task_list_current_turn_finality_fixture(materialize_inbox: bool) -> i64 { +fn seed_task_list_current_turn_quiescence_fixture(materialize_inbox: bool) -> i64 { let now = chrono::Utc::now().to_rfc3339(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( @@ -2906,10 +2900,10 @@ fn seed_task_list_current_turn_finality_fixture(materialize_inbox: bool) -> i64 inbox.id } -fn has_finality_blocker(value: &Value, field: &str, kind: &str, count: i64) -> bool { +fn has_quiescence_blocker(value: &Value, field: &str, kind: &str, count: i64) -> bool { value["run_summary"][field] .as_array() - .expect("finality blocker array") + .expect("Quiescence blocker array") .iter() .any(|blocker| blocker["kind"] == kind && blocker["count"] == count) } @@ -2917,7 +2911,7 @@ fn has_finality_blocker(value: &Value, field: &str, kind: &str, count: i64) -> b #[tokio::test] async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { let _sandbox = task_tools_sandbox(); - let inbox_id = seed_task_list_current_turn_finality_fixture(true); + let inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let call_ctx = crate::tools::call_context::CallContext::for_turn( "task-list-current-turn", "root-tools-1", @@ -2937,15 +2931,15 @@ async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { "the raw snapshot still includes the currently running coordinator intent" ); assert_eq!(value["run_summary"]["unread_inbox_count"], 1); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, - "current_finality_blockers", + "current_quiescence_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, - "current_finality_blockers", + "current_quiescence_blockers", "unread_inbox", 1, )); @@ -2956,7 +2950,7 @@ async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { #[tokio::test] async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { let _sandbox = task_tools_sandbox(); - let projected_inbox_id = seed_task_list_current_turn_finality_fixture(true); + let projected_inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let now = chrono::Utc::now().to_rfc3339(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( @@ -2996,13 +2990,13 @@ async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { assert_eq!(value["run_summary"]["pending_worker_turn_intent_count"], 2); assert_eq!(value["run_summary"]["unread_inbox_count"], 2); assert_eq!(value["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, "completion_blockers", "unread_inbox", @@ -3013,7 +3007,7 @@ async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { #[tokio::test] async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_receipt() { let _sandbox = task_tools_sandbox(); - let inbox_id = seed_task_list_current_turn_finality_fixture(true); + let inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let list = TaskListTool::new(ctx(COORDINATOR_MEMBER_ID)); let wrong_session_ctx = crate::tools::call_context::CallContext::for_turn( @@ -3030,13 +3024,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode wrong-session result"); assert_eq!(wrong_session["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_session, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_session, "completion_blockers", "unread_inbox", @@ -3057,13 +3051,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode wrong-intent result"); assert_eq!(wrong_intent["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_intent, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_intent, "completion_blockers", "unread_inbox", @@ -3091,13 +3085,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode missing-receipt result"); assert_eq!(missing_receipt["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &missing_receipt, "completion_blockers", "unread_inbox", 1, )); - assert!(!has_finality_blocker( + assert!(!has_quiescence_blocker( &missing_receipt, "completion_blockers", "in_flight_turn_intents", diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs index d68fb54676..5393dfd7c6 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs @@ -28,8 +28,8 @@ use crate::tools::impls::orchestration::org_send_message::{InboxWakeHook, NoopIn /// Production hook: persist a `MemberIdle` envelope into the inbox, then wake the coordinator. /// -/// The hook contract is synchronous because finality must observe this durable -/// notification before it can complete the Run. When called from Tokio's +/// The hook contract is synchronous because Quiescence must observe this durable +/// notification before it can move the Team to Idle. When called from Tokio's /// multi-thread runtime we therefore use an explicit `block_in_place` section: /// executor capacity is handed to another worker while ordering is preserved. pub struct InboxStoreMemberIdleHook { @@ -310,7 +310,7 @@ mod tests { let _sandbox = test_env::sandbox(); let conn = database::db::get_connection().expect("test connection"); agent_inbox::init_schema(&conn).expect("agent inbox schema"); - seed_run(&conn, "run-terminal", "completed"); + seed_run(&conn, "run-terminal", "archived"); let wake_hook = Arc::new(RecordingWakeHook::default()); let hook = InboxStoreMemberIdleHook::new(wake_hook.clone()); diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs index 3eb7fad7e0..9c0b15f89f 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs @@ -72,7 +72,15 @@ impl UnifiedSubagentHandler { .collect(); let count = persistable.len(); - event_pipeline_bridge::persist_events("subagent-child-persist", &sid, &persistable, 5); + if let Err(error) = + event_pipeline_bridge::persist_events("subagent-child-persist", &sid, &persistable, 5) + { + tracing::warn!( + session_id = %sid, + error = %error, + "failed to persist subagent child events" + ); + } tracing::info!( "[subagent:{}] Persisted {} child events for session {}", self.config.subagent_type, diff --git a/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs b/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs index 18e3b202ef..fe81f34134 100644 --- a/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs +++ b/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs @@ -120,8 +120,12 @@ pub type FinalizePlanRevisionEventsFn = /// Persist a batch of `SessionEvent`s synchronously with retry. The wire /// side converts each to its on-disk `CachedEvent` representation. `label` /// is used in retry log lines. -pub type PersistEventsFn = - fn(label: &'static str, session_id: &str, events: &[SessionEvent], max_retries: u32); +pub type PersistEventsFn = fn( + label: &'static str, + session_id: &str, + events: &[SessionEvent], + max_retries: u32, +) -> Result<(), String>; /// Fire-and-forget variant: spawns `persist_events` onto a blocking thread. pub type PersistEventsAsyncFn = @@ -444,16 +448,13 @@ pub fn persist_events( session_id: &str, events: &[SessionEvent], max_retries: u32, -) { +) -> Result<(), String> { if let Some(f) = PERSIST_EVENTS.get() { - f(label, session_id, events, max_retries); - } else { - tracing::warn!( - "[event-pipeline-bridge] persist_events ({}) called before register for {}", - label, - session_id - ); + return f(label, session_id, events, max_retries); } + Err(format!( + "event pipeline persistence is not registered ({label}, session {session_id})" + )) } pub fn persist_events_async( diff --git a/src-tauri/crates/agent-core/src/foundation/session_bridge.rs b/src-tauri/crates/agent-core/src/foundation/session_bridge.rs index 116293e068..b4f49ff71b 100644 --- a/src-tauri/crates/agent-core/src/foundation/session_bridge.rs +++ b/src-tauri/crates/agent-core/src/foundation/session_bridge.rs @@ -444,7 +444,7 @@ impl TurnIntentBridgeStatus { } /// Canonical persisted wire values for turn intents that may still execute. -/// Agent Org finality queries bind these values instead of independently +/// Agent Org Quiescence queries bind these values instead of independently /// hard-coding a second lifecycle definition. pub const IN_FLIGHT_TURN_INTENT_STATUSES: [&str; 3] = [ TurnIntentBridgeStatus::Optimistic.as_str(), @@ -499,6 +499,16 @@ pub type UpsertTurnIntentFn = fn( status: TurnIntentBridgeStatus, ); +pub type UpsertTurnIntentWithConnectionFn = fn( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: TurnIntentBridgeSource, + status: TurnIntentBridgeStatus, +) -> Result<(), String>; + pub type UpdateTurnIntentStatusFn = fn(session_id: &str, turn_intent_id: &str, new_status: TurnIntentBridgeStatus); @@ -508,6 +518,8 @@ pub type GetTurnIntentStatusFn = pub type MarkPendingTurnIntentsStaleFn = fn(session_id: &str); static UPSERT_TURN_INTENT: OnceLock = OnceLock::new(); +static UPSERT_TURN_INTENT_WITH_CONNECTION: OnceLock = + OnceLock::new(); static UPDATE_TURN_INTENT_STATUS: OnceLock = OnceLock::new(); static GET_TURN_INTENT_STATUS: OnceLock = OnceLock::new(); static MARK_PENDING_TURN_INTENTS_STALE: OnceLock = OnceLock::new(); @@ -516,6 +528,12 @@ pub fn register_upsert_turn_intent(implementation: UpsertTurnIntentFn) { let _ = UPSERT_TURN_INTENT.set(implementation); } +pub fn register_upsert_turn_intent_with_connection( + implementation: UpsertTurnIntentWithConnectionFn, +) { + let _ = UPSERT_TURN_INTENT_WITH_CONNECTION.set(implementation); +} + pub fn register_update_turn_intent_status(implementation: UpdateTurnIntentStatusFn) { let _ = UPDATE_TURN_INTENT_STATUS.set(implementation); } @@ -553,6 +571,34 @@ pub fn upsert_turn_intent( } } +/// Connection-scoped form for lifecycle owners that must accept an intent in +/// the same SQLite transaction as an adjacent Agent Org state transition. +pub fn upsert_turn_intent_with_connection( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: TurnIntentBridgeSource, + status: TurnIntentBridgeStatus, +) -> Result<(), String> { + if turn_intent_id.is_empty() { + return Err("turn_intent_id must not be empty".to_string()); + } + let implementation = UPSERT_TURN_INTENT_WITH_CONNECTION + .get() + .ok_or_else(|| "turn-intent persistence bridge is not registered".to_string())?; + implementation( + connection, + session_id, + turn_intent_id, + client_message_id, + org_run_id, + source, + status, + ) +} + /// Patch the status of an existing lifecycle row. Illegal transitions are /// silently rejected by the implementation — callers do not need to handle /// the error case. diff --git a/src-tauri/crates/agent-core/src/lifecycle.rs b/src-tauri/crates/agent-core/src/lifecycle.rs index 1893c7da64..a9430a96c4 100644 --- a/src-tauri/crates/agent-core/src/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/lifecycle.rs @@ -11,9 +11,7 @@ use tauri::Emitter; use crate::bus::{broadcast_event, event_pipeline_bridge}; use crate::coordination::agent_inbox::MemberIdleReason; -use crate::coordination::agent_org_runs::{ - AgentOrgRunContext, AgentOrgRunStatus, AgentOrgRunStore, -}; +use crate::coordination::agent_org_runs::{AgentOrgRunContext, AgentOrgRunStore}; use crate::coordination::agent_org_tasks::{ self, AgentOrgTaskStore, Task, TASK_METADATA_REQUIRED_ROLE, }; @@ -476,13 +474,27 @@ pub fn finalize_agent_org_member_turn( // after every member/coordinator boundary so an all-completed, fully // quiescent run closes without requiring the user to pause/resume it. // The store re-checks tasks, inbox, interventions and queued turn - // intents in one IMMEDIATE transaction before committing finality. + // intents in one IMMEDIATE transaction before committing quiescence. if let Some(run_id) = reconcile_run_id { - match AgentOrgRunStore::reconcile_run_finality(&run_id) { - Ok(Some(AgentOrgRunStatus::Completed)) => { - tracing::info!(run_id = %run_id, "[lifecycle] completed quiescent Agent Org run"); + let transition = AgentOrgRunStore::assess_run_quiescence(&run_id).and_then(|assessment| { + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + AgentOrgRunStore::try_transition_working_to_idle(&run_id, generation, work_revision) + }); + match transition { + Ok(true) => { + tracing::info!(run_id = %run_id, "[lifecycle] idled quiescent Agent Org run"); } - Ok(_) => {} + Ok(false) => {} Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "[lifecycle] failed to reconcile Agent Org run after turn finalization"); } @@ -576,7 +588,7 @@ pub async fn finalize_session( if is_agent_org_member_session { // Member finalization performs several synchronous SQLite operations // under the shared writer lock (task requeue, recovery-budget cleanup, - // MemberIdle persistence, and run finality reconciliation). Keep the + // MemberIdle persistence, and Team quiescence reconciliation). Keep the // complete blocking phase off the Tokio worker that is finalizing the // provider turn; moving only the first query still leaves the later // writes able to stall unrelated async sessions. diff --git a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs index 0db3a75639..844b5db7d0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs @@ -32,6 +32,14 @@ pub(super) struct SessionIdentity { pub(super) account_id: Option, pub(super) native_harness_type: Option, pub(super) workspace_root: PathBuf, + /// A loaded Agent Org runtime already knows its authoritative Run id. + /// Keeping that hint here lets the send boundary enforce Team lifecycle + /// before it creates a Provider without querying ordinary SDE sessions. + pub(super) agent_org_run_id_hint: Option, + /// Unloaded Agent Org root/member sessions carry a canonical member id in + /// persistence. Only those sessions need the bounded parent-walk lookup; + /// ordinary SDE sessions stay on the existing zero-Agent-Org-query path. + pub(super) has_persisted_agent_org_identity: bool, } /// Caller-supplied overrides. Fields that are `None` are resolved from @@ -104,6 +112,10 @@ pub(super) async fn resolve_session_identity( let native_harness_after_l2 = overrides .native_harness_type .or_else(|| cached_runtime.as_ref().and_then(|r| r.native_harness_type)); + let agent_org_run_id_hint = cached_runtime + .as_ref() + .and_then(|runtime| runtime.agent_org_context.as_ref()) + .map(|context| context.run_id.clone()); // ── Layer 3: DB (lazy — only when at least one field still needs it) ─ let needs_db = model_after_l2.is_none() @@ -163,6 +175,10 @@ pub(super) async fn resolve_session_identity( }) .transpose()?; let native_harness_type = native_harness_after_l2.or(native_harness_from_db); + let has_persisted_agent_org_identity = db_record + .as_ref() + .and_then(|record| record.org_member_id.as_deref()) + .is_some(); // ── Workspace Root ─────────────────────────────────────────────────── // @@ -211,6 +227,8 @@ pub(super) async fn resolve_session_identity( account_id, workspace_root, native_harness_type, + agent_org_run_id_hint, + has_persisted_agent_org_identity, }) } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs index 093539ef57..fbeeff52e1 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs @@ -93,11 +93,10 @@ pub(super) fn promote_agent_org_wake_session_to_running( .map_err(|error| error.to_string()) } -/// Promote a direct Rust Agent Org turn unless deletion has established the -/// run's terminal `cancelled` fence. Direct user turns intentionally retain -/// their existing behavior for completed/failed historical runs; this guard -/// only closes the race where a message was queued while hierarchy deletion -/// was stopping the run. +/// Promote a direct Rust Agent Org turn only while its Team is still Running. +/// Submit preflight is only a snapshot: a queued turn must re-check the +/// durable lifecycle fence immediately before execution so Starting, Idle, +/// Paused, Failed, or Archived can never start a Provider turn. pub(super) fn promote_agent_org_direct_session_to_running( conn: &rusqlite::Connection, run_id: &str, @@ -113,7 +112,9 @@ pub(super) fn promote_agent_org_direct_session_to_running( ) .optional() .map_err(|error| error.to_string())?; - if run_status.as_deref() == Some("cancelled") || run_status.is_none() { + if run_status.as_deref() + != Some(crate::coordination::agent_org_runs::AgentOrgRunStatus::Running.as_str()) + { return Ok(0); } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs index 183b8153fc..949fa37e97 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs @@ -27,6 +27,77 @@ use super::org_wake::{ resolve_agent_org_wake_mode, }; +pub(super) fn ensure_agent_org_turn_is_runnable( + run_id: &str, + status: crate::coordination::agent_org_runs::AgentOrgRunStatus, +) -> Result<(), String> { + use crate::coordination::agent_org_runs::AgentOrgRunStatus; + + match status { + AgentOrgRunStatus::Running => Ok(()), + AgentOrgRunStatus::Starting => Err(format!( + "team_not_ready: Agent Org run {run_id} is still materializing" + )), + AgentOrgRunStatus::Paused => Err(format!( + "team_paused: Agent Org run {run_id} cannot start a turn in this lifecycle slice" + )), + AgentOrgRunStatus::Idle => Err(format!( + "team_idle: Agent Org run {run_id} has no formal activation for a new turn" + )), + AgentOrgRunStatus::Failed => Err(format!( + "team_unavailable: Agent Org run {run_id} failed during materialization" + )), + AgentOrgRunStatus::Archived => Err(format!( + "team_archived: Agent Org run {run_id} is read-only" + )), + } +} + +async fn preflight_agent_org_turn_before_runtime( + session_id: &str, + explicit_run_id: Option<&str>, + run_id_hint: Option<&str>, + has_persisted_agent_org_identity: bool, +) -> Result, String> { + if let (Some(explicit), Some(hint)) = (explicit_run_id, run_id_hint) { + if explicit != hint { + return Err(format!( + "Agent Org turn intent run mismatch for session {session_id}: explicit run {explicit}, runtime run {hint}" + )); + } + } + + let run_id = match explicit_run_id.or(run_id_hint) { + Some(run_id) => Some(run_id.to_string()), + None if has_persisted_agent_org_identity => { + let session_id = session_id.to_string(); + tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_runs::AgentOrgRunStore::run_id_for_session_with_parent_walk( + &session_id, + ) + }) + .await + .map_err(|error| format!("Agent Org run lookup worker failed: {error}"))?? + } + None => None, + }; + + let Some(run_id) = run_id else { + return Ok(None); + }; + + crate::coordination::agent_org_runs::require_agent_org_redesign()?; + let status_run_id = run_id.clone(); + let status = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_runs::AgentOrgRunStore::get_run_status(&status_run_id) + }) + .await + .map_err(|error| format!("Agent Org status worker failed: {error}"))?? + .ok_or_else(|| format!("team_unavailable: Agent Org run {run_id} does not exist"))?; + ensure_agent_org_turn_is_runnable(&run_id, status)?; + Ok(Some(run_id)) +} + pub(super) fn should_divert_to_mid_turn_steering( source: TurnIntentBridgeSource, is_resume: bool, @@ -108,6 +179,23 @@ pub(crate) async fn send_message_impl( // ── 1. Resolve session identity (unified — single code path) ───────── let identity = resolve_session_identity(state, &session_id, overrides).await?; + let explicit_org_run_id = match (org_wake_run_id.as_deref(), intent_org_run_id.as_deref()) { + (Some(wake_run_id), Some(intent_run_id)) if wake_run_id != intent_run_id => { + return Err(format!( + "Agent Org wake/intent run mismatch for session {session_id}: wake run {wake_run_id}, intent run {intent_run_id}" + )); + } + (Some(run_id), _) | (None, Some(run_id)) => Some(run_id), + (None, None) => None, + }; + let preflight_org_run_id = preflight_agent_org_turn_before_runtime( + &session_id, + explicit_org_run_id, + identity.agent_org_run_id_hint.as_deref(), + identity.has_persisted_agent_org_identity, + ) + .await?; + // Goal loop: a real user submission becomes (or replaces) the // session's standing goal and resets the continuation counter. // `Queue`-sourced messages (goal continuations, queued flushes) and @@ -157,7 +245,7 @@ pub(crate) async fn send_message_impl( } (Some(_), _) => intent_org_run_id, (None, Some(_)) => runtime_org_run_id, - (None, None) => None, + (None, None) => preflight_org_run_id, }; // Wingman resume: reopen the bottom bar. On fresh start the frontend @@ -477,9 +565,9 @@ pub(crate) async fn send_message_impl( // Queued and coalesced messages are not running sessions. Promote // the DB state only when the scheduler actually begins execution. - // Agent Org wakes require a running run. Direct Agent Org turns - // retain their historical-run behavior but refuse the terminal - // `cancelled` fence established by hierarchy deletion. + // Both Agent Org wakes and direct turns require a Running Team. + // This execute-time check closes the race after submit preflight: + // a queued turn becomes a no-op if lifecycle advances first. let status_sid = sid.clone(); let status_wake_run_id = org_wake_run_id.clone(); let status_intent_run_id = intent_org_run_id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs index b5a4d0c019..7e90f77657 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs @@ -10,7 +10,10 @@ use super::org_wake::{ promote_agent_org_direct_session_to_running, promote_agent_org_wake_session_to_running, resolve_agent_org_wake_mode, }; -use super::send::{should_divert_to_mid_turn_steering, terminal_intent_status_override}; +use super::send::{ + ensure_agent_org_turn_is_runnable, should_divert_to_mid_turn_steering, + terminal_intent_status_override, +}; use crate::coordination::agent_inbox::{ AgentInboxStore, AgentMessage, InsertInboxParams, RequestId, }; @@ -287,28 +290,71 @@ fn queued_agent_org_wake_rechecks_run_member_and_intervention_at_turn_start() { } #[test] -fn direct_agent_org_turn_refuses_cancelled_delete_fence() { +fn direct_agent_org_turn_only_promotes_while_run_is_running() { let fixture = setup_wake_mode_fixture("build", TaskStatus::Pending); let conn = database::db::get_connection().expect("test db"); + for status in [ + AgentOrgRunStatus::Starting, + AgentOrgRunStatus::Paused, + AgentOrgRunStatus::Idle, + AgentOrgRunStatus::Failed, + AgentOrgRunStatus::Archived, + ] { + conn.execute( + "UPDATE agent_org_runs SET status=?1 WHERE id=?2", + rusqlite::params![status.as_str(), &fixture.run_id], + ) + .expect("set non-runnable run status"); + assert_eq!( + promote_agent_org_direct_session_to_running( + &conn, + &fixture.run_id, + &fixture.session_id, + ) + .expect("non-running run claim is a no-op"), + 0, + "{status:?} must not promote the member Session" + ); + let session_status = conn + .query_row( + "SELECT status FROM agent_sessions WHERE session_id=?1", + [&fixture.session_id], + |row| row.get::<_, String>(0), + ) + .expect("load member status"); + assert_eq!(session_status, "idle"); + } + conn.execute( - "UPDATE agent_org_runs SET status='cancelled' WHERE id=?1", - [&fixture.run_id], + "UPDATE agent_org_runs SET status=?1 WHERE id=?2", + rusqlite::params![AgentOrgRunStatus::Running.as_str(), &fixture.run_id], ) - .expect("establish delete fence"); - + .expect("restore running run"); assert_eq!( promote_agent_org_direct_session_to_running(&conn, &fixture.run_id, &fixture.session_id) - .expect("cancelled run claim is a no-op"), - 0 + .expect("running run promotes the member Session"), + 1 ); - let status = conn - .query_row( - "SELECT status FROM agent_sessions WHERE session_id=?1", - [&fixture.session_id], - |row| row.get::<_, String>(0), - ) - .expect("load member status"); - assert_eq!(status, "idle"); +} + +#[test] +fn provider_preflight_exhaustively_rejects_every_non_running_team_status() { + assert!(ensure_agent_org_turn_is_runnable("run", AgentOrgRunStatus::Running).is_ok()); + + for (status, code) in [ + (AgentOrgRunStatus::Starting, "team_not_ready"), + (AgentOrgRunStatus::Paused, "team_paused"), + (AgentOrgRunStatus::Idle, "team_idle"), + (AgentOrgRunStatus::Failed, "team_unavailable"), + (AgentOrgRunStatus::Archived, "team_archived"), + ] { + let error = ensure_agent_org_turn_is_runnable("run", status) + .expect_err("non-running Team cannot initialize a turn"); + assert!( + error.starts_with(code), + "{status:?} should return {code}, got {error}" + ); + } } #[test] diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs index 7fb67a4ce7..9c03bb1d2d 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs @@ -12,7 +12,9 @@ use serde::Serialize; use crate::coordination::agent_inbox::{ AgentInboxRecord, AgentInboxStore, AgentMessage, InsertInboxParams, USER_SENDER_ID, }; -use crate::coordination::agent_org_runs::{AgentOrgRunContext, COORDINATOR_MEMBER_ID}; +use crate::coordination::agent_org_runs::{ + AgentOrgRunContext, AgentOrgRunStatus, COORDINATOR_MEMBER_ID, +}; use crate::state::AgentAppState; use super::context::session_org_read_context; @@ -75,6 +77,7 @@ pub async fn agent_org_group_chat_history_page_impl( before_id: Option, limit: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; if before_id.is_some_and(|id| id <= 0) { return Err("before_id must be a positive Inbox row id".to_string()); } @@ -307,6 +310,7 @@ async fn agent_org_send_group_chat_message_impl_with_display( content: String, display_text: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let content = content.trim(); if content.is_empty() { return Err("Agent Org group chat message content is required".to_string()); @@ -418,17 +422,26 @@ pub(super) fn persist_group_chat_message( ) .optional() .map_err(|err| err.to_string())?; - match run_status.as_deref() { - Some("running" | "paused") => {} - Some(status) => { + let Some(run_status) = run_status else { + return Err(format!("Agent Org run {} no longer exists", context.run_id)); + }; + let run_status = AgentOrgRunStatus::parse(&run_status).ok_or_else(|| { + format!( + "Agent Org run {} has an unrecognized status", + context.run_id + ) + })?; + match run_status { + AgentOrgRunStatus::Running | AgentOrgRunStatus::Paused => {} + AgentOrgRunStatus::Starting + | AgentOrgRunStatus::Idle + | AgentOrgRunStatus::Failed + | AgentOrgRunStatus::Archived => { return Err(format!( - "Agent Org run {} is {status}; terminal runs do not accept new group messages", - context.run_id + "Agent Org run {} is {}; this status does not accept new group messages", + context.run_id, run_status )); } - None => { - return Err(format!("Agent Org run {} no longer exists", context.run_id)); - } } let row = AgentInboxStore::insert_in_tx( diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs index 4335760be9..1f5576ec41 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs @@ -43,6 +43,7 @@ pub async fn agent_org_session_enter_intervention( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; @@ -76,6 +77,7 @@ pub async fn agent_org_session_intervention_state( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(AgentOrgSessionInterventionState { intervention: None }); }; @@ -146,6 +148,7 @@ pub async fn agent_org_session_return_to_work_impl( state: &AgentAppState, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(state, &session_id).await? else { return Ok(false); }; @@ -189,6 +192,7 @@ pub async fn agent_org_send_user_message_to_member_impl( member_id: String, content: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let member_id = member_id.trim(); if member_id.is_empty() { return Err("Agent Org member id is required".to_string()); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs index 8c567f979c..711106f764 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs @@ -24,13 +24,14 @@ use super::context::session_org_read_context; /// Pause the Agent Org run that the given session belongs to. Transitions /// `running → paused`; already non-running runs return `Ok(false)` (idempotent). -/// The run remains queryable while paused — polling and member switching are -/// unaffected. The coordinator and members stop receiving dispatch until resumed. +/// The run remains available to explicit reads while paused, but PR1's +/// fallback poller deliberately observes only Starting and Running Teams. #[tauri::command] pub async fn agent_org_pause_run( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; @@ -61,6 +62,7 @@ pub async fn agent_org_resume_run( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs index 195c0e4bdb..adf4e2cee0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs @@ -3,7 +3,7 @@ //! When a run's plan-approval policy routes a plan revision to the user, these //! commands fetch the revision detail and record the user's decision (approve, //! approve-with-edits, or request-changes), then wake the affected members and -//! reconcile run finality off the durable transaction. +//! reconcile Team quiescence off the durable transaction. use crate::coordination::agent_inbox::USER_SENDER_ID; use crate::coordination::agent_org_plan_approvals::{ @@ -31,6 +31,7 @@ pub async fn agent_org_plan_approval_detail( approval_id: String, plan_revision_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Err(format!( "Session {session_id} is not part of an Agent Org run" @@ -71,6 +72,7 @@ pub async fn agent_org_plan_approval_respond( edited_content: Option, feedback: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Err(format!( "Session {session_id} is not part of an Agent Org run" @@ -169,7 +171,23 @@ pub async fn agent_org_plan_approval_respond( let reconcile_run_id = run_id.clone(); tokio::spawn(async move { match tokio::task::spawn_blocking(move || { - AgentOrgRunStore::reconcile_run_finality(&reconcile_run_id) + let assessment = AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + AgentOrgRunStore::try_transition_working_to_idle( + &reconcile_run_id, + generation, + work_revision, + ) }) .await { diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs index 01d743df6a..a409fcd3ad 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs @@ -141,6 +141,7 @@ pub struct AgentOrgRunTaskOverview { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AgentOrgRunPhase { + Starting, Coordinating, Dispatching, MembersWorking, @@ -148,10 +149,9 @@ pub enum AgentOrgRunPhase { AwaitingPlanApproval, Finalizing, Paused, - Completed, + Idle, Failed, - Cancelled, - Abandoned, + Archived, } /// The Run View is a live operational snapshot, not an inbox-history API. @@ -172,6 +172,7 @@ pub async fn agent_org_session_run_view_impl( state: &AgentAppState, session_id: &str, ) -> Result, String> { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(state, session_id).await? else { return Ok(None); }; @@ -193,7 +194,7 @@ pub async fn agent_org_session_run_view_impl( Ok(Some(view)) } -fn build_agent_org_run_view( +pub(super) fn build_agent_org_run_view( context: &AgentOrgRunContext, current_member_id: String, ) -> Result { @@ -201,8 +202,8 @@ fn build_agent_org_run_view( let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Deferred) .map_err(|err| err.to_string())?; - let finality = AgentOrgRunStore::finality_assessment_with_connection(&tx, &context.run_id)?; - let run_status_value = finality + let quiescence = AgentOrgRunStore::quiescence_assessment_with_connection(&tx, &context.run_id)?; + let run_status_value = quiescence .facts .run_status .ok_or_else(|| format!("Agent Org run {} no longer exists", context.run_id))?; @@ -217,11 +218,11 @@ fn build_agent_org_run_view( RUN_VIEW_TASK_LIMIT, )?; let task_overview = AgentOrgRunTaskOverview { - total: finality.facts.task_count, - pending: finality.facts.pending_task_count, - in_progress: finality.facts.in_progress_task_count, - completed: finality.facts.completed_task_count, - corrupt: finality.facts.corrupt_task_count, + total: quiescence.facts.task_count, + pending: quiescence.facts.pending_task_count, + in_progress: quiescence.facts.in_progress_task_count, + completed: quiescence.facts.completed_task_count, + corrupt: quiescence.facts.corrupt_task_count, visible: task_page.tasks.len(), truncated: task_page.has_more, }; @@ -307,7 +308,7 @@ fn build_agent_org_run_view( run_status_value, &members, &task_overview, - finality.facts.unread_inbox_count, + quiescence.facts.unread_inbox_count, &pending_plan_approvals, ); @@ -322,7 +323,7 @@ fn build_agent_org_run_view( tasks, task_overview, inbox, - unread_inbox_count: finality.facts.unread_inbox_count, + unread_inbox_count: quiescence.facts.unread_inbox_count, pending_plan_approvals, }) } @@ -335,11 +336,11 @@ pub(super) fn project_run_phase( pending_plan_approvals: &[AgentOrgPlanApprovalSummary], ) -> AgentOrgRunPhase { match run_status { + AgentOrgRunStatus::Starting => AgentOrgRunPhase::Starting, AgentOrgRunStatus::Paused => AgentOrgRunPhase::Paused, - AgentOrgRunStatus::Completed => AgentOrgRunPhase::Completed, + AgentOrgRunStatus::Idle => AgentOrgRunPhase::Idle, AgentOrgRunStatus::Failed => AgentOrgRunPhase::Failed, - AgentOrgRunStatus::Cancelled => AgentOrgRunPhase::Cancelled, - AgentOrgRunStatus::Abandoned => AgentOrgRunPhase::Abandoned, + AgentOrgRunStatus::Archived => AgentOrgRunPhase::Archived, AgentOrgRunStatus::Running => { let all_tasks_completed = task_overview.total > 0 && task_overview.pending == 0 diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs index b5bf8d3a47..8e160cd8f0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs @@ -68,7 +68,7 @@ fn prepare_command_run(status: &str) -> AgentOrgRunContext { id, org_id, coordinator_agent_id, root_session_id, org_snapshot_json, entry_mode, status, work_item_id, project_slug, routine_fire_id, summary, last_error, - created_at, updated_at, completed_at + created_at, updated_at, idled_at ) VALUES (?1, ?2, ?3, ?4, NULL, 'standalone_session', ?5, NULL, NULL, NULL, NULL, NULL, ?6, ?6, NULL)", params![ @@ -167,7 +167,7 @@ fn task_for_resume(owner: Option<&str>, status: TaskStatus) -> Task { } #[test] -fn run_phase_projects_all_completed_running_board_as_finalizing() { +fn run_phase_projects_completed_work_as_finalizing_then_idle() { let overview = AgentOrgRunTaskOverview { total: 1, pending: 0, @@ -183,7 +183,7 @@ fn run_phase_projects_all_completed_running_board_as_finalizing() { ); assert_eq!( project_run_phase( - AgentOrgRunStatus::Completed, + AgentOrgRunStatus::Idle, &[], &AgentOrgRunTaskOverview { total: 0, @@ -197,10 +197,88 @@ fn run_phase_projects_all_completed_running_board_as_finalizing() { 0, &[], ), - AgentOrgRunPhase::Completed + AgentOrgRunPhase::Idle ); } +#[test] +fn run_view_is_a_pure_read_and_does_not_advance_updated_at() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db connection"); + crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); + crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) + .expect("session snapshot schema"); + crate::session::persistence::init(&conn).expect("session schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS code_sessions ( + session_id TEXT PRIMARY KEY, + cli_agent_type TEXT NOT NULL, + status TEXT NOT NULL, + parent_session_id TEXT, + org_member_id TEXT, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS session_turn_intents ( + session_id TEXT NOT NULL, + turn_intent_id TEXT NOT NULL, + client_message_id TEXT, + org_run_id TEXT, + source TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (session_id, turn_intent_id) + );", + ) + .expect("runtime support schemas"); + drop(conn); + + let context = prepare_command_run("running"); + crate::session::persistence::upsert_session( + &crate::session::persistence::UnifiedSessionRecord { + session_id: "root-shared-agent".to_string(), + name: "Coordinator".to_string(), + status: crate::session::SessionStatus::Idle.as_str().to_string(), + session_type: "agent".to_string(), + agent_definition_id: Some("builtin:sde".to_string()), + org_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + created_at: "2026-05-28T00:00:00Z".to_string(), + updated_at: "2026-05-28T00:00:00Z".to_string(), + ..Default::default() + }, + ) + .expect("persist coordinator Session"); + let observer = get_connection().expect("observer connection"); + let before_data_version: i64 = observer + .query_row("PRAGMA data_version", [], |row| row.get(0)) + .expect("read data version"); + let before_updated_at: String = observer + .query_row( + "SELECT updated_at FROM agent_org_runs WHERE id=?1", + [&context.run_id], + |row| row.get(0), + ) + .expect("read run timestamp"); + + let view = build_agent_org_run_view(&context, COORDINATOR_MEMBER_ID.to_string()) + .expect("build pure Run View"); + + let after_data_version: i64 = observer + .query_row("PRAGMA data_version", [], |row| row.get(0)) + .expect("read data version after Run View"); + let after_updated_at: String = observer + .query_row( + "SELECT updated_at FROM agent_org_runs WHERE id=?1", + [&context.run_id], + |row| row.get(0), + ) + .expect("read run timestamp after Run View"); + assert_eq!(view.run_status, "running"); + assert_eq!(after_data_version, before_data_version); + assert_eq!(after_updated_at, before_updated_at); +} + #[test] fn task_runtime_projects_execution_mode_on_the_wire() { let task = AgentOrgTaskRuntime { @@ -325,9 +403,9 @@ fn resume_wake_requires_unread_inbox() { } #[test] -fn terminal_group_message_writes_neither_inbox_nor_intervention_clear() { +fn archived_group_message_writes_neither_inbox_nor_intervention_clear() { let _sandbox = test_helpers::test_env::sandbox(); - let context = prepare_command_run("completed"); + let context = prepare_command_run("archived"); AgentMemberInterventionStore::enter(EnterMemberInterventionParams { org_run_id: context.run_id.clone(), member_id: "member-planner".to_string(), @@ -342,12 +420,12 @@ fn terminal_group_message_writes_neither_inbox_nor_intervention_clear() { &context, "builtin:sde", "member-planner", - "This must not enter a terminal run", + "This must not enter an Archived run", None, ) - .expect_err("terminal run rejects group message"); + .expect_err("Archived run rejects group message"); - assert!(error.contains("terminal runs do not accept")); + assert!(error.contains("this status does not accept")); assert_eq!(inbox_count_for_member(&context, "member-planner"), 0); assert!( AgentMemberInterventionStore::active_for_member(&context.run_id, "member-planner") @@ -460,10 +538,10 @@ fn group_chat_history_pages_all_rows_and_preserves_long_display_text_after_reloa let conn = get_connection().expect("db connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id=?1", + "UPDATE agent_org_runs SET status='archived' WHERE id=?1", params![&context.run_id], ) - .expect("terminalize run"); + .expect("archive run"); assert_eq!( load_group_chat_history_page(&context, None, 100) .expect("terminal history stays readable") diff --git a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs index 89964e14bb..dcd9fac74c 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; use crate::coordination::agent_org_runs::AgentOrgRunStore; use crate::interaction::plan_approval::persistence::PlanApprovalStore; @@ -52,8 +51,6 @@ pub async fn agent_list_all_sessions() -> Result, String> } const MAX_AGENT_ORG_DELETE_SESSIONS: usize = 1_024; -const AGENT_ORG_DELETE_STOP_TIMEOUT: Duration = Duration::from_secs(10); -const AGENT_ORG_DELETE_STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -103,46 +100,14 @@ pub async fn agent_delete_session( }); }; - let (plan, quiesced_runtime_session_ids) = if matches!( - plan.run_status, - crate::coordination::agent_org_runs::AgentOrgRunStatus::Running - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Paused - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - ) { - let fenced_plan = - tokio::task::spawn_blocking(move || establish_agent_org_delete_fence(&plan)) - .await - .map_err(|err| format!("Agent Org deletion fence worker failed: {err}"))??; - let quiesced_runtime_session_ids = if fenced_plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - { - stop_agent_org_runtime_sessions(&state, &fenced_plan).await? - } else { - ensure_agent_org_runtime_sessions_idle(&state, &fenced_plan).await?; - HashSet::new() - }; - let root_session_id = fenced_plan.root_session_id.clone(); - let current_plan = tokio::task::spawn_blocking(move || { - let conn = get_connection().map_err(|err| err.to_string())?; - load_agent_org_session_delete_plan(&conn, &root_session_id)?.ok_or_else(|| { - format!( - "Refusing to delete Agent Org root {root_session_id}: ownership disappeared while stopping" - ) - }) - }) - .await - .map_err(|err| format!("Agent Org post-stop planning worker failed: {err}"))??; - if !agent_org_delete_topology_matches(&fenced_plan, ¤t_plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed while stopping", - fenced_plan.run_id - )); - } - (current_plan, quiesced_runtime_session_ids) - } else { - ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; - (plan, HashSet::new()) - }; + if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { + return Err(format!( + "Refusing to delete Agent Org run {}: Archive is required before Delete", + plan.run_id + )); + } + ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; + let quiesced_runtime_session_ids = HashSet::new(); validate_agent_org_delete_ready(&plan, &quiesced_runtime_session_ids)?; ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; @@ -375,89 +340,11 @@ fn load_agent_org_session_delete_plan( })) } -fn agent_org_delete_topology_matches( - expected: &AgentOrgSessionDeletePlan, - current: &AgentOrgSessionDeletePlan, -) -> bool { - expected.run_id == current.run_id - && expected.root_session_id == current.root_session_id - && expected.sessions.len() == current.sessions.len() - && expected - .sessions - .iter() - .zip(¤t.sessions) - .all(|(left, right)| { - left.session_id == right.session_id - && left.parent_session_id == right.parent_session_id - && left.depth == right.depth - }) -} - -fn establish_agent_org_delete_fence( - expected_plan: &AgentOrgSessionDeletePlan, -) -> Result { - let (current_plan, changed) = with_sessions_writer(|| { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let mut current_plan = - load_agent_org_session_delete_plan(&tx, &expected_plan.root_session_id)?.ok_or_else( - || { - format!( - "Refusing to delete Agent Org run {}: root ownership changed before stopping", - expected_plan.run_id - ) - }, - )?; - if !agent_org_delete_topology_matches(expected_plan, ¤t_plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed before stopping", - expected_plan.run_id - )); - } - - let changed = match current_plan.run_status { - crate::coordination::agent_org_runs::AgentOrgRunStatus::Running - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Paused => { - let changed = - AgentOrgRunStore::cancel_for_delete_with_connection(&tx, ¤t_plan.run_id)?; - if !changed { - return Err(format!( - "Refusing to delete Agent Org run {}: run status changed before cancellation", - current_plan.run_id - )); - } - current_plan.run_status = - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled; - true - } - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled => false, - status if status.is_terminal() => false, - status => { - return Err(format!( - "Refusing to delete Agent Org run {}: unsupported run status {}", - current_plan.run_id, - status.as_str() - )); - } - }; - tx.commit().map_err(|err| err.to_string())?; - Ok::<_, String>((current_plan, changed)) - })?; - if changed { - crate::coordination::agent_org_run_events::notify_agent_org_run_changed( - ¤t_plan.run_id, - ); - } - Ok(current_plan) -} - fn validate_agent_org_delete_ready( plan: &AgentOrgSessionDeletePlan, - quiesced_runtime_session_ids: &HashSet, + _quiesced_runtime_session_ids: &HashSet, ) -> Result<(), String> { - if !plan.run_status.is_terminal() { + if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { return Err(format!( "Refusing to delete Agent Org run {}: run status is {}", plan.run_id, @@ -466,13 +353,7 @@ fn validate_agent_org_delete_ready( } for node in &plan.sessions { - let allowed = node.status == SessionStatus::Idle - || node.status.is_terminal() - || (plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - && (matches!(node.status, SessionStatus::Pending | SessionStatus::Paused) - || (node.status.is_in_flight() - && quiesced_runtime_session_ids.contains(&node.session_id)))); + let allowed = node.status == SessionStatus::Idle || node.status.is_terminal(); if !allowed { return Err(format!( "Refusing to delete Agent Org run {}: session {} status is {}", @@ -518,47 +399,6 @@ async fn agent_org_runtime_blockers( blockers } -async fn stop_agent_org_runtime_sessions( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Result, String> { - stop_agent_org_runtime_sessions_with_timeout(state, plan, AGENT_ORG_DELETE_STOP_TIMEOUT).await -} - -async fn stop_agent_org_runtime_sessions_with_timeout( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, - timeout: Duration, -) -> Result, String> { - let runtime_sessions = agent_org_runtime_sessions(state, plan).await; - let runtime_session_ids = runtime_sessions - .iter() - .map(|(session_id, _)| session_id.clone()) - .collect::>(); - - for (_, session) in &runtime_sessions { - session - .cancel_active_turn(CancelReason::AgentOrgDelete) - .await; - } - - let deadline = tokio::time::Instant::now() + timeout; - loop { - let blockers = agent_org_runtime_blockers(&runtime_sessions).await; - if blockers.is_empty() { - return Ok(runtime_session_ids); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "Timed out stopping Agent Org run {} before deletion: {}", - plan.run_id, - blockers.join(", ") - )); - } - tokio::time::sleep(AGENT_ORG_DELETE_STOP_POLL_INTERVAL).await; - } -} - async fn ensure_agent_org_runtime_sessions_idle( state: &AgentAppState, plan: &AgentOrgSessionDeletePlan, @@ -1102,7 +942,7 @@ mod tests { } fn seed_run(run_id: &str, root_session_id: &str) { - seed_run_with_status(run_id, root_session_id, "completed"); + seed_run_with_status(run_id, root_session_id, "archived"); } fn seed_session_owned_rows(session_id: &str) { @@ -1273,7 +1113,7 @@ mod tests { } #[test] - fn session_hierarchy_delete_fences_active_run_and_requires_quiesced_sessions() { + fn session_hierarchy_delete_requires_archived_without_mutating_active_run() { let _sandbox = test_helpers::test_env::sandbox(); ensure_test_schemas(); let root = "hierarchy-active-root"; @@ -1287,11 +1127,9 @@ mod tests { .expect("load running hierarchy") .expect("root owns run"); drop(conn); - let fenced = establish_agent_org_delete_fence(&plan).expect("cancel run for deletion"); - assert_eq!( - fenced.run_status, - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - ); + let error = validate_agent_org_delete_ready(&plan, &HashSet::new()) + .expect_err("Delete must fail closed before the Archive transition exists"); + assert!(error.contains("run status is running")); assert_eq!( get_connection() .expect("sandbox DB") @@ -1300,18 +1138,9 @@ mod tests { [], |row| row.get::<_, String>(0) ) - .expect("load fenced status"), - "cancelled" + .expect("load unchanged status"), + "running" ); - - let error = validate_agent_org_delete_ready(&fenced, &HashSet::new()) - .expect_err("unobserved running worker must fail closed"); - assert!(error.contains(worker)); - assert!(error.contains("running")); - - let quiesced = HashSet::from([worker.to_string()]); - validate_agent_org_delete_ready(&fenced, &quiesced) - .expect("a stopped live runtime may retain a stale running row"); assert!(row_exists("agent_sessions", "session_id", root)); assert!(row_exists("agent_sessions", "session_id", worker)); assert!(row_exists("agent_org_runs", "id", "hierarchy-active-run")); @@ -1600,185 +1429,4 @@ mod tests { "hierarchy-trigger-change-run" )); } - - #[tokio::test] - async fn session_hierarchy_delete_stops_active_runtime_and_discards_pending_work() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-root"; - let state = AgentAppState::new(); - let root_runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let turn_started = std::sync::Arc::new(tokio::sync::Notify::new()); - let turn_started_for_job = std::sync::Arc::clone(&turn_started); - let runtime_for_job = std::sync::Arc::clone(&root_runtime); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-processing".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-processing-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let runtime = std::sync::Arc::clone(&runtime_for_job); - let started = std::sync::Arc::clone(&turn_started_for_job); - Box::pin(async move { - runtime.begin_turn("still running".to_string()).await; - started.notify_one(); - while !runtime - .cancel_flag - .load(std::sync::atomic::Ordering::SeqCst) - { - tokio::task::yield_now().await; - } - runtime - .end_turn( - crate::session::DialogTurnState::Cancelled, - crate::session::TurnStats::default(), - ) - .await; - Err("cancelled for hierarchy deletion".to_string()) - }) - }), - }) - .await - .expect("enqueue processing work"); - tokio::time::timeout(std::time::Duration::from_secs(1), turn_started.notified()) - .await - .expect("turn starts processing"); - let pending_executed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let pending_executed_for_job = std::sync::Arc::clone(&pending_executed); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-pending".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-pending-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let executed = std::sync::Arc::clone(&pending_executed_for_job); - Box::pin(async move { - executed.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue pending work"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&root_runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let quiesced = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_secs(1), - ) - .await - .expect("active Rust runtime stops"); - assert_eq!(quiesced, HashSet::from([root.to_string()])); - assert_eq!(root_runtime.scheduler.pending_count(), 0); - assert!(!root_runtime.scheduler.is_processing()); - assert!(root_runtime.active_turn.lock().await.is_none()); - assert!(!pending_executed.load(std::sync::atomic::Ordering::SeqCst)); - validate_agent_org_delete_ready(&plan, &quiesced) - .expect("quiesced active status is safe behind cancelled fence"); - } - - #[tokio::test] - async fn session_hierarchy_delete_times_out_without_removing_runtime() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-timeout-root"; - let state = AgentAppState::new(); - let runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let release = std::sync::Arc::new(tokio::sync::Notify::new()); - let release_for_job = std::sync::Arc::clone(&release); - runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Maintenance, - message_id: "hierarchy-runtime-timeout".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-timeout-intent".to_string(), - org_run_id: Some("hierarchy-runtime-timeout-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let release = std::sync::Arc::clone(&release_for_job); - Box::pin(async move { - release.notified().await; - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue non-cooperative maintenance"); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance starts"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-timeout-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let error = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_millis(50), - ) - .await - .expect_err("non-cooperative work must time out"); - assert!(error.contains("Timed out stopping")); - assert!(error.contains(root)); - assert!(state.get_session(root).await.is_some()); - release.notify_one(); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance finishes after the timeout assertion"); - } } diff --git a/src-tauri/crates/agent-core/src/state/unified.rs b/src-tauri/crates/agent-core/src/state/unified.rs index 14ea6f05c9..413bd51af1 100644 --- a/src-tauri/crates/agent-core/src/state/unified.rs +++ b/src-tauri/crates/agent-core/src/state/unified.rs @@ -165,7 +165,7 @@ impl AgentAppState { } // Interventions cannot survive a process restart: their in-memory - // sessions were abandoned above. Clear them before finality checks so + // sessions were abandoned above. Clear them before Quiescence checks so // a fully-resolved run is not needlessly paused by an expired control // lease from the previous process. match crate::coordination::agent_member_interventions::AgentMemberInterventionStore::clear_all_active_on_startup() { @@ -180,38 +180,6 @@ impl AgentAppState { ), } - // Runs whose tasks were already resolved may have been kept open only - // by an orphaned queued intent. Close them through the normal atomic - // finality path before pausing genuinely unfinished work. - match crate::coordination::agent_org_runs::AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() { - Ok(0) => {} - Ok(n) => info!( - "[agent-state] Completed {} fully-resolved Agent Org run(s) during startup recovery", - n - ), - Err(err) => warn!( - "[agent-state] Failed to reconcile resolved Agent Org runs on startup: {}", - err - ), - } - - // Transition any Agent Org runs that were `running` when the previous - // process exited to `paused`. Their member sessions are now `abandoned` - // (see above), so `reconcile_run_finality` would auto-terminate the run - // if it remained `running`. By moving to `paused` instead, the run stays - // visible (non-terminal) and can be resumed from the UI. - match crate::coordination::agent_org_runs::AgentOrgRunStore::mark_all_running_as_paused_on_startup() { - Ok(0) => {} - Ok(n) => info!( - "[agent-state] Paused {} Agent Org run(s) interrupted by app exit", - n - ), - Err(err) => warn!( - "[agent-state] Failed to pause interrupted Agent Org runs on startup: {}", - err - ), - } - let bus = Arc::new(Mutex::new(AgentMessageBus::new())); let sessions: Arc>>> = Arc::new(Mutex::new(HashMap::new())); diff --git a/src-tauri/crates/session-persistence/src/agent_core_bridge.rs b/src-tauri/crates/session-persistence/src/agent_core_bridge.rs index 3c296f918d..b92c63e21c 100644 --- a/src-tauri/crates/session-persistence/src/agent_core_bridge.rs +++ b/src-tauri/crates/session-persistence/src/agent_core_bridge.rs @@ -154,6 +154,28 @@ fn upsert_turn_intent_adapter( } } +fn upsert_turn_intent_with_connection_adapter( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: session_bridge::TurnIntentBridgeSource, + status: session_bridge::TurnIntentBridgeStatus, +) -> Result<(), String> { + turn_intents::upsert_initial_on( + connection, + session_id, + turn_intent_id, + client_message_id, + org_run_id, + map_bridge_source(source), + map_bridge_status(status), + ) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + fn update_turn_intent_status_adapter( session_id: &str, turn_intent_id: &str, @@ -209,6 +231,9 @@ pub fn register() { session_bridge::register_record_token_usage(record_token_usage_adapter); session_bridge::register_record_usage_telemetry_batch(record_usage_telemetry_batch_adapter); session_bridge::register_upsert_turn_intent(upsert_turn_intent_adapter); + session_bridge::register_upsert_turn_intent_with_connection( + upsert_turn_intent_with_connection_adapter, + ); session_bridge::register_update_turn_intent_status(update_turn_intent_status_adapter); session_bridge::register_get_turn_intent_status(get_turn_intent_status_adapter); session_bridge::register_mark_pending_turn_intents_stale( diff --git a/src-tauri/crates/session-persistence/src/turn_intents.rs b/src-tauri/crates/session-persistence/src/turn_intents.rs index 1f8aeeb6f2..b4e1bf3eb0 100644 --- a/src-tauri/crates/session-persistence/src/turn_intents.rs +++ b/src-tauri/crates/session-persistence/src/turn_intents.rs @@ -421,13 +421,13 @@ pub fn mark_pending_stale(session_id: &str) -> Result { Ok(affected) } -/// Close every in-flight intent left by a previous process. +/// Close ordinary SDE in-flight intents left by a previous process. /// /// The scheduler queue is memory-only, so after a process restart no /// `optimistic` or `queued` row can still execute; they are stale. A `running` /// row means the process died during execution and is recorded as failed. -/// This is intentionally connection-scoped so the app can call it from the -/// database initialization hook without recursively opening the database. +/// Agent Org intents have their own durable Starting recovery and must be +/// reconciled only after the Agent Org schema is available. pub fn reconcile_in_flight_after_restart(conn: &Connection) -> Result { let now = Utc::now().to_rfc3339(); let affected = conn.execute( @@ -437,7 +437,37 @@ pub fn reconcile_in_flight_after_restart(conn: &Connection) -> Result Result { + let now = Utc::now().to_rfc3339(); + let affected = conn.execute( + "UPDATE session_turn_intents + SET status = 'stale', updated_at = ?1 + WHERE org_run_id IS NOT NULL + AND status IN ('optimistic', 'queued') + AND NOT ( + status = 'queued' + AND EXISTS ( + SELECT 1 FROM agent_org_initial_inputs initial + WHERE initial.org_run_id=session_turn_intents.org_run_id + AND initial.turn_intent_id=session_turn_intents.turn_intent_id + AND initial.status IN ('queued', 'dispatched') + ) + )", [now], )?; Ok(affected) @@ -822,7 +852,7 @@ mod tests { } #[test] - fn restart_reconciliation_closes_every_in_flight_intent() { + fn restart_reconciliation_closes_every_generic_in_flight_intent() { with_temp_orgii_home(|| { let session = "test-session-restart-intents"; for (intent, status) in [ @@ -870,4 +900,84 @@ mod tests { assert_eq!(by_id["completed-d"], TurnIntentStatus::Completed); }); } + + #[test] + fn agent_org_restart_preserves_running_and_replayable_initial_input_only() { + with_temp_orgii_home(|| { + let session = "test-agent-org-restart"; + let run_id = "agent-org-restart-run"; + let conn = get_connection().expect("open sessions DB"); + agent_core::coordination::init_agent_org_schemas(&conn) + .expect("init Agent Org schemas"); + let now = Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, + entry_mode, status, has_initial_work, created_at, updated_at + ) VALUES (?1, 'restart-org', 'coordinator', ?2, + 'standalone_session', 'running', 1, ?3, ?3)", + params![run_id, session, &now], + ) + .expect("seed run"); + + for (intent, status) in [ + ("optimistic-noninitial", TurnIntentStatus::Optimistic), + ("queued-noninitial", TurnIntentStatus::Queued), + ("running-final-not-committed", TurnIntentStatus::Running), + ("queued-canonical-initial", TurnIntentStatus::Queued), + ] { + upsert_initial( + session, + intent, + Some(&format!("message-{intent}")), + Some(run_id), + TurnIntentSource::AgentOrg, + status, + ) + .expect("seed Agent Org intent"); + } + conn.execute( + "INSERT INTO agent_org_initial_inputs ( + org_run_id, turn_intent_id, message_id, content, + payload_json, status, created_at, updated_at + ) VALUES (?1, 'queued-canonical-initial', 'initial-message', + 'initial input', ?2, 'queued', ?3, ?3)", + params![ + run_id, + serde_json::json!({ + "version": 1, + "images": null, + "ideContext": null, + "subAgentIds": [], + }) + .to_string(), + &now, + ], + ) + .expect("seed canonical initial input receipt"); + + assert_eq!(reconcile_in_flight_after_restart(&conn).unwrap(), 0); + assert_eq!( + reconcile_agent_org_in_flight_after_restart(&conn).unwrap(), + 2 + ); + let rows = list_for_session(session).expect("load reconciled intents"); + let by_id = rows + .into_iter() + .map(|row| (row.turn_intent_id, row.status)) + .collect::>(); + assert_eq!(by_id["optimistic-noninitial"], TurnIntentStatus::Stale); + assert_eq!(by_id["queued-noninitial"], TurnIntentStatus::Stale); + assert_eq!( + by_id["running-final-not-committed"], + TurnIntentStatus::Running, + "unknown post-crash side effects and an uncommitted final answer must block Idle" + ); + assert_eq!( + by_id["queued-canonical-initial"], + TurnIntentStatus::Queued, + "only the stable initial input receipt is safe to replay with the same ids" + ); + }); + } } diff --git a/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs b/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs index fcd8d170ed..45cd1da6f3 100644 --- a/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs +++ b/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs @@ -267,8 +267,8 @@ pub(super) async fn finalize_session_run( // CLI member sessions inside an Agent Org run must land on `Idle` after each // successful turn so they remain available for the next coordinator dispatch. - // `Completed` is terminal (is_terminal() == true) and would cause - // `reconcile_run_finality` to prematurely end the run. + // `Completed` is terminal (is_terminal() == true) and would make the + // member unavailable to the canonical Team Quiescence projection. let is_org_member = session.org_member_id.is_some(); let final_status = if raw_final_status == SessionStatus::Completed && is_org_member { SessionStatus::Idle diff --git a/src-tauri/src/agent_sessions/cli/types.rs b/src-tauri/src/agent_sessions/cli/types.rs index 7c20829188..62bf82e199 100644 --- a/src-tauri/src/agent_sessions/cli/types.rs +++ b/src-tauri/src/agent_sessions/cli/types.rs @@ -20,7 +20,7 @@ pub enum SessionStatus { Running, /// Session is idle — waiting for the next dispatch (non-terminal). /// Used for Agent Org member sessions after each successful turn so - /// `reconcile_run_finality` does not prematurely end the run. + /// Team Quiescence can distinguish a reusable member from active work. Idle, Completed, Failed, diff --git a/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs b/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs index 9bb7c1d078..e63275b6db 100644 --- a/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs +++ b/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs @@ -398,9 +398,9 @@ fn persist_events_adapter( session_id: &str, events: &[SessionEvent], max_retries: u32, -) { +) -> Result<(), String> { let cached: Vec<_> = events.iter().map(session_event_to_cached_event).collect(); - let _ = save_events_retry(label, session_id, &cached, max_retries); + save_events_retry(label, session_id, &cached, max_retries).map_err(|error| error.to_string()) } fn persist_events_async_adapter( diff --git a/src-tauri/src/api/agent/test/agent_org.rs b/src-tauri/src/api/agent/test/agent_org.rs index 8820ad53f3..bab0589a0e 100644 --- a/src-tauri/src/api/agent/test/agent_org.rs +++ b/src-tauri/src/api/agent/test/agent_org.rs @@ -2991,7 +2991,7 @@ pub async fn test_agent_org_tasks_list( /// /// Seeds a minimal Agent Org run with a CLI member session at a specified /// status in `code_sessions`. Used by deterministic E2E scenarios that -/// verify `reconcile_run_finality` does not prematurely end a run when a +/// verify Team Quiescence does not prematurely idle a run when a /// CLI member session is `idle` (non-terminal, between turns). /// /// Body: @@ -3206,11 +3206,13 @@ pub async fn test_agent_org_pause_run( /// member task disposition as production startup. /// 5. `clear_all_active_on_startup` — clears interventions whose in-memory /// sessions no longer exist. -/// 6. `reconcile_resolved_running_runs_on_startup` — completes runs whose -/// tasks were already fully resolved. -/// 7. `mark_all_running_as_paused_on_startup` — transitions -/// every `running` org run to `paused` so `reconcile_run_finality` cannot -/// auto-terminate the run when it sees all sessions abandoned. +/// 6. `reconcile_agent_org_in_flight_after_restart` — preserves Agent Org +/// Running intents as explicit quiescence blockers and only retains a +/// replayable queued canonical initial input. +/// +/// Run lifecycle state is deliberately not changed here. In particular, +/// startup never maps `running` to `paused` and never infers terminality from +/// abandoned Session rows. /// /// Caller-path probe: drives the same sequence that `AgentAppState:: /// with_browser` calls, so this endpoint stays in sync if any of those @@ -3228,6 +3230,11 @@ pub async fn test_agent_org_simulate_app_restart() -> Json { let intents_reconciled = session_persistence::turn_intents::reconcile_in_flight_after_restart(&conn) .map_err(|err| format!("reconcile_in_flight_after_restart failed: {err}"))?; + let agent_org_intents_reconciled = + session_persistence::turn_intents::reconcile_agent_org_in_flight_after_restart(&conn) + .map_err(|err| { + format!("reconcile_agent_org_in_flight_after_restart failed: {err}") + })?; let terminal_sessions_reconciled = reconcile_sessions_with_terminal_turn_markers() .map_err(|err| { format!("reconcile_sessions_with_terminal_turn_markers failed: {err}") @@ -3238,18 +3245,17 @@ pub async fn test_agent_org_simulate_app_restart() -> Json { .map_err(|err| format!("requeue_abandoned_member_tasks_on_startup failed: {err}"))?; let interventions_cleared = AgentMemberInterventionStore::clear_all_active_on_startup() .map_err(|err| format!("clear_all_active_on_startup failed: {err}"))?; - let runs_completed = AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .map_err(|err| format!("reconcile_resolved_running_runs_on_startup failed: {err}"))?; - let runs_paused = AgentOrgRunStore::mark_all_running_as_paused_on_startup() - .map_err(|err| format!("mark_all_running_as_paused_on_startup failed: {err}"))?; Ok::(serde_json::json!({ "ok": true, "intents_reconciled": intents_reconciled, + "agent_org_intents_reconciled": agent_org_intents_reconciled, "terminal_sessions_reconciled": terminal_sessions_reconciled, "sessions_abandoned": sessions_abandoned, "tasks_requeued": tasks_requeued, - "runs_completed": runs_completed, - "runs_paused": runs_paused, + // Kept for old E2E clients; canonical PR1 startup performs neither + // transition, so both counters are intentionally always zero. + "runs_completed": 0, + "runs_paused": 0, "interventions_cleared": interventions_cleared, })) }) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7c7795501e..fd6b6f7f84 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -740,8 +740,40 @@ pub fn run() { ); tracing::info!("[MemberIdle] Member idle hook installed"); - agent_core::coordination::agent_org_watchdog::spawn(app.handle().clone()); - tracing::info!("[AgentOrgWatchdog] Agent Org watchdog started"); + if agent_core::coordination::agent_org_runs::agent_org_redesign_enabled() { + agent_core::coordination::agent_org_watchdog::spawn(app.handle().clone()); + tracing::info!("[AgentOrgWatchdog] Agent Org watchdog started"); + } else { + tracing::info!("[AgentOrgWatchdog] Agent Org redesign is disabled"); + } + + // Plan artifacts have their own one-shot startup owner. Keep this + // repair independent from the bounded Working-only watchdog so a + // global filesystem scan can never consume its 250 ms Team scan + // budget or run every 60 seconds. + tauri::async_runtime::spawn(async move { + match tokio::task::spawn_blocking(|| { + agent_core::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore::repair_latest_plan_artifacts() + }) + .await + { + Ok(Ok(report)) if report.repaired > 0 || report.failed > 0 => tracing::info!( + inspected = report.inspected, + repaired = report.repaired, + failed = report.failed, + "[AgentOrgPlanArtifacts] one-shot startup reconciliation finished" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!( + error = %error, + "[AgentOrgPlanArtifacts] startup reconciliation failed" + ), + Err(error) => tracing::warn!( + error = %error, + "[AgentOrgPlanArtifacts] startup worker failed" + ), + } + }); // Install the production `SubagentCompletionWakeHook` so a // background subagent that finishes while its parent is idle @@ -757,9 +789,15 @@ pub fn run() { tracing::info!("[SubagentWake] Subagent completion wake hook installed"); let housekeeper_compaction_state = unified_state.clone(); + let agent_org_startup_state = unified_state.clone(); app.manage(unified_state); tracing::info!("[UnifiedAgent] Unified agent state initialized"); + agent_core::core::session::launch::spawn_agent_org_startup_recovery( + agent_org_startup_state, + ); + tracing::info!("[AgentOrgStartup] one-shot lifecycle recovery scheduled"); + agent_core::session::housekeeper_compaction::spawn( housekeeper_compaction_state, ); diff --git a/src-tauri/src/setup/hooks.rs b/src-tauri/src/setup/hooks.rs index 97f88ddf09..a64424558a 100644 --- a/src-tauri/src/setup/hooks.rs +++ b/src-tauri/src/setup/hooks.rs @@ -65,6 +65,17 @@ pub(crate) fn register_database_schemas() { } agent_core::coordination::init_agent_org_schemas(conn)?; + match session_persistence::turn_intents::reconcile_agent_org_in_flight_after_restart(conn) { + Ok(0) => {} + Ok(count) => tracing::info!( + "[startup] Reconciled {} Agent Org turn intent(s) from the previous process", + count + ), + Err(err) => tracing::warn!( + "[startup] Failed to reconcile Agent Org turn intents: {}", + err + ), + } // Pending plan-approval snapshots (one row per session with a Build // button still awaiting the user). Persists the pending action so the diff --git a/src/api/tauri/agent/orgTasks.ts b/src/api/tauri/agent/orgTasks.ts index a2185b3227..b0139999d1 100644 --- a/src/api/tauri/agent/orgTasks.ts +++ b/src/api/tauri/agent/orgTasks.ts @@ -79,18 +79,19 @@ export interface AgentOrgRunMemberView { } export const AGENT_ORG_RUN_STATUS = { + STARTING: "starting", RUNNING: "running", PAUSED: "paused", - COMPLETED: "completed", + IDLE: "idle", FAILED: "failed", - CANCELLED: "cancelled", - ABANDONED: "abandoned", + ARCHIVED: "archived", } as const; export type AgentOrgRunStatus = (typeof AGENT_ORG_RUN_STATUS)[keyof typeof AGENT_ORG_RUN_STATUS]; export const AGENT_ORG_RUN_PHASE = { + STARTING: "starting", COORDINATING: "coordinating", DISPATCHING: "dispatching", MEMBERS_WORKING: "members_working", @@ -98,10 +99,9 @@ export const AGENT_ORG_RUN_PHASE = { AWAITING_PLAN_APPROVAL: "awaiting_plan_approval", FINALIZING: "finalizing", PAUSED: "paused", - COMPLETED: "completed", + IDLE: "idle", FAILED: "failed", - CANCELLED: "cancelled", - ABANDONED: "abandoned", + ARCHIVED: "archived", } as const; export type AgentOrgRunPhase = diff --git a/src/api/tauri/agent/session.ts b/src/api/tauri/agent/session.ts index 5cc9606b39..9f3c2cbc57 100644 --- a/src/api/tauri/agent/session.ts +++ b/src/api/tauri/agent/session.ts @@ -7,6 +7,7 @@ import { invoke } from "@tauri-apps/api/core"; import { rpc } from "@src/api/tauri/rpc"; import type { CliAgentType, NativeHarnessType } from "@src/api/types/keys"; +import { requireAgentOrgRedesign } from "@src/config/agentOrgRedesign"; import type { OrgMemberLaunchOverride } from "@src/modules/MainApp/AgentOrgs/types"; import type { WorkspaceSnapshot } from "@src/services/context/workspaceSnapshot"; import type { SessionStatus } from "@src/types/session/session"; @@ -520,6 +521,7 @@ export interface SessionLaunchResult { export async function sessionLaunch( params: SessionLaunchParams ): Promise { + if (params.agentOrgId) requireAgentOrgRedesign(); return rpc.agentSession.sessionLaunch({ params, }) as Promise; diff --git a/src/config/agentOrgRedesign.ts b/src/config/agentOrgRedesign.ts new file mode 100644 index 0000000000..bb0a7cb01f --- /dev/null +++ b/src/config/agentOrgRedesign.ts @@ -0,0 +1,15 @@ +/** + * One internal rollout gate for the long-lived Agent Org redesign stack. + * It is intentionally absent from Team settings and model/tool context. + */ +export const AGENT_ORG_REDESIGN_ENABLED = + process.env.NODE_ENV === "test" || + process.env.ORGII_AGENT_ORG_REDESIGN === "1"; + +export function requireAgentOrgRedesign(): void { + if (!AGENT_ORG_REDESIGN_ENABLED) { + throw new Error( + "agent_org_redesign_disabled: the long-lived Agent Team lifecycle is not enabled" + ); + } +} diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts index b859de01db..f0829e06ca 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts @@ -33,7 +33,7 @@ async function flushPromises(): Promise { } function runView( - runStatus: "running" | "paused" | "completed", + runStatus: "starting" | "running" | "paused" | "idle" | "failed" | "archived", interventionResumeAfter?: string ) { return { @@ -106,6 +106,18 @@ function runView( }; } +function runViewForRoot( + runStatus: "starting" | "running" | "paused" | "idle" | "failed" | "archived", + runId: string, + rootSessionId: string +) { + const view = runView(runStatus); + view.context.runId = runId; + view.context.rootSessionId = rootSessionId; + view.members[0].sessionRuntime.sessionId = rootSessionId; + return view; +} + function deferred() { let resolve!: (value: T) => void; const promise = new Promise((resolvePromise) => { @@ -118,10 +130,11 @@ afterEach(() => { agentOrgRunViewStoreTestApi.reset(); vi.useRealTimers(); vi.clearAllMocks(); + vi.unstubAllGlobals(); }); describe("Agent Org run-view store", () => { - it("shares one fallback per run, coalesces pushes, and stops terminal runs", async () => { + it("shares one fallback per run, coalesces pushes, and stops immediately on Idle", async () => { vi.useFakeTimers(); let stateChangeHandler: ((sessionId: string) => void) | undefined; let backendChangeHandler: @@ -151,7 +164,7 @@ describe("Agent Org run-view store", () => { ); mocks.getAgentOrgSessionRunView .mockResolvedValueOnce(runView("running")) - .mockResolvedValueOnce(runView("completed")); + .mockResolvedValueOnce(runView("idle")); const rootSubscriber = vi.fn(); const secondRootSubscriber = vi.fn(); @@ -213,6 +226,104 @@ describe("Agent Org run-view store", () => { unsubscribe(); }); + it.each(["paused", "idle", "failed", "archived"] as const)( + "does not retain a fallback interval when the initial Team is %s", + async (status) => { + vi.useFakeTimers(); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockResolvedValue(runView(status)); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + unsubscribe(); + } + ); + + it("destroys the shared interval when the last pollable Team becomes Idle", async () => { + vi.useFakeTimers(); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => { + if (sessionId === "root-a") { + return Promise.resolve(runViewForRoot("running", "run-a", "root-a")); + } + return Promise.resolve(runViewForRoot("running", "run-b", "root-b")); + }); + + const unsubscribeA = subscribeAgentOrgRunView("root-a", vi.fn()); + const unsubscribeB = subscribeAgentOrgRunView("root-b", vi.fn()); + await flushPromises(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => + Promise.resolve( + sessionId === "root-a" + ? runViewForRoot("idle", "run-a", "root-a") + : runViewForRoot("running", "run-b", "root-b") + ) + ); + await agentOrgRunViewStoreTestApi.refresh("root-a"); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => + Promise.resolve( + sessionId === "root-b" + ? runViewForRoot("idle", "run-b", "root-b") + : runViewForRoot("idle", "run-a", "root-a") + ) + ); + await agentOrgRunViewStoreTestApi.refresh("root-b"); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + + unsubscribeA(); + unsubscribeB(); + }); + + it("clears polling while hidden and performs one bounded refresh when visible", async () => { + vi.useFakeTimers(); + let hidden = false; + let visibilityChange: (() => void) | undefined; + vi.stubGlobal("document", { + get hidden() { + return hidden; + }, + addEventListener: vi.fn((event: string, handler: () => void) => { + if (event === "visibilitychange") visibilityChange = handler; + }), + removeEventListener: vi.fn(), + }); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockResolvedValue(runView("running")); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + + hidden = true; + visibilityChange?.(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + + hidden = false; + visibilityChange?.(); + await flushPromises(); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(2); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + unsubscribe(); + }); + it("refreshes a retained view immediately when the session is reopened", async () => { vi.useFakeTimers(); mocks.subscribeAgentOrgStateChanges.mockReturnValue( @@ -263,13 +374,13 @@ describe("Agent Org run-view store", () => { // discovery hangs, the second is released after the bounded join timeout; // request ordering must still reject the first request's late result. await vi.advanceTimersByTimeAsync(AGENT_ORG_BOOTSTRAP_JOIN_TIMEOUT_MS); - workerRequest.resolve(runView("completed")); + workerRequest.resolve(runView("idle")); await flushPromises(); rootRequest.resolve(runView("running")); await flushPromises(); expect(getAgentOrgRunViewSnapshot("session-root").view?.runStatus).toBe( - "completed" + "idle" ); unsubscribeRoot(); unsubscribeWorker(); diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts index 38252a6e86..b5a58f6f10 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts @@ -12,11 +12,9 @@ export const AGENT_ORG_RUN_VIEW_CACHE_RETENTION_MS = 30_000; export const AGENT_ORG_BOOTSTRAP_JOIN_TIMEOUT_MS = 1_000; const MAX_NON_ORG_DISCOVERY_ATTEMPTS = 1; -const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ - "completed", - "failed", - "cancelled", - "abandoned", +export const POLLABLE_RUN_STATUSES: ReadonlySet = new Set([ + "starting", + "running", ]); export interface AgentOrgRunViewSnapshot { @@ -94,8 +92,8 @@ function viewForSession( return { ...view, currentMemberId }; } -function isTerminal(view: AgentOrgRunView | null): boolean { - return view !== null && TERMINAL_RUN_STATUSES.has(view.runStatus); +function isPollable(view: AgentOrgRunView | null): boolean { + return view !== null && POLLABLE_RUN_STATUSES.has(view.runStatus); } function findEntryCoveringSession(sessionId: string): RunViewEntry | undefined { @@ -138,6 +136,7 @@ function evictEntry(entry: RunViewEntry): void { if (!isCurrentEntry(entry) || entry.subscribers.size > 0) return; entry.retired = true; entriesBySessionId.delete(entry.sessionId); + reconcilePollingTimer(); if (bootstrapOwner === entry) bootstrapOwner = null; const sessionKey = `session:${entry.sessionId}`; @@ -207,6 +206,7 @@ function publishEntry( entry.snapshot = { view, error }; entry.serializedView = serializedView; for (const subscriber of entry.subscribers) subscriber(); + reconcilePollingTimer(); return true; } @@ -438,7 +438,10 @@ function pollActiveRuns(): void { const representatives = new Map(); for (const entry of entriesBySessionId.values()) { - if (entry.subscribers.size === 0 || isTerminal(entry.snapshot.view)) + if ( + entry.subscribers.size === 0 || + (entry.snapshot.view !== null && !isPollable(entry.snapshot.view)) + ) continue; if ( entry.snapshot.view === null && @@ -454,13 +457,42 @@ function pollActiveRuns(): void { } function handleVisibilityChange(): void { - if (isDocumentVisible()) pollActiveRuns(); + if (!isDocumentVisible()) { + reconcilePollingTimer(); + return; + } + pollActiveRuns(); + reconcilePollingTimer(); } -function startScheduler(): void { +function hasPollableSubscriber(): boolean { + for (const entry of entriesBySessionId.values()) { + if (entry.subscribers.size === 0) continue; + if (entry.snapshot.view !== null) { + if (isPollable(entry.snapshot.view)) return true; + continue; + } + if (entry.discoveryAttempts < MAX_NON_ORG_DISCOVERY_ATTEMPTS) return true; + } + return false; +} + +function reconcilePollingTimer(): void { + const shouldPoll = + activeSubscriberCount > 0 && isDocumentVisible() && hasPollableSubscriber(); + if (!shouldPoll) { + if (pollingTimer) { + clearInterval(pollingTimer); + pollingTimer = undefined; + } + return; + } if (!pollingTimer) { pollingTimer = setInterval(pollActiveRuns, AGENT_ORG_RUN_VIEW_FALLBACK_MS); } +} + +function startScheduler(): void { if (!unsubscribeStateChanges) { unsubscribeStateChanges = subscribeAgentOrgStateChanges((sessionId) => { const entry = @@ -514,6 +546,7 @@ function startScheduler(): void { scheduledRuns.add(view.context.runId); scheduleInterventionExpiryRefresh(view); } + reconcilePollingTimer(); } function stopScheduler(): void { @@ -552,6 +585,7 @@ export function subscribeAgentOrgRunView( entry.subscribers.add(subscription); activeSubscriberCount += 1; if (activeSubscriberCount === 1) startScheduler(); + else reconcilePollingTimer(); if (entry.snapshot.view === null && entry.discoveryAttempts === 0) { void refreshAgentOrgRunViewInternal(sessionId, false); } else if (isReturningSubscriber && entry.snapshot.view !== null) { @@ -565,6 +599,7 @@ export function subscribeAgentOrgRunView( if (!entry.subscribers.delete(subscription)) return; activeSubscriberCount -= 1; if (activeSubscriberCount === 0) stopScheduler(); + else reconcilePollingTimer(); scheduleEntryEviction(entry); }; } @@ -592,6 +627,9 @@ export const agentOrgRunViewStoreTestApi = { )?.sessionId ?? null ); }, + hasPollingTimer(): boolean { + return pollingTimer !== undefined; + }, reset(): void { stopScheduler(); for (const entry of entriesBySessionId.values()) { diff --git a/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts b/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts index a1582aefaa..696f8e7149 100644 --- a/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts +++ b/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts @@ -1,5 +1,6 @@ import { useCallback, useMemo, useSyncExternalStore } from "react"; +import { AGENT_ORG_REDESIGN_ENABLED } from "@src/config/agentOrgRedesign"; import { getAgentOrgRunViewSnapshot, refreshAgentOrgRunView, @@ -21,6 +22,7 @@ const EMPTY_RESULT = { view: null, error: null } as const; */ export function useAgentOrgRunView(sessionId: string | null) { const canFetchRunView = + AGENT_ORG_REDESIGN_ENABLED && !!sessionId && !isCliSession(sessionId) && !isImportedHistorySession(sessionId); diff --git a/src/i18n/locales/en/sessions.json b/src/i18n/locales/en/sessions.json index 93f0f63114..aff5b7dfe3 100644 --- a/src/i18n/locales/en/sessions.json +++ b/src/i18n/locales/en/sessions.json @@ -2518,6 +2518,7 @@ "submitFailed": "The plan response could not be saved. Please try again." }, "phase": { + "starting": "Starting", "coordinating": "Coordinating", "dispatching": "Dispatching", "members_working": "Members working", @@ -2525,10 +2526,9 @@ "awaiting_plan_approval": "Awaiting plan approval", "finalizing": "Finalizing", "paused": "Paused", - "completed": "Completed", + "idle": "Idle", "failed": "Failed", - "cancelled": "Cancelled", - "abandoned": "Abandoned" + "archived": "Archived" } }, "agentOrgInbox": { diff --git a/src/i18n/locales/zh/sessions.json b/src/i18n/locales/zh/sessions.json index 8bd60e4c98..7d8dac511f 100644 --- a/src/i18n/locales/zh/sessions.json +++ b/src/i18n/locales/zh/sessions.json @@ -2492,6 +2492,7 @@ "submitFailed": "计划审批结果未能保存，请重试。" }, "phase": { + "starting": "正在建立团队", "coordinating": "正在协调", "dispatching": "正在派发", "members_working": "成员工作中", @@ -2499,10 +2500,9 @@ "awaiting_plan_approval": "等待计划审批", "finalizing": "正在收尾", "paused": "已暂停", - "completed": "已完成", + "idle": "待命", "failed": "失败", - "cancelled": "已取消", - "abandoned": "已放弃" + "archived": "已归档" } }, "agentOrgInbox": { diff --git a/webpack.config.js b/webpack.config.js index 0ad0e91df9..b43d3895e5 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -564,6 +564,9 @@ module.exports = (env, argv) => { "process.env.ORGII_DEEP_LINK_SCHEME": JSON.stringify( process.env.ORGII_DEEP_LINK_SCHEME ?? "orgii" ), + "process.env.ORGII_AGENT_ORG_REDESIGN": JSON.stringify( + process.env.ORGII_AGENT_ORG_REDESIGN ?? "0" + ), "process.env.E2E_BASE_URL": JSON.stringify( process.env.E2E_BASE_URL ?? `http://127.0.0.1:${process.env.ORGII_IDE_SERVER_PORT ?? "13847"}` From afb87aef8d635f1da80ab45507c5fd017e57d2f9 Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:17:46 -0700 Subject: [PATCH 02/11] fix(agent-org): legacy passthrough when the redesign gate is off With ORGII_AGENT_ORG_REDESIGN unset (production default) the send preflight hard-failed require_agent_org_redesign() for any session with a persisted org_member_id, bricking every pre-redesign org session. Gate-off now skips the Team lifecycle fence entirely: the message flows as an ordinary session turn (old behavior), and the turn claims no durable run ownership so the execute-time org promote cannot silently no-op it. Per-endpoint gate-off decisions for the remaining require sites, based on each frontend consumer: - agent_org_session_run_view -> Ok(None). READ; polled for arbitrary sessions; None is the canonical 'not an org session' reply. - agent_org_group_chat_history_page -> empty page. READ; fetched unconditionally once the Group Chat view mounts. - agent_org_run_list -> Ok(vec![]). READ; Inbox flat list renders empty. - agent_org_session_enter_intervention -> Ok(false). Called by the CLI transport for every direct user message; interventions do not exist pre-redesign, so 'nothing changed' is correct instead of a warn-logged agent_org_redesign_disabled error per send. - agent_org_session_intervention_state -> intervention: None. READ. - Kept the structured error (fail closed): pause/resume, return-to-work, send-user-message-to-member, group-chat send, plan-approval detail and respond, and the AgentOrg launch target. All are mutations (or reads reachable only through a non-null run view) that must not pretend the redesign lifecycle exists while it is disabled. Note: rollout::is_enabled() is hardwired on in cfg(test) builds, so the gate-off branches are exercised only in production configuration; the configured_enabled() unit tests continue to pin the gate parsing. --- .../src/core/definitions/commands.rs | 7 +++- .../state/commands/session/message/send.rs | 38 +++++++++++++------ .../commands/session/org_tasks/group_chat.rs | 11 +++++- .../session/org_tasks/intervention.rs | 14 ++++++- .../commands/session/org_tasks/run_view.rs | 8 +++- 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/definitions/commands.rs b/src-tauri/crates/agent-core/src/core/definitions/commands.rs index 0b5ab93091..65b8290d8a 100644 --- a/src-tauri/crates/agent-core/src/core/definitions/commands.rs +++ b/src-tauri/crates/agent-core/src/core/definitions/commands.rs @@ -128,7 +128,12 @@ pub struct InboxRunSummary { #[tauri::command] pub async fn agent_org_run_list(limit: Option) -> Result, String> { use crate::core::coordination::agent_org_runs::AgentOrgRunStore; - crate::core::coordination::agent_org_runs::require_agent_org_redesign()?; + // Gate off (production default): the Inbox list is a read surface; an + // empty list ("no org chats") degrades gracefully where a raw + // `agent_org_redesign_disabled` error would break the whole Inbox render. + if !crate::core::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(Vec::new()); + } const MAX_LIMIT: usize = 200; let effective_limit = limit.map(|n| n.min(MAX_LIMIT)).unwrap_or(MAX_LIMIT); // This command backs a read-only Inbox list. Quiescence reconciliation is a diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs index 949fa37e97..83dd7a52cd 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs @@ -59,6 +59,14 @@ async fn preflight_agent_org_turn_before_runtime( run_id_hint: Option<&str>, has_persisted_agent_org_identity: bool, ) -> Result, String> { + // Legacy passthrough: with the redesign gate off (the production default) + // the durable Team lifecycle fence does not exist. A session persisted by + // the pre-redesign stack still carries `org_member_id`, so hard-failing + // here would brick every historical org session. Skip the fence entirely + // and let the message flow as an ordinary session turn (old behavior). + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(None); + } if let (Some(explicit), Some(hint)) = (explicit_run_id, run_id_hint) { if explicit != hint { return Err(format!( @@ -86,7 +94,6 @@ async fn preflight_agent_org_turn_before_runtime( return Ok(None); }; - crate::coordination::agent_org_runs::require_agent_org_redesign()?; let status_run_id = run_id.clone(); let status = tokio::task::spawn_blocking(move || { crate::coordination::agent_org_runs::AgentOrgRunStore::get_run_status(&status_run_id) @@ -234,18 +241,27 @@ pub(crate) async fn send_message_impl( .agent_org_context .as_ref() .map(|context| context.run_id.clone()); - let effective_intent_org_run_id = match ( - intent_org_run_id.as_deref(), - runtime_org_run_id.as_deref(), + // Gate-off legacy passthrough: without the redesign fence the turn must + // not claim durable Agent Org run ownership either — the execute-time + // promote would otherwise fence on run status and silently no-op the + // turn for legacy runs. Keep the whole path an ordinary session turn. + let effective_intent_org_run_id = if !crate::coordination::agent_org_runs::agent_org_redesign_enabled( ) { - (Some(explicit), Some(runtime_id)) if explicit != runtime_id => { - return Err(format!( - "Agent Org turn intent run mismatch for session {session_id}: explicit run {explicit}, runtime run {runtime_id}" - )); + None + } else { + match ( + intent_org_run_id.as_deref(), + runtime_org_run_id.as_deref(), + ) { + (Some(explicit), Some(runtime_id)) if explicit != runtime_id => { + return Err(format!( + "Agent Org turn intent run mismatch for session {session_id}: explicit run {explicit}, runtime run {runtime_id}" + )); + } + (Some(_), _) => intent_org_run_id, + (None, Some(_)) => runtime_org_run_id, + (None, None) => preflight_org_run_id, } - (Some(_), _) => intent_org_run_id, - (None, Some(_)) => runtime_org_run_id, - (None, None) => preflight_org_run_id, }; // Wingman resume: reopen the bottom bar. On fresh start the frontend diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs index 9c03bb1d2d..20148a776b 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs @@ -77,7 +77,16 @@ pub async fn agent_org_group_chat_history_page_impl( before_id: Option, limit: Option, ) -> Result { - crate::coordination::agent_org_runs::require_agent_org_redesign()?; + // Gate off (production default): history is a read surface the Group Chat + // view fetches unconditionally once mounted. Return an empty page instead + // of a raw `agent_org_redesign_disabled` error. + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(AgentOrgGroupChatHistoryPage { + rows: Vec::new(), + has_more: false, + next_before_id: None, + }); + } if before_id.is_some_and(|id| id <= 0) { return Err("before_id must be a positive Inbox row id".to_string()); } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs index 1f5576ec41..d9fb4891bb 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs @@ -43,7 +43,13 @@ pub async fn agent_org_session_enter_intervention( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { - crate::coordination::agent_org_runs::require_agent_org_redesign()?; + // Gate off (production default): the CLI transport calls this for every + // direct user message without checking session kind first. Interventions + // do not exist pre-redesign, so "nothing changed" is the correct legacy + // answer — not an `agent_org_redesign_disabled` error logged per send. + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(false); + } let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; @@ -77,7 +83,11 @@ pub async fn agent_org_session_intervention_state( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { - crate::coordination::agent_org_runs::require_agent_org_redesign()?; + // Gate off (production default): read surface — report "no intervention" + // instead of a raw `agent_org_redesign_disabled` error. + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(AgentOrgSessionInterventionState { intervention: None }); + } let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(AgentOrgSessionInterventionState { intervention: None }); }; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs index a409fcd3ad..3ac0decf0b 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs @@ -172,7 +172,13 @@ pub async fn agent_org_session_run_view_impl( state: &AgentAppState, session_id: &str, ) -> Result, String> { - crate::coordination::agent_org_runs::require_agent_org_redesign()?; + // Gate off (production default): the frontend polls this projection for + // any session; `None` is the canonical "not an Agent Org session" reply, + // so an unconditional caller degrades gracefully instead of surfacing a + // raw `agent_org_redesign_disabled` error. + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return Ok(None); + } let Some(read_context) = session_org_read_context(state, session_id).await? else { return Ok(None); }; From dd104e27b38f7a11f16ecac7687f3c68c2fe2bea Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:22:44 -0700 Subject: [PATCH 03/11] fix(agent-org): allow deleting failed and idle teams, not only archived MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent_delete_session required run_status == archived, but nothing in the lifecycle ever produces Archived — every org run was permanently undeletable. Deletion is now allowed for the terminal/quiescent statuses idle, failed, and archived; starting/running/paused keep the structured refusal. Idle-delete guard: an idle team must actually be quiescent when the delete commits. The plan is reloaded and revalidated inside the IMMEDIATE delete transaction (status re-check), and for idle runs the transaction additionally refuses when any in-flight (optimistic/queued/running) durable turn intent still references the run — a concurrent re-activation cannot be deleted out from under. Failed/archived teams deliberately skip the intent assertion: a crash-orphaned running intent must not make a dead team immortal. The existing in-memory runtime quiescence check (active turn / scheduler pending) continues to run before planning for all statuses. Tests: failed and idle hierarchies delete cleanly; an idle run with an in-flight intent fails closed; the running-run refusal test is unchanged. --- .../src/state/commands/session/persistence.rs | 98 ++++++++++++++++++- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs index dcd9fac74c..42d78631da 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs @@ -100,10 +100,11 @@ pub async fn agent_delete_session( }); }; - if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { + if !agent_org_run_status_allows_delete(plan.run_status) { return Err(format!( - "Refusing to delete Agent Org run {}: Archive is required before Delete", - plan.run_id + "Refusing to delete Agent Org run {}: run status is {}; only idle, failed, or archived teams can be deleted", + plan.run_id, + plan.run_status.as_str() )); } ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; @@ -340,11 +341,24 @@ fn load_agent_org_session_delete_plan( })) } +/// Terminal/quiescent run statuses that may be deleted. `starting`, `running` +/// and `paused` teams keep the structured refusal: they own (or can resume) +/// in-flight work, so Delete must not race the lifecycle. +fn agent_org_run_status_allows_delete( + status: crate::coordination::agent_org_runs::AgentOrgRunStatus, +) -> bool { + use crate::coordination::agent_org_runs::AgentOrgRunStatus; + matches!( + status, + AgentOrgRunStatus::Idle | AgentOrgRunStatus::Failed | AgentOrgRunStatus::Archived + ) +} + fn validate_agent_org_delete_ready( plan: &AgentOrgSessionDeletePlan, _quiesced_runtime_session_ids: &HashSet, ) -> Result<(), String> { - if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { + if !agent_org_run_status_allows_delete(plan.run_status) { return Err(format!( "Refusing to delete Agent Org run {}: run status is {}", plan.run_id, @@ -444,6 +458,28 @@ fn delete_agent_org_session_hierarchy( )); } validate_agent_org_delete_ready(¤t_plan, quiesced_runtime_session_ids)?; + // An idle team must actually be quiescent when the delete commits: a + // run being re-activated concurrently could hold in-flight durable + // turn intents even though the plan snapshot looked settled. Failed + // and archived teams stay deletable — a crash-orphaned intent must + // not make a dead team immortal. + if current_plan.run_status == crate::coordination::agent_org_runs::AgentOrgRunStatus::Idle { + let in_flight_intents: i64 = tx + .query_row( + "SELECT COUNT(*) FROM session_turn_intents + WHERE org_run_id=?1 + AND status IN ('optimistic', 'queued', 'running')", + params![&expected_plan.run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + if in_flight_intents > 0 { + return Err(format!( + "Refusing to delete Agent Org run {}: {in_flight_intents} in-flight turn intent(s) still reference the idle team", + expected_plan.run_id + )); + } + } for node in &expected_plan.sessions { session_persistence::delete_session_with_connection(&tx, &node.session_id) @@ -1112,6 +1148,60 @@ mod tests { )); } + #[test] + fn session_hierarchy_delete_accepts_failed_and_idle_terminal_runs() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_test_schemas(); + for (suffix, run_status) in [("failed", "failed"), ("idle", "idle")] { + let root = format!("hierarchy-terminal-{suffix}-root"); + let worker = format!("hierarchy-terminal-{suffix}-worker"); + let run_id = format!("hierarchy-terminal-{suffix}-run"); + seed_session(&root, None); + seed_session_with_status(&worker, Some(&root), "failed"); + seed_run_with_status(&run_id, &root, run_status); + + let conn = get_connection().expect("sandbox DB"); + let plan = load_agent_org_session_delete_plan(&conn, &root) + .expect("plan terminal hierarchy") + .expect("root owns run"); + drop(conn); + delete_agent_org_session_hierarchy(&plan, &HashSet::new()) + .unwrap_or_else(|err| panic!("{run_status} run must be deletable: {err}")); + assert!(!row_exists("agent_sessions", "session_id", &root)); + assert!(!row_exists("agent_sessions", "session_id", &worker)); + assert!(!row_exists("agent_org_runs", "id", &run_id)); + } + } + + #[test] + fn session_hierarchy_delete_refuses_idle_run_with_in_flight_turn_intent() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_test_schemas(); + let root = "hierarchy-idle-intent-root"; + let run_id = "hierarchy-idle-intent-run"; + seed_session(root, None); + seed_run_with_status(run_id, root, "idle"); + let conn = get_connection().expect("sandbox DB"); + conn.execute( + "INSERT INTO session_turn_intents ( + session_id, turn_intent_id, org_run_id, source, status, + created_at, updated_at + ) VALUES (?1, 'idle-wedged-intent', ?2, 'agent_org', 'running', ?3, ?3)", + rusqlite::params![root, run_id, "2026-07-16T00:00:00Z"], + ) + .expect("seed in-flight intent"); + let plan = load_agent_org_session_delete_plan(&conn, root) + .expect("plan idle hierarchy") + .expect("root owns run"); + drop(conn); + + let error = delete_agent_org_session_hierarchy(&plan, &HashSet::new()) + .expect_err("idle run with in-flight intent must fail closed"); + assert!(error.contains("in-flight turn intent"), "{error}"); + assert!(row_exists("agent_sessions", "session_id", root)); + assert!(row_exists("agent_org_runs", "id", run_id)); + } + #[test] fn session_hierarchy_delete_requires_archived_without_mutating_active_run() { let _sandbox = test_helpers::test_env::sandbox(); From 57e7e2c842dc96d761dab7a986aed05b72290b6e Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:23:22 -0700 Subject: [PATCH 04/11] fix(agent-org): typed permanent-error classification for launch recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit launch_org.rs classified permanent identity failures with .contains("identity mismatch") / .contains("Session identity"), but the store emits 'materialization session mismatch for ...' and 'materialized Session ... is missing for ...' — neither matched, so a permanently wrong identity was retried forever instead of failing Starting. Replace substring matching with stable machine prefixes defined next to the error construction in AgentOrgRunStore: - MATERIALIZATION_IDENTITY_MISMATCH_PREFIX now stamps all three permanent identity failures in mark_materialization_succeeded and the certified-row revalidation in finish_starting; - STARTING_INPUT_CERTIFICATE_ERROR_PREFIX stamps the two permanent initial-input certificate failures in finish_starting; - is_materialization_identity_mismatch_error() / is_permanent_finish_starting_error() are the only classification points. Sweep of substring classifications introduced by this PR: - launch_org.rs materialization receipt classification -> typed helper; - launch/mod.rs both finish_starting sites (background launch + startup recovery) -> is_permanent_finish_starting_error(); - launch/mod.rs initial-input payload decode classification -> shared prefix constants + is_permanent_initial_input_payload_error(), used by both the constructor and the classifier. --- .../core/coordination/agent_org_runs/mod.rs | 6 ++- .../core/coordination/agent_org_runs/store.rs | 40 ++++++++++++++++--- .../src/core/session/launch/launch_org.rs | 9 ++++- .../agent-core/src/core/session/launch/mod.rs | 32 +++++++++------ 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs index 6ac6ff116f..ca8a59a972 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs @@ -30,7 +30,11 @@ pub use quiescence::{ pub use rollout::{ is_enabled as agent_org_redesign_enabled, require_enabled as require_agent_org_redesign, }; -pub use store::AgentOrgRunStore; +pub use store::{ + is_materialization_identity_mismatch_error, is_permanent_finish_starting_error, + AgentOrgRunStore, MATERIALIZATION_IDENTITY_MISMATCH_PREFIX, + STARTING_INPUT_CERTIFICATE_ERROR_PREFIX, +}; pub(crate) use worker::recovery_dispatch_recipient_is_available; pub use worker::{WorkerSessionInfo, WorkerSessionRuntime}; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index b410c81186..b13ab6ed13 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -31,6 +31,32 @@ use super::{ CreateStartingAgentOrgRunParams, COORDINATOR_MEMBER_ID, }; +/// Stable machine prefix for permanent Session-identity failures raised by +/// the materialization / finish-Starting certificate checks in this store. +/// Launch recovery classifies retryable-vs-permanent on these prefixes; never +/// match the human-readable remainder of the message. +pub const MATERIALIZATION_IDENTITY_MISMATCH_PREFIX: &str = "materialization_identity_mismatch:"; + +/// Stable machine prefix for permanent initial-input certificate failures +/// raised by [`AgentOrgRunStore::finish_starting`]. +pub const STARTING_INPUT_CERTIFICATE_ERROR_PREFIX: &str = "starting_input_certificate_invalid:"; + +/// True for permanent Session-identity failures from +/// [`AgentOrgRunStore::mark_materialization_succeeded`] / +/// [`AgentOrgRunStore::finish_starting`]. Retrying these can never succeed: +/// the durable identity certificate itself is wrong. +pub fn is_materialization_identity_mismatch_error(error: &str) -> bool { + error.starts_with(MATERIALIZATION_IDENTITY_MISMATCH_PREFIX) +} + +/// True for every permanent (non-retryable) failure class that +/// [`AgentOrgRunStore::finish_starting`] can return. Everything else from +/// that call is treated as transient and retried by the recovery owners. +pub fn is_permanent_finish_starting_error(error: &str) -> bool { + is_materialization_identity_mismatch_error(error) + || error.starts_with(STARTING_INPUT_CERTIFICATE_ERROR_PREFIX) +} + pub struct AgentOrgRunStore; pub(crate) struct AgentOrgRunDeleteOutcome { @@ -336,7 +362,7 @@ impl AgentOrgRunStore { }; if expected_session_id != session_id { return Err(format!( - "materialization session mismatch for {run_id}/{member_id}: expected {expected_session_id}, got {session_id}" + "{MATERIALIZATION_IDENTITY_MISMATCH_PREFIX} materialization session mismatch for {run_id}/{member_id}: expected {expected_session_id}, got {session_id}" )); } if status == "succeeded" { @@ -357,7 +383,7 @@ impl AgentOrgRunStore { persisted_identity else { return Err(format!( - "materialized Session {session_id} is missing for {run_id}/{member_id}" + "{MATERIALIZATION_IDENTITY_MISMATCH_PREFIX} materialized Session {session_id} is missing for {run_id}/{member_id}" )); }; let expected_parent = (member_id != COORDINATOR_MEMBER_ID).then_some(root_session_id); @@ -366,7 +392,7 @@ impl AgentOrgRunStore { || parent_session_id != expected_parent { return Err(format!( - "materialized Session identity mismatch for {run_id}/{member_id}" + "{MATERIALIZATION_IDENTITY_MISMATCH_PREFIX} materialized Session identity mismatch for {run_id}/{member_id}" )); } let changed = transaction @@ -473,7 +499,7 @@ impl AgentOrgRunStore { .map_err(|error| error.to_string())?; if invalid_materialized_identities != 0 { return Err(format!( - "materialization_identity_mismatch: {invalid_materialized_identities} certified Session identity row(s) are invalid for {run_id}" + "{MATERIALIZATION_IDENTITY_MISMATCH_PREFIX} {invalid_materialized_identities} certified Session identity row(s) are invalid for {run_id}" )); } let incomplete_materializations: i64 = transaction @@ -492,7 +518,9 @@ impl AgentOrgRunStore { let initial_input = load_initial_input_with_connection(&transaction, run_id)?; if has_initial_work { let input = initial_input.as_ref().ok_or_else(|| { - format!("initial input certificate missing for Starting run {run_id}") + format!( + "{STARTING_INPUT_CERTIFICATE_ERROR_PREFIX} initial input certificate missing for Starting run {run_id}" + ) })?; let message_exists: bool = transaction .query_row( @@ -534,7 +562,7 @@ impl AgentOrgRunStore { .map_err(|error| error.to_string())?; } else if initial_input.is_some() { return Err(format!( - "unexpected initial input certificate for no-work Starting run {run_id}" + "{STARTING_INPUT_CERTIFICATE_ERROR_PREFIX} unexpected initial input certificate for no-work Starting run {run_id}" )); } diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs index 50c7e4560e..d6ddf71b58 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs @@ -228,7 +228,14 @@ pub(super) async fn materialize_org_member_sessions( &persisted_session_id, ) .map_err(|error| { - if error.contains("identity mismatch") || error.contains("Session identity") { + // Typed classification: the store stamps permanent identity + // failures with a stable machine prefix. Matching prose (the old + // `.contains("identity mismatch")`) missed the store's + // "materialization session mismatch" wording and retried a + // permanently wrong identity forever. + if crate::coordination::agent_org_runs::is_materialization_identity_mismatch_error( + &error, + ) { AgentOrgMaterializationError::permanent("materialization_identity_mismatch", error) } else { AgentOrgMaterializationError::retryable(error) diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index 72d642932b..f26003102f 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -45,6 +45,20 @@ pub(crate) const MAX_AUTO_NAME_LEN: usize = 80; const AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION: u8 = 1; +/// Stable machine prefixes for permanent initial-input payload decode +/// failures. Classification sites match on these constants (shared with the +/// construction below) instead of free-hand substrings. +const INVALID_INITIAL_INPUT_PAYLOAD_PREFIX: &str = "invalid Starting initial input payload"; +const UNSUPPORTED_INITIAL_INPUT_PAYLOAD_PREFIX: &str = + "unsupported Starting initial input payload version"; + +/// True when the decode failure is permanent: the durable payload itself is +/// malformed or from an unknown version, so no retry can ever succeed. +fn is_permanent_initial_input_payload_error(error: &str) -> bool { + error.starts_with(INVALID_INITIAL_INPUT_PAYLOAD_PREFIX) + || error.starts_with(UNSUPPORTED_INITIAL_INPUT_PAYLOAD_PREFIX) +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct AgentOrgInitialInputPayload { @@ -60,13 +74,13 @@ fn decode_agent_org_initial_input_payload( let payload: AgentOrgInitialInputPayload = serde_json::from_str(&input.payload_json).map_err(|error| { format!( - "invalid Starting initial input payload for {}: {error}", + "{INVALID_INITIAL_INPUT_PAYLOAD_PREFIX} for {}: {error}", input.org_run_id ) })?; if payload.version != AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION { return Err(format!( - "unsupported Starting initial input payload version {} for {}", + "{UNSUPPORTED_INITIAL_INPUT_PAYLOAD_PREFIX} {} for {}", payload.version, input.org_run_id )); } @@ -936,10 +950,7 @@ pub(crate) async fn launch_rust_agent_run( } if let Err(error) = AgentOrgRunStore::finish_starting(run_id, starting_generation) { - if error.starts_with("materialization_identity_mismatch:") - || error.contains("initial input certificate missing") - || error.contains("unexpected initial input certificate") - { + if crate::coordination::agent_org_runs::is_permanent_finish_starting_error(&error) { handle_background_launch_failure( &session_id_for_background, Some(run_id), @@ -1340,9 +1351,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St if let Some(input) = initial_input { if let Err(error) = persist_starting_initial_input(state, root_session_id, &input).await { - if error.starts_with("invalid Starting initial input payload") - || error.starts_with("unsupported Starting initial input payload version") - { + if is_permanent_initial_input_payload_error(&error) { AgentOrgRunStore::fail_starting( &run.id, run.activation_generation, @@ -1358,10 +1367,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St } } if let Err(error) = AgentOrgRunStore::finish_starting(&run.id, run.activation_generation) { - if error.starts_with("materialization_identity_mismatch:") - || error.contains("initial input certificate missing") - || error.contains("unexpected initial input certificate") - { + if crate::coordination::agent_org_runs::is_permanent_finish_starting_error(&error) { AgentOrgRunStore::fail_starting( &run.id, run.activation_generation, From ed309078acd222476abb900692ad8127507cda49 Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:24:15 -0700 Subject: [PATCH 05/11] fix(agent-org): stop failing the coordinator session of a Running team MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_background_launch_failure discarded fail_starting's Ok(false) (returned when the run already left Starting) and then marked the coordinator session failed anyway — a Running team with a failed coordinator session row, and a log that claimed the run was marked failed when it was not. When fail_starting reports no transition, the handler now reloads the actual run status and logs error-level with that status, stating explicitly that the run status was left unchanged. For a Running run the coordinator session status is also left untouched (the watchdog and the initial-dispatch recovery own retries for a live team); the launch error is still broadcast and persisted to the transcript so the failure stays visible. Non-Running outcomes keep the previous behavior. --- .../src/core/session/launch/launch_helpers.rs | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs index b39871376d..f2d718684a 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs @@ -7,7 +7,9 @@ use std::collections::HashMap; use core_types::key_source::KeySource; -use crate::coordination::agent_org_runs::{AgentOrgRunStore, AgentOrgStartingFailure}; +use crate::coordination::agent_org_runs::{ + AgentOrgRunStatus, AgentOrgRunStore, AgentOrgStartingFailure, +}; use crate::definitions::orgs::{OrgMember, OrgMemberRuntimeConfig}; use crate::session::turn::streaming::{ broadcast_agent_error_structured, classify_streaming_error_message, StreamingError, @@ -29,6 +31,11 @@ pub(super) async fn handle_background_launch_failure( session_mark_warning: &str, ) { tracing::warn!("{}", message); + // A failure that arrives after the run already left Starting (typically: + // `finish_starting` committed Running and only the initial send failed) + // must not fail the coordinator session out from under a live team — the + // watchdog/initial-dispatch recovery owns retries for a Running run. + let mut fail_coordinator_session = true; if let Some(run_id) = agent_org_run_id { let failure_result = AgentOrgRunStore::load(run_id).and_then(|run| { let run = run.ok_or_else(|| format!("Agent Org run not found: {run_id}"))?; @@ -37,21 +44,46 @@ pub(super) async fn handle_background_launch_failure( run.activation_generation, &AgentOrgStartingFailure::new("starting_convergence_failed", message), ) - .map(|_| ()) }); - if let Err(mark_err) = failure_result { - tracing::warn!( - run_id = %run_id, - error = %mark_err, - "{}", - run_mark_warning - ); + match failure_result { + Ok(true) => {} + Ok(false) => { + let current_status = AgentOrgRunStore::load(run_id) + .ok() + .flatten() + .map(|run| run.status); + tracing::error!( + run_id = %run_id, + status = ?current_status.map(AgentOrgRunStatus::as_str), + failure = %message, + "[session_launch] launch failure arrived after the run left Starting; run status was left unchanged" + ); + if current_status == Some(AgentOrgRunStatus::Running) { + fail_coordinator_session = false; + } + } + Err(mark_err) => { + tracing::warn!( + run_id = %run_id, + error = %mark_err, + "{}", + run_mark_warning + ); + } } } release_work_item_execution_lock_if_present(project_slug, work_item_id, session_id, app_handle) .await; broadcast_launch_send_error(session_id, message); crate::lifecycle::persist_session_error_event(app_handle, session_id, message); + if !fail_coordinator_session { + tracing::error!( + session_id = %session_id, + failure = %message, + "[session_launch] leaving coordinator session status untouched: its Agent Org run is Running" + ); + return; + } if let Err(mark_err) = mark_session_failed(session_id.to_string()).await { tracing::warn!( session_id = %session_id, From e84c4ed6520a4ebd49afd0e14a112baa3e8981bc Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:24:49 -0700 Subject: [PATCH 06/11] fix(agent-org): watchdog repair door for crash-wedged Working teams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A crash mid-turn leaves an org turn intent 'running' forever: restart reconciliation deliberately retains running Agent Org intents, quiescence counts them as InFlightTurnIntents, and the watchdog mapped that blocker to {} — the team stayed Working with no repair path. Add a bounded repair: when inspection sees a Working run whose ONLY quiescence blockers are in-flight turn intents, it selects 'running' intent rows older than STALE_INTENT_REPAIR_GRACE_SECS (15 min, following the PENDING_MATERIALIZATION_GRACE_SECS pattern) into the plan. The executor terminalizes them (running -> failed, a legal state-machine transition) inside a writer transaction that revalidates run status, intent status, and staleness, logging error-level per repaired intent with the reason. Young intents are never touched, and queued canonical initial inputs keep their own recovery owner. The next quiescence pass idles the team. The terminalization is performed with direct SQL in agent_core (matching how quiescence and run deletion already address session_turn_intents from this crate); the session-persistence state machine is respected by construction via the status='running' guard. Tests: wedged intent + aged clock -> plan carries the repair -> repair marks it failed -> try_transition_working_to_idle succeeds; a young running intent is never selected. --- .../core/coordination/agent_org_watchdog.rs | 10 +- .../agent_org_watchdog/inspect.rs | 52 +++++ .../coordination/agent_org_watchdog/plan.rs | 15 ++ .../agent_org_watchdog/recover.rs | 82 ++++++++ .../coordination/agent_org_watchdog/tests.rs | 180 +++++++++++++++++- 5 files changed, 336 insertions(+), 3 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs index 94ba20f85e..84b5e94575 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs @@ -41,7 +41,9 @@ pub(crate) use budget::member_rewake_fingerprint; #[cfg(test)] pub use budget::test_only_mark_failed_rewake_attempt; pub use inspect::inspect_stalled_run; -pub use plan::{MemberContinuationAction, MemberTaskAssignmentAction, StallRecoveryPlan}; +pub use plan::{ + MemberContinuationAction, MemberTaskAssignmentAction, StallRecoveryPlan, StaleTurnIntentRepair, +}; pub use recover::{recover_stalled_run, spawn}; pub(crate) use reservation::{ commit_member_rewake_reservation, refund_member_rewake_reservation, @@ -75,6 +77,12 @@ const WATCHDOG_MAX_RUNS: usize = 100; const WATCHDOG_SCAN_BUDGET: Duration = Duration::from_millis(250); const RECOVERY_DELAYS_SECS: [i64; 3] = [60, 5 * 60, 15 * 60]; const PENDING_MATERIALIZATION_GRACE_SECS: i64 = 2 * 60; +/// A `running` turn intent whose scheduler died (process crash mid-turn) can +/// never terminalize itself; it blocks quiescence forever. When a Working run +/// is blocked *only* by such intents and they are older than this grace, the +/// watchdog marks them failed. Young intents are never touched — a live turn +/// refreshes its row at every status transition. +const STALE_INTENT_REPAIR_GRACE_SECS: i64 = 15 * 60; const MEMBER_REWAKE: &str = "member_rewake"; const COORDINATOR_NOTICE: &str = "coordinator_notice"; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs index d841e37bee..164705d697 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs @@ -760,6 +760,53 @@ fn coordinator_unread_recovery_with_connection( )) } +/// Select crash-orphaned `running` turn intents for repair. The door is +/// deliberately narrow: the run must be Working, every quiescence blocker must +/// be `InFlightTurnIntents` (any active session, open task, unread row, etc. +/// disqualifies the run), and only `running` rows older than +/// [`STALE_INTENT_REPAIR_GRACE_SECS`] are selected. Queued canonical initial +/// inputs keep their own recovery owner and are never touched here. +pub(super) fn stale_in_flight_intent_repairs_with_connection( + conn: &Connection, + run_id: &str, + assessment: &crate::coordination::agent_org_runs::AgentOrgQuiescenceAssessment, +) -> Result, String> { + let only_in_flight_blockers = !assessment.blockers.is_empty() + && assessment.blockers.iter().all(|blocker| { + matches!( + blocker, + AgentOrgQuiescenceBlocker::InFlightTurnIntents { .. } + ) + }); + if !only_in_flight_blockers { + return Ok(Vec::new()); + } + let stale_before = + (Utc::now() - ChronoDuration::seconds(STALE_INTENT_REPAIR_GRACE_SECS)).to_rfc3339(); + let mut stmt = conn + .prepare( + "SELECT session_id, turn_intent_id, updated_at + FROM session_turn_intents + WHERE org_run_id=?1 + AND status='running' + AND (datetime(updated_at) IS NULL + OR datetime(updated_at)<=datetime(?2)) + ORDER BY session_id ASC, turn_intent_id ASC", + ) + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map(params![run_id, stale_before], |row| { + Ok(StaleTurnIntentRepair { + session_id: row.get(0)?, + turn_intent_id: row.get(1)?, + updated_at: row.get(2)?, + }) + }) + .map_err(|err| err.to_string())?; + rows.map(|row| row.map_err(|err| err.to_string())) + .collect() +} + pub fn inspect_stalled_run(run_id: &str) -> Result { let mut conn = get_connection().map_err(|err| err.to_string())?; let tx = conn @@ -844,6 +891,7 @@ pub(super) fn inspect_stalled_run_with_connection( .flatten(), coordinator_repair_active: true, clear_coordinator_notice_budget: false, + stale_intent_repairs: Vec::new(), terminal_candidate: false, }); } @@ -964,6 +1012,7 @@ pub(super) fn inspect_stalled_run_with_connection( .flatten(), coordinator_repair_active, clear_coordinator_notice_budget, + stale_intent_repairs: Vec::new(), terminal_candidate: false, }); } @@ -1350,6 +1399,8 @@ pub(super) fn inspect_stalled_run_with_connection( let coordinator_repair_active = !needs_repair.is_empty(); let clear_coordinator_notice_budget = !coordinator_repair_active && coordinator_notice_budget_exists_with_connection(conn, run_id)?; + let stale_intent_repairs = + stale_in_flight_intent_repairs_with_connection(conn, run_id, &quiescence_assessment)?; Ok(StallRecoveryPlan { wake_member_ids, @@ -1367,6 +1418,7 @@ pub(super) fn inspect_stalled_run_with_connection( .flatten(), coordinator_repair_active, clear_coordinator_notice_budget, + stale_intent_repairs, terminal_candidate, }) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/plan.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/plan.rs index fd799a60b2..b7d1cfea75 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/plan.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/plan.rs @@ -52,6 +52,12 @@ pub struct StallRecoveryPlan { /// the analyzer sees no current repair *and* a budget row actually exists, /// avoiding one no-op writer transaction per healthy run per tick. pub clear_coordinator_notice_budget: bool, + /// Crash-orphaned `running` turn intents older than + /// [`STALE_INTENT_REPAIR_GRACE_SECS`] on a Working run whose only + /// quiescence blockers are those intents. The executor marks them failed + /// (with revalidation under the writer lock) so the next quiescence pass + /// can idle the team. + pub stale_intent_repairs: Vec, /// Every task resolved + every worker terminal: the run can be /// reconciled to a terminal status. pub terminal_candidate: bool, @@ -67,11 +73,20 @@ impl StallRecoveryPlan { && self.coordinator_repair_work_revision.is_none() && self.coordinator_repair_task_fingerprint.is_none() && self.coordinator_repair_inbox_fingerprint.is_none() + && self.stale_intent_repairs.is_empty() && !self.clear_coordinator_notice_budget && !self.terminal_candidate } } +/// One crash-orphaned `running` turn intent selected for terminalization. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StaleTurnIntentRepair { + pub session_id: String, + pub turn_intent_id: String, + pub updated_at: String, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemberContinuationAction { pub member_id: String, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index f53d467700..b1fc5f2662 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -101,6 +101,20 @@ fn execute_stall_recovery_plan( plan: StallRecoveryPlan, wake_hook: &dyn InboxWakeHook, ) -> Result { + // Terminalize crash-orphaned running intents first. The next quiescence + // pass (usually the next tick) can then idle the team; this tick's plan + // deliberately does not re-assess. + if !plan.stale_intent_repairs.is_empty() { + match repair_stale_in_flight_intents(run_id, &plan.stale_intent_repairs) { + Ok(_) => {} + Err(err) => tracing::warn!( + run_id = %run_id, + error = %err, + "[agent_org_watchdog] stale turn-intent repair failed; will retry next tick" + ), + } + } + // Reconcile first: when the run actually closes there is nothing // left to wake or repair. When reconciliation declines (e.g. the // coordinator root session is still open), fall through and deliver @@ -226,6 +240,74 @@ fn execute_stall_recovery_plan( Ok(plan) } +/// Mark analyzed crash-orphaned `running` intents failed. Every row is +/// revalidated under the writer lock: it must still be `running`, still owned +/// by this run, and still older than the repair grace — a turn that revived +/// (impossible today, defensive) or a fresh intent with a recycled id is left +/// untouched. Returns the number of intents terminalized. +pub(super) fn repair_stale_in_flight_intents( + run_id: &str, + repairs: &[StaleTurnIntentRepair], +) -> Result { + with_sessions_writer(|| -> Result { + let mut conn = get_connection().map_err(|err| err.to_string())?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|err| err.to_string())?; + let running: bool = tx + .query_row( + "SELECT EXISTS( + SELECT 1 FROM agent_org_runs WHERE id=?1 AND status='running' + )", + params![run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + if !running { + tx.commit().map_err(|err| err.to_string())?; + return Ok(0); + } + let stale_before = + (Utc::now() - ChronoDuration::seconds(STALE_INTENT_REPAIR_GRACE_SECS)).to_rfc3339(); + let now = Utc::now().to_rfc3339(); + let mut repaired = 0usize; + for repair in repairs { + // `running → failed` is a legal state-machine transition; the + // status guard makes this exactly that transition and nothing else. + let changed = tx + .execute( + "UPDATE session_turn_intents + SET status='failed', updated_at=?5 + WHERE session_id=?1 AND turn_intent_id=?2 AND org_run_id=?3 + AND status='running' + AND (datetime(updated_at) IS NULL + OR datetime(updated_at)<=datetime(?4))", + params![ + &repair.session_id, + &repair.turn_intent_id, + run_id, + &stale_before, + &now + ], + ) + .map_err(|err| err.to_string())?; + if changed == 1 { + repaired += 1; + tracing::error!( + run_id = %run_id, + session_id = %repair.session_id, + turn_intent_id = %repair.turn_intent_id, + intent_updated_at = %repair.updated_at, + grace_secs = STALE_INTENT_REPAIR_GRACE_SECS, + "[agent_org_watchdog] crash-orphaned running turn intent had no live scheduler owner and exceeded the repair grace; marked failed so team quiescence can settle" + ); + } + } + tx.commit().map_err(|err| err.to_string())?; + Ok(repaired) + }) +} + fn clear_coordinator_notice_budget_if_recovered(run_id: &str) -> Result<(), String> { with_sessions_writer(|| -> Result<(), String> { let mut conn = get_connection().map_err(|err| err.to_string())?; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs index b5a2b1f3b2..c3252cfa56 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs @@ -2,9 +2,12 @@ use super::budget::{ budget_disposition, coordinator_notice_allowed, rewake_budget_exhausted, BudgetDisposition, }; use super::inspect::is_wakeable_status; -use super::recover::recover_listed_runs; +use super::recover::{recover_listed_runs, repair_stale_in_flight_intents}; use super::*; -use crate::coordination::agent_org_runs::{AgentOrgRunEntryMode, AgentOrgRunRecord}; +use crate::coordination::agent_org_runs::{ + AgentOrgRunEntryMode, AgentOrgRunRecord, CreateAgentOrgRunParams, +}; +use crate::definitions::orgs::OrgDefinition; fn fake_run(id: &str) -> AgentOrgRunRecord { let now = Utc::now().to_rfc3339(); @@ -203,6 +206,179 @@ fn coordinator_notice_budget_backs_off_and_resets_on_new_reason() { assert!(coordinator_notice_allowed(&run_id, "task b stuck").expect("new reason")); } +fn ensure_watchdog_runtime_schemas() { + let conn = get_connection().expect("test sqlite connection"); + crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); + crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) + .expect("agent sessions schema"); + crate::core::session::persistence::init(&conn).expect("unified session schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org runtime schemas"); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS code_sessions ( + session_id TEXT PRIMARY KEY, + cli_agent_type TEXT NOT NULL, + status TEXT NOT NULL, + parent_session_id TEXT, + org_member_id TEXT, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS session_turn_intents ( + session_id TEXT NOT NULL, + turn_intent_id TEXT NOT NULL, + client_message_id TEXT, + org_run_id TEXT, + source TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (session_id, turn_intent_id) + ); + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL + );", + ) + .expect("runtime session schemas"); +} + +/// A Working run that is quiescent in every dimension except one `running` +/// turn intent whose scheduler no longer exists (crash mid-turn). +fn seed_wedged_working_run(intent_updated_at: &str) -> String { + ensure_watchdog_runtime_schemas(); + let root_session_id = format!("wedged-root-{}", uuid::Uuid::new_v4()); + crate::core::session::persistence::upsert_session( + &crate::core::session::persistence::UnifiedSessionRecord { + session_id: root_session_id.clone(), + name: "wedged root".to_string(), + status: SessionStatus::Idle.as_str().to_string(), + session_type: "agent".to_string(), + agent_definition_id: Some("agent-coord".to_string()), + created_at: Utc::now().to_rfc3339(), + updated_at: Utc::now().to_rfc3339(), + ..Default::default() + }, + ) + .expect("upsert quiescent coordinator session"); + let run = AgentOrgRunStore::create(CreateAgentOrgRunParams { + org_id: "wedged-org".to_string(), + coordinator_agent_id: "agent-coord".to_string(), + root_session_id: Some(root_session_id.clone()), + org_snapshot: OrgDefinition { + id: "wedged-org".to_string(), + name: "Wedged Org".to_string(), + role: "lead".to_string(), + agent_id: "agent-coord".to_string(), + description: None, + hierarchy_mode: Default::default(), + plan_approval_policy: Default::default(), + children: Vec::new(), + }, + entry_mode: AgentOrgRunEntryMode::StandaloneSession, + status: AgentOrgRunStatus::Running, + work_item_id: None, + project_slug: None, + routine_fire_id: None, + }) + .expect("create Working run"); + agent_org_tasks::AgentOrgTaskStore::create(agent_org_tasks::CreateTaskParams { + id: "wedged-task-done".to_string(), + org_run_id: run.id.clone(), + subject: "done".to_string(), + description: String::new(), + active_form: None, + owner: None, + status: TaskStatus::Completed, + blocks: Vec::new(), + blocked_by: Vec::new(), + metadata: None, + }) + .expect("create completed task"); + let revision = AgentOrgRunStore::stage_coordinator_work_revision(&run.id) + .expect("stage coordinator work revision") + .expect("running run has a work revision"); + AgentOrgRunStore::mark_coordinator_observed_work_revision(&run.id, revision) + .expect("mark coordinator observed revision"); + + let conn = get_connection().expect("test sqlite connection"); + conn.execute( + "INSERT INTO session_turn_intents ( + session_id, turn_intent_id, org_run_id, source, status, + created_at, updated_at + ) VALUES (?1, 'wedged-turn-intent', ?2, 'agent_org', 'running', ?3, ?3)", + params![&root_session_id, &run.id, intent_updated_at], + ) + .expect("seed crash-orphaned running intent"); + run.id +} + +#[test] +fn wedged_running_intent_older_than_grace_is_repaired_and_run_can_idle() { + let _sandbox = test_helpers::test_env::sandbox(); + let aged = + (Utc::now() - ChronoDuration::seconds(STALE_INTENT_REPAIR_GRACE_SECS + 300)).to_rfc3339(); + let run_id = seed_wedged_working_run(&aged); + + let assessment = + AgentOrgRunStore::assess_run_quiescence(&run_id).expect("assess wedged run"); + assert_eq!( + assessment.decision, + AgentOrgQuiescenceDecision::KeepWorking, + "the crash-orphaned intent must block quiescence before repair" + ); + assert_eq!(assessment.facts.in_flight_turn_intent_count, 1); + + let plan = inspect_stalled_run(&run_id).expect("inspect wedged run"); + assert_eq!(plan.stale_intent_repairs.len(), 1); + assert_eq!(plan.stale_intent_repairs[0].turn_intent_id, "wedged-turn-intent"); + assert!(!plan.terminal_candidate); + + let repaired = + repair_stale_in_flight_intents(&run_id, &plan.stale_intent_repairs).expect("repair"); + assert_eq!(repaired, 1); + let status: String = get_connection() + .expect("db") + .query_row( + "SELECT status FROM session_turn_intents WHERE turn_intent_id='wedged-turn-intent'", + [], + |row| row.get(0), + ) + .expect("load repaired intent"); + assert_eq!(status, "failed"); + + // The next quiescence pass can now settle the team. + let assessment = + AgentOrgRunStore::assess_run_quiescence(&run_id).expect("assess repaired run"); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::Quiescent); + let generation = assessment + .facts + .activation_generation + .expect("activation generation"); + let work_revision = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + .expect("work revision"); + assert!( + AgentOrgRunStore::try_transition_working_to_idle(&run_id, generation, work_revision) + .expect("idle transition"), + "the repaired team must be reconcilable to Idle" + ); +} + +#[test] +fn young_running_intent_is_never_auto_repaired() { + let _sandbox = test_helpers::test_env::sandbox(); + let run_id = seed_wedged_working_run(&Utc::now().to_rfc3339()); + + let plan = inspect_stalled_run(&run_id).expect("inspect young-intent run"); + assert!( + plan.stale_intent_repairs.is_empty(), + "an intent inside the grace window must never be terminalized" + ); + assert!(!plan.terminal_candidate); +} + #[test] fn rewake_budget_exhaustion_requires_all_attempts_and_an_expired_cooldown() { let _sandbox = test_helpers::test_env::sandbox(); From dfc0494a3cc46bae083f14a112c6d35695f71bbf Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:49:36 -0700 Subject: [PATCH 07/11] fix(agent-org): tick starting retries; isolate dispatch recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transient launch errors (workspace prep, materialization, initial-input persistence, finish_starting) deferred to the next app restart while the frontend polled the Starting team forever — nothing owned retries within a session. The watchdog tick now runs a bounded Starting retry pass after the running-scan: at most STARTING_RETRY_MAX_RUNS_PER_TICK (10) starting runs older than STARTING_RETRY_GRACE_SECS (2 min; younger runs are still owned by their in-process launch task) are re-driven through the same per-run startup recovery routine, sharing the tick's 250ms deadline. A team that reaches Running this way may still hold its queued canonical initial input, so the bounded initial-dispatch pass is re-driven under the same deadline. recover_agent_org_initial_dispatches also used ? inside its loop, so one bad team (payload decode failure, send error) starved every later team on every boot. Failures are now isolated per team (log + continue), and a permanently undecodable payload on a Running run takes a terminal disposition: the queued canonical intent is rejected (a legal queued->rejected transition through the turn-intent bridge), so the input stops matching the recoverable query forever instead of being retried every boot and every tick. This was chosen over adding a new initial-input status because it needs no DDL change and the recoverable query already keys on the intent status. Tests: select_aged_starting_runs bounds, grace filtering (young runs are never selected; corrupt updated_at escalates as aged). --- .../agent_org_watchdog/recover.rs | 26 +- .../src/core/session/launch/launch_tests.rs | 80 +++++- .../agent-core/src/core/session/launch/mod.rs | 261 ++++++++++++++---- 3 files changed, 305 insertions(+), 62 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index b1fc5f2662..769bd6574b 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -23,8 +23,13 @@ pub fn spawn(app_handle: AppHandle) { interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { interval.tick().await; + // One deadline for the whole tick: the Working scan and the + // Starting retry pass share the same bounded budget. + let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; let handle = app_handle.clone(); - match tokio::task::spawn_blocking(move || recover_all_stalled_runs(handle)).await { + match tokio::task::spawn_blocking(move || recover_all_stalled_runs(handle, deadline)) + .await + { Ok(Ok(())) => {} Ok(Err(err)) => { tracing::warn!(error = %err, "[agent_org_watchdog] watchdog scan failed") @@ -33,12 +38,27 @@ pub fn spawn(app_handle: AppHandle) { tracing::warn!(error = %err, "[agent_org_watchdog] watchdog task join failed") } } + // Bounded Starting retry owner: transient launch errors otherwise + // wait for the next app restart while the frontend polls the + // Starting team forever. + use tauri::Manager; + if let Some(state) = app_handle.try_state::() { + if let Err(err) = crate::core::session::launch::recover_aged_starting_runs_from_tick( + &state, deadline, + ) + .await + { + tracing::warn!( + error = %err, + "[agent_org_watchdog] Starting retry pass failed" + ); + } + } } }); } -fn recover_all_stalled_runs(app_handle: AppHandle) -> Result<(), String> { - let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; +fn recover_all_stalled_runs(app_handle: AppHandle, deadline: Instant) -> Result<(), String> { let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS)?; recover_listed_runs_until(app_handle, runs, deadline, recover_stalled_run) } diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs index 00c608615c..6cd3fcf322 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs @@ -3,7 +3,7 @@ use super::launch_helpers::{ member_runtime_key_source, member_runtime_model, member_runtime_native_harness_type, validate_launch_agent_definitions, }; -use crate::coordination::agent_org_runs::COORDINATOR_MEMBER_ID; +use crate::coordination::agent_org_runs::{AgentOrgRunRecord, COORDINATOR_MEMBER_ID}; use crate::definitions::builtin::SDE_AGENT_ID; use crate::definitions::orgs::{ HierarchyMode, OrgDefinition, OrgMember, OrgMemberLaunchOverride, OrgMemberRuntimeConfig, @@ -32,6 +32,84 @@ fn session_marker_writes_explicit_build_for_legacy_null_product_mode() { assert_eq!(marker["capabilities"], serde_json::json!(["work.read"])); } +fn starting_run_updated_at(id: &str, updated_at: &str) -> AgentOrgRunRecord { + AgentOrgRunRecord { + id: id.to_string(), + org_id: "starting-org".to_string(), + coordinator_agent_id: "agent-coord".to_string(), + root_session_id: Some(format!("root-{id}")), + org_snapshot_json: None, + entry_mode: crate::coordination::agent_org_runs::AgentOrgRunEntryMode::StandaloneSession, + status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Starting, + activation_generation: 1, + has_initial_work: true, + work_item_id: None, + project_slug: None, + routine_fire_id: None, + summary: None, + last_error: None, + failure_json: None, + last_activity_outcome: None, + created_at: updated_at.to_string(), + updated_at: updated_at.to_string(), + idled_at: None, + } +} + +#[test] +fn watchdog_tick_selects_only_aged_starting_runs_within_the_per_tick_bound() { + let now = chrono::Utc::now(); + let aged_at = + (now - chrono::Duration::seconds(super::STARTING_RETRY_GRACE_SECS + 60)).to_rfc3339(); + let young_at = now.to_rfc3339(); + + let mut runs = vec![starting_run_updated_at("starting-young", &young_at)]; + for index in 0..(super::STARTING_RETRY_MAX_RUNS_PER_TICK + 3) { + runs.push(starting_run_updated_at( + &format!("starting-aged-{index:02}"), + &aged_at, + )); + } + runs.push(starting_run_updated_at("starting-corrupt-clock", "not-a-timestamp")); + + let selected = super::select_aged_starting_runs( + runs, + now, + super::STARTING_RETRY_GRACE_SECS, + super::STARTING_RETRY_MAX_RUNS_PER_TICK, + ); + + assert_eq!( + selected.len(), + super::STARTING_RETRY_MAX_RUNS_PER_TICK, + "the tick retry pass is bounded" + ); + assert!( + selected.iter().all(|run| run.id != "starting-young"), + "a Starting run inside the grace window is still owned by its launch task" + ); + assert!( + selected.iter().all(|run| run.id.starts_with("starting-aged-")), + "aged runs feed the tick-driven recovery" + ); + + // The recovery driver is invoked for exactly the selected set: an aged + // run reaches the per-run routine, a young one never does. + let two = super::select_aged_starting_runs( + vec![ + starting_run_updated_at("starting-aged-only", &aged_at), + starting_run_updated_at("starting-young-only", &young_at), + ], + now, + super::STARTING_RETRY_GRACE_SECS, + super::STARTING_RETRY_MAX_RUNS_PER_TICK, + ); + assert_eq!( + two.iter().map(|run| run.id.as_str()).collect::>(), + vec!["starting-aged-only"] + ); +} + #[test] fn launch_validation_rejects_missing_agent_definition_before_session_create() { let _sandbox = test_helpers::test_env::sandbox(); diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index f26003102f..9eb790f05f 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -1254,6 +1254,21 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St .map_err(|error| error.to_string())??; for run in runs { + recover_one_agent_org_starting_run(state, &run).await?; + } + Ok(()) +} + +/// One Starting run's recovery step: rematerialize members, persist the +/// initial input, and finish (or fail) Starting. Shared by the startup +/// one-shot and the bounded watchdog-tick retry owner. +async fn recover_one_agent_org_starting_run( + state: &AgentAppState, + run: &crate::coordination::agent_org_runs::AgentOrgRunRecord, +) -> Result<(), String> { + { + // Body kept at its historical indentation to preserve blame; the + // former loop `continue`s are per-run `return Ok(())` outcomes. let Some(root_session_id) = run.root_session_id.as_deref() else { AgentOrgRunStore::fail_starting( &run.id, @@ -1263,7 +1278,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St "Starting run has no coordinator Session identity", ), )?; - continue; + return Ok(()); }; let root = persistence::get_session(root_session_id) .map_err(|error| error.to_string())? @@ -1279,7 +1294,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St message, ), )?; - continue; + return Ok(()); } }; let Some(snapshot_raw) = run.org_snapshot_json.as_deref() else { @@ -1291,7 +1306,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St format!("Starting run {} has no launch snapshot", run.id), ), )?; - continue; + return Ok(()); }; let snapshot: crate::definitions::orgs::OrgDefinition = match serde_json::from_str(snapshot_raw) { @@ -1305,7 +1320,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St error.to_string(), ), )?; - continue; + return Ok(()); } }; let workspace_path = root @@ -1335,17 +1350,17 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St run.activation_generation, error.failure(), )?; - continue; + return Ok(()); } tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] materialization retry deferred"); - continue; + return Ok(()); } let initial_input = match AgentOrgRunStore::initial_input(&run.id) { Ok(input) => input, Err(error) => { tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input lookup retry deferred"); - continue; + return Ok(()); } }; if let Some(input) = initial_input { @@ -1363,7 +1378,7 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St } else { tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input persistence retry deferred"); } - continue; + return Ok(()); } } if let Err(error) = AgentOrgRunStore::finish_starting(&run.id, run.activation_generation) { @@ -1384,65 +1399,195 @@ async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), St Ok(()) } -async fn recover_agent_org_initial_dispatches(state: &AgentAppState) -> Result<(), String> { - let inputs = tokio::task::spawn_blocking(|| { - AgentOrgRunStore::recoverable_initial_inputs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) +/// Starting runs younger than this are still owned by their in-process +/// launch task; only older ones are presumed abandoned by a transient error +/// and re-driven from the watchdog tick. +pub(crate) const STARTING_RETRY_GRACE_SECS: i64 = 2 * 60; +/// Bounded per-tick Starting retries so the watchdog tick stays cheap. +pub(crate) const STARTING_RETRY_MAX_RUNS_PER_TICK: usize = 10; + +/// Select the bounded set of Starting runs old enough for a tick-driven +/// retry. An unparseable `updated_at` counts as aged: a corrupt timestamp +/// must escalate into recovery rather than silently exempt the run forever. +pub(crate) fn select_aged_starting_runs( + runs: Vec, + now: chrono::DateTime, + grace_secs: i64, + limit: usize, +) -> Vec { + let aged_before = now - chrono::Duration::seconds(grace_secs); + runs.into_iter() + .filter(|run| { + match chrono::DateTime::parse_from_rfc3339(&run.updated_at) { + Ok(updated_at) => updated_at.with_timezone(&chrono::Utc) <= aged_before, + Err(error) => { + tracing::warn!( + run_id = %run.id, + updated_at = %run.updated_at, + error = %error, + "[agent-org-watchdog] unparseable Starting run updated_at; treating as aged" + ); + true + } + } + }) + .take(limit) + .collect() +} + +/// Bounded Starting retry owner driven from the watchdog tick. Transient +/// launch errors used to defer to the next app restart while the frontend +/// polled the Starting team forever; this pass re-drives the same per-run +/// startup recovery routine for at most +/// [`STARTING_RETRY_MAX_RUNS_PER_TICK`] aged runs per tick, sharing the +/// watchdog's scan deadline. A team that reaches Running here may still hold +/// its queued canonical initial input, so the bounded dispatch pass is +/// re-driven under the same deadline. +pub(crate) async fn recover_aged_starting_runs_from_tick( + state: &AgentAppState, + deadline: std::time::Instant, +) -> Result<(), String> { + let runs = tokio::task::spawn_blocking(|| { + AgentOrgRunStore::list_starting_runs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) }) .await .map_err(|error| error.to_string())??; + let aged = select_aged_starting_runs( + runs, + chrono::Utc::now(), + STARTING_RETRY_GRACE_SECS, + STARTING_RETRY_MAX_RUNS_PER_TICK, + ); + for run in aged { + if std::time::Instant::now() >= deadline { + return Ok(()); + } + if let Err(error) = recover_one_agent_org_starting_run(state, &run).await { + tracing::warn!( + run_id = %run.id, + error = %error, + "[agent-org-watchdog] Starting retry failed for one team; continuing" + ); + } + } + recover_agent_org_initial_dispatches_bounded( + state, + STARTING_RETRY_MAX_RUNS_PER_TICK, + Some(deadline), + ) + .await +} + +async fn recover_agent_org_initial_dispatches(state: &AgentAppState) -> Result<(), String> { + recover_agent_org_initial_dispatches_bounded(state, AGENT_ORG_STARTUP_RECOVERY_LIMIT, None) + .await +} + +/// Re-dispatch recoverable canonical initial inputs. Failures are isolated +/// per team: the previous `?`-in-loop meant one bad payload starved every +/// later team on every boot. `deadline` lets the bounded watchdog-tick caller +/// share its scan budget. +pub(crate) async fn recover_agent_org_initial_dispatches_bounded( + state: &AgentAppState, + limit: usize, + deadline: Option, +) -> Result<(), String> { + let inputs = + tokio::task::spawn_blocking(move || AgentOrgRunStore::recoverable_initial_inputs(limit)) + .await + .map_err(|error| error.to_string())??; for input in inputs { - let Some(run) = AgentOrgRunStore::load(&input.org_run_id)? else { - continue; - }; - let Some(root_session_id) = run.root_session_id.as_deref() else { - continue; - }; - let Some(root) = - persistence::get_session(root_session_id).map_err(|error| error.to_string())? - else { - continue; - }; - persistence::update_status(root_session_id, crate::session::SessionStatus::Idle) - .map_err(|error| error.to_string())?; - let workspace_path = root - .worktree_path - .clone() - .or(root.workspace_path.clone()) - .unwrap_or_default(); - let native_harness_type = root - .native_harness_type - .as_deref() - .map(|raw| { - core_types::providers::NativeHarnessType::parse(raw) - .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) - }) - .transpose()?; - let payload = decode_agent_org_initial_input_payload(&input)?; - send_initial_turn( - state, - root_session_id, - input.content.clone(), - root.model, - root.account_id, - workspace_path, - native_harness_type, - root.agent_exec_mode, - payload.images, - payload.ide_context, - Some(run.coordinator_agent_id), - payload.sub_agent_ids, - Some(run.id.clone()), - Some(input.message_id.clone()), - Some(input.turn_intent_id.clone()), - crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, - ) - .await?; - AgentOrgRunStore::mark_initial_input_dispatched(&run.id, &input.turn_intent_id)?; + if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) { + break; + } + if let Err(error) = recover_one_agent_org_initial_dispatch(state, &input).await { + tracing::warn!( + run_id = %input.org_run_id, + turn_intent_id = %input.turn_intent_id, + error = %error, + "[agent-org-startup] initial dispatch recovery failed for one team; continuing" + ); + } } Ok(()) } +async fn recover_one_agent_org_initial_dispatch( + state: &AgentAppState, + input: &crate::coordination::agent_org_runs::AgentOrgInitialInput, +) -> Result<(), String> { + let Some(run) = AgentOrgRunStore::load(&input.org_run_id)? else { + return Ok(()); + }; + let Some(root_session_id) = run.root_session_id.as_deref() else { + return Ok(()); + }; + let Some(root) = persistence::get_session(root_session_id).map_err(|error| error.to_string())? + else { + return Ok(()); + }; + // An undecodable durable payload on a Running run can never dispatch; + // retrying it every boot (and every watchdog pass) is pure noise. Reject + // the queued canonical intent — a legal queued→rejected transition — so + // the input stops matching the recoverable query forever. + let payload = match decode_agent_org_initial_input_payload(input) { + Ok(payload) => payload, + Err(error) if is_permanent_initial_input_payload_error(&error) => { + tracing::error!( + run_id = %run.id, + root_session_id = %root_session_id, + turn_intent_id = %input.turn_intent_id, + error = %error, + "[agent-org-startup] initial input payload is permanently undecodable; rejecting the canonical intent so recovery stops retrying it" + ); + crate::foundation::session_bridge::update_turn_intent_status( + root_session_id, + &input.turn_intent_id, + crate::foundation::session_bridge::TurnIntentBridgeStatus::Rejected, + ); + return Ok(()); + } + Err(error) => return Err(error), + }; + persistence::update_status(root_session_id, crate::session::SessionStatus::Idle) + .map_err(|error| error.to_string())?; + let workspace_path = root + .worktree_path + .clone() + .or(root.workspace_path.clone()) + .unwrap_or_default(); + let native_harness_type = root + .native_harness_type + .as_deref() + .map(|raw| { + core_types::providers::NativeHarnessType::parse(raw) + .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) + }) + .transpose()?; + send_initial_turn( + state, + root_session_id, + input.content.clone(), + root.model, + root.account_id, + workspace_path, + native_harness_type, + root.agent_exec_mode, + payload.images, + payload.ide_context, + Some(run.coordinator_agent_id.clone()), + payload.sub_agent_ids, + Some(run.id.clone()), + Some(input.message_id.clone()), + Some(input.turn_intent_id.clone()), + crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, + ) + .await?; + AgentOrgRunStore::mark_initial_input_dispatched(&run.id, &input.turn_intent_id)?; + Ok(()) +} + async fn persist_starting_initial_input( state: &AgentAppState, session_id: &str, From 9c8f4de887861bea45c7ff8baa36732489755c1b Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:53:43 -0700 Subject: [PATCH 08/11] fix(agent-org): rotate the watchdog scan so no Working team starves list_runs_by_status orders 'updated_at ASC LIMIT 100'; with more than 100 Working teams the same oldest batch was selected every tick and teams 101+ were never inspected (a stalled team's run row is not written by inspection, so its updated_at never moves it into the window). Add an in-process keyset cursor (last visited updated_at/id): each tick continues strictly after the previous batch via list_running_runs_after(), and wraps around to the front (deduplicating within the batch) when the tail is reached, so every Working team is visited across ticks regardless of population size. Test: a population larger than the batch limit is fully visited across ticks with a small limit, and successive ticks do not restart at the identical oldest batch. --- .../core/coordination/agent_org_runs/store.rs | 61 +++++++++++++++++++ .../agent_org_watchdog/recover.rs | 50 ++++++++++++++- .../coordination/agent_org_watchdog/tests.rs | 56 ++++++++++++++++- 3 files changed, 165 insertions(+), 2 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index b13ab6ed13..fd9bca9f9d 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -1074,6 +1074,67 @@ impl AgentOrgRunStore { Self::list_runs_by_status(AgentOrgRunStatus::Running, limit) } + /// Keyset continuation of [`Self::list_running_runs`]: rows strictly after + /// `(updated_at, id)` in the same `updated_at ASC, id ASC` order. `None` + /// starts from the front. The watchdog's rotation cursor uses this so a + /// population larger than one batch is still fully visited across ticks. + pub fn list_running_runs_after( + cursor: Option<(&str, &str)>, + limit: usize, + ) -> Result, String> { + let Some((cursor_updated_at, cursor_id)) = cursor else { + return Self::list_running_runs(limit); + }; + if limit == 0 { + return Ok(Vec::new()); + } + let bounded_limit = i64::try_from(limit) + .map_err(|_| format!("Agent Org run list limit is too large: {limit}"))?; + let conn = get_connection().map_err(|err| err.to_string())?; + let mut stmt = conn + .prepare( + "SELECT id, + org_id, + coordinator_agent_id, + root_session_id, + org_snapshot_json, + entry_mode, + status, + activation_generation, + has_initial_work, + work_item_id, + project_slug, + routine_fire_id, + summary, + last_error, + failure_json, + last_activity_outcome, + created_at, + updated_at, + idled_at + FROM agent_org_runs + WHERE root_session_id IS NOT NULL + AND status = ?1 + AND (updated_at > ?2 OR (updated_at = ?2 AND id > ?3)) + ORDER BY updated_at ASC, id ASC + LIMIT ?4", + ) + .map_err(|err| err.to_string())?; + let rows = stmt + .query_map( + params![ + AgentOrgRunStatus::Running.as_str(), + cursor_updated_at, + cursor_id, + bounded_limit + ], + row_to_run, + ) + .map_err(|err| err.to_string())?; + rows.collect::, _>>() + .map_err(|err| err.to_string()) + } + fn list_runs_by_status( status: AgentOrgRunStatus, limit: usize, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index 769bd6574b..ae3e18629d 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -58,11 +58,59 @@ pub fn spawn(app_handle: AppHandle) { }); } +/// In-process rotation cursor over the Working scan (last visited +/// `(updated_at, id)`). `list_runs_by_status` orders `updated_at ASC LIMIT N`, +/// so without a cursor teams 101+ would be starved forever whenever more than +/// [`WATCHDOG_MAX_RUNS`] teams are Working at once. +static RUNNING_SCAN_CURSOR: std::sync::Mutex> = + std::sync::Mutex::new(None); + fn recover_all_stalled_runs(app_handle: AppHandle, deadline: Instant) -> Result<(), String> { - let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS)?; + let runs = { + let mut cursor = RUNNING_SCAN_CURSOR + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + next_running_scan_batch(&mut cursor, WATCHDOG_MAX_RUNS)? + }; recover_listed_runs_until(app_handle, runs, deadline, recover_stalled_run) } +/// Produce the next rotated batch of Working runs and advance the cursor. +/// Successive ticks continue past the previous batch; when the tail of the +/// keyset is reached the scan wraps around to the front (deduplicating rows +/// already in this batch), so every Working team is visited across ticks +/// regardless of population size. +pub(super) fn next_running_scan_batch( + cursor: &mut Option<(String, String)>, + limit: usize, +) -> Result, String> { + let mut runs = AgentOrgRunStore::list_running_runs_after( + cursor + .as_ref() + .map(|(updated_at, id)| (updated_at.as_str(), id.as_str())), + limit, + )?; + if runs.len() < limit { + let seen = runs + .iter() + .map(|run| run.id.clone()) + .collect::>(); + for run in AgentOrgRunStore::list_running_runs_after(None, limit)? { + if runs.len() >= limit { + break; + } + if seen.contains(&run.id) { + continue; + } + runs.push(run); + } + } + *cursor = runs + .last() + .map(|run| (run.updated_at.clone(), run.id.clone())); + Ok(runs) +} + #[cfg(test)] pub(super) fn recover_listed_runs( handle: H, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs index c3252cfa56..553886c493 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs @@ -2,7 +2,9 @@ use super::budget::{ budget_disposition, coordinator_notice_allowed, rewake_budget_exhausted, BudgetDisposition, }; use super::inspect::is_wakeable_status; -use super::recover::{recover_listed_runs, repair_stale_in_flight_intents}; +use super::recover::{ + next_running_scan_batch, recover_listed_runs, repair_stale_in_flight_intents, +}; use super::*; use crate::coordination::agent_org_runs::{ AgentOrgRunEntryMode, AgentOrgRunRecord, CreateAgentOrgRunParams, @@ -194,6 +196,58 @@ fn running_query_is_limited_and_never_visits_quiet_states() { assert_eq!(runs.last().map(|run| run.id.as_str()), Some("running-099")); } +#[test] +fn rotation_cursor_visits_every_working_run_across_ticks() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + const POPULATION: usize = 5; + const TICK_LIMIT: usize = 2; + for index in 0..POPULATION { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'rotation-org', 'coordinator', ?2, 'standalone_session', + 'running', ?3, ?3)", + params![ + format!("rotation-{index:02}"), + format!("root-rotation-{index:02}"), + format!("2026-08-01T00:00:0{index}Z"), + ], + ) + .expect("seed Working run"); + } + + let mut cursor: Option<(String, String)> = None; + let mut visited = HashSet::new(); + let mut first_batch_ids = Vec::new(); + for tick in 0..3 { + let batch = next_running_scan_batch(&mut cursor, TICK_LIMIT).expect("rotated batch"); + assert!(batch.len() <= TICK_LIMIT); + if tick == 0 { + first_batch_ids = batch.iter().map(|run| run.id.clone()).collect(); + } + for run in batch { + visited.insert(run.id); + } + } + assert_eq!( + visited.len(), + POPULATION, + "a population larger than one batch must be fully visited across ticks" + ); + + // The wrap-around continues rotating instead of resetting to the same + // oldest batch every tick (the starvation being fixed). + let batch = next_running_scan_batch(&mut cursor, TICK_LIMIT).expect("post-wrap batch"); + assert!( + batch.iter().any(|run| !first_batch_ids.contains(&run.id)) + || first_batch_ids.len() < TICK_LIMIT, + "successive ticks must not restart at the identical oldest batch" + ); +} + #[test] fn coordinator_notice_budget_backs_off_and_resets_on_new_reason() { let _sandbox = test_helpers::test_env::sandbox(); From 2ff5f2ad741143ec4dec8d52a2d42e260b3462bd Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:59:38 -0700 Subject: [PATCH 09/11] fix(agent-org): startup prune of leaked reservations and dead budgets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A crash between reserve and commit/refund left a reservation_token behind forever; quiescence fails closed on ActiveRecoveryReservations, so the team could never idle again. Reservations are process-scoped claims by definition, so the startup one-shot now clears ALL reservation tokens unconditionally, then prunes agent_org_recovery_attempts rows whose run is no longer starting/running (missing/paused/idle/failed/archived) in LIMIT-bounded writer batches (256/batch, hard cap) so a large backlog never holds the writer lock for one long scan. The same one-shot also re-wires AgentOrgPlanApprovalStore::cancel_pending_for_terminal_or_missing_runs, which had lost its only production caller in this PR — pending approvals for dead teams are cancelled again at boot. Test: leaked tokens on running/idle/missing-run rows are all cleared; only the running team's budget row survives the prune. --- .../core/coordination/agent_org_watchdog.rs | 4 +- .../coordination/agent_org_watchdog/budget.rs | 65 +++++++++++++++++++ .../coordination/agent_org_watchdog/tests.rs | 60 +++++++++++++++++ .../agent-core/src/core/session/launch/mod.rs | 41 ++++++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs index 84b5e94575..e3dd48e79b 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs @@ -35,8 +35,10 @@ mod reservation; #[cfg(test)] mod tests; -pub use budget::clear_rewake_budget; pub use budget::init_schema; +pub use budget::{ + clear_rewake_budget, startup_prune_recovery_state, AgentOrgRecoveryStartupPruneReport, +}; pub(crate) use budget::member_rewake_fingerprint; #[cfg(test)] pub use budget::test_only_mark_failed_rewake_attempt; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs index b57782a6a4..68b9ef312f 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs @@ -166,6 +166,71 @@ pub(super) fn record_attempt_with_connection( Ok(()) } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct AgentOrgRecoveryStartupPruneReport { + pub reservations_cleared: usize, + pub attempts_pruned: usize, +} + +impl AgentOrgRecoveryStartupPruneReport { + pub fn changed(&self) -> bool { + self.reservations_cleared > 0 || self.attempts_pruned > 0 + } +} + +/// Startup one-shot repair of durable recovery bookkeeping. +/// +/// 1. **Reservation tokens are process-scoped claims.** No reservation can +/// survive the process that minted it, and quiescence fails closed on +/// `ActiveRecoveryReservations`, so a crash between reserve and +/// commit/refund would otherwise block Idle forever. Clear every token. +/// 2. **Budget rows for non-running runs are dead weight.** Their +/// fingerprints reset on any new durable input anyway; prune them in +/// LIMIT-bounded writer batches so one boot never holds the writer lock +/// for an unbounded scan. +pub fn startup_prune_recovery_state() -> Result { + const PRUNE_BATCH: usize = 256; + const MAX_PRUNE_BATCHES: usize = 64; + + let mut report = AgentOrgRecoveryStartupPruneReport::default(); + report.reservations_cleared = with_sessions_writer(|| -> Result { + let conn = get_connection().map_err(|err| err.to_string())?; + conn.execute( + "UPDATE agent_org_recovery_attempts + SET reservation_token=NULL + WHERE reservation_token IS NOT NULL", + [], + ) + .map_err(|err| err.to_string()) + })?; + + for _ in 0..MAX_PRUNE_BATCHES { + let deleted = with_sessions_writer(|| -> Result { + let conn = get_connection().map_err(|err| err.to_string())?; + conn.execute( + "DELETE FROM agent_org_recovery_attempts + WHERE rowid IN ( + SELECT attempt.rowid + FROM agent_org_recovery_attempts attempt + WHERE NOT EXISTS ( + SELECT 1 FROM agent_org_runs run + WHERE run.id=attempt.org_run_id + AND run.status IN ('starting', 'running') + ) + LIMIT ?1 + )", + params![PRUNE_BATCH as i64], + ) + .map_err(|err| err.to_string()) + })?; + report.attempts_pruned += deleted; + if deleted < PRUNE_BATCH { + break; + } + } + Ok(report) +} + pub fn clear_rewake_budget(run_id: &str, member_id: &str) -> Result<(), String> { with_sessions_writer(|| { let conn = get_connection().map_err(|err| err.to_string())?; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs index 553886c493..13b8290345 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs @@ -196,6 +196,66 @@ fn running_query_is_limited_and_never_visits_quiet_states() { assert_eq!(runs.last().map(|run| run.id.as_str()), Some("running-099")); } +#[test] +fn startup_prune_clears_all_reservations_and_non_running_budget_rows() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + let now = Utc::now().to_rfc3339(); + for (run_id, status) in [("prune-running", "running"), ("prune-idle", "idle")] { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'prune-org', 'coordinator', ?2, 'standalone_session', ?3, ?4, ?4)", + params![run_id, format!("root-{run_id}"), status, &now], + ) + .expect("seed run"); + } + for (run_id, token) in [ + ("prune-running", Some("leaked-token-running")), + ("prune-idle", Some("leaked-token-idle")), + ("prune-missing-run", None::<&str>), + ] { + conn.execute( + "INSERT INTO agent_org_recovery_attempts + (org_run_id, action_kind, target_key, reason_fingerprint, attempts, + next_allowed_at, updated_at, reservation_token) + VALUES (?1, ?2, 'member-x', 'fp', 1, ?3, ?3, ?4)", + params![run_id, MEMBER_REWAKE, &now, token], + ) + .expect("seed recovery attempt"); + } + + let report = startup_prune_recovery_state().expect("startup prune"); + assert_eq!(report.reservations_cleared, 2); + assert_eq!(report.attempts_pruned, 2); + + let leaked_tokens: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_recovery_attempts + WHERE reservation_token IS NOT NULL", + [], + |row| row.get(0), + ) + .expect("count tokens"); + assert_eq!(leaked_tokens, 0, "no reservation survives its process"); + let remaining: Vec = { + let mut stmt = conn + .prepare("SELECT org_run_id FROM agent_org_recovery_attempts ORDER BY org_run_id") + .expect("prepare"); + let rows = stmt + .query_map([], |row| row.get::<_, String>(0)) + .expect("query"); + rows.collect::, _>>().expect("collect") + }; + assert_eq!( + remaining, + vec!["prune-running".to_string()], + "budget rows survive only for still-running teams" + ); +} + #[test] fn rotation_cursor_visits_every_working_run_across_ticks() { let _sandbox = test_helpers::test_env::sandbox(); diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index 9eb790f05f..f81ed8fc2c 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -1237,6 +1237,47 @@ pub fn spawn_agent_org_startup_recovery(state: AgentAppState) { return; } tauri::async_runtime::spawn(async move { + // Durable recovery bookkeeping first: reservation tokens cannot + // survive their process (quiescence fails closed on them), and budget + // rows for non-running teams are pruned in bounded batches. + match tokio::task::spawn_blocking( + crate::coordination::agent_org_watchdog::startup_prune_recovery_state, + ) + .await + { + Ok(Ok(report)) if report.changed() => tracing::info!( + reservations_cleared = report.reservations_cleared, + attempts_pruned = report.attempts_pruned, + "[agent-org-startup] pruned durable recovery bookkeeping" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::warn!(error = %error, "[agent-org-startup] recovery bookkeeping prune failed") + } + Err(error) => { + tracing::warn!(error = %error, "[agent-org-startup] recovery bookkeeping prune worker failed") + } + } + // Pending plan approvals whose team is gone or terminal are cancelled + // in the same one-shot; a dead approval would otherwise wedge the UI + // approval surfaces forever. + match tokio::task::spawn_blocking( + crate::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore::cancel_pending_for_terminal_or_missing_runs, + ) + .await + { + Ok(Ok(cancelled)) if cancelled > 0 => tracing::info!( + cancelled, + "[agent-org-startup] cancelled pending plan approvals for terminal or missing teams" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::warn!(error = %error, "[agent-org-startup] orphaned plan-approval cancellation failed") + } + Err(error) => { + tracing::warn!(error = %error, "[agent-org-startup] orphaned plan-approval worker failed") + } + } if let Err(error) = recover_agent_org_starting_runs(&state).await { tracing::warn!(error = %error, "[agent-org-startup] Starting recovery failed"); } From 6b710f58d94ec96e7a8aa79e9ab2ddff80c01161 Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:08:43 -0700 Subject: [PATCH 10/11] refactor(agent-org): hoist assess->CAS idle; fence test-only create; DDL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hygiene batch from the PR 781 audit: - The copy-pasted assess-quiescence -> extract generation/work_revision -> try_transition_working_to_idle block (4 sites: scheduler post-intent reconcile, lifecycle turn finalization, plan-approval follow-up, watchdog terminal candidate) is hoisted into AgentOrgRunStore::try_reconcile_to_idle() so the certificate protocol cannot drift per caller. - AgentOrgRunStore::create is production-dead (launch goes exclusively through create_starting) but is needed by unit tests and the #![cfg(debug_assertions)] /test endpoints, which seed arbitrary-status runs create_starting cannot express (no coordinator identity, non- Starting statuses, NULL root). Gate it #[cfg(any(test, debug_assertions))] — present exactly where its consumers exist, absent from release binaries. - Drop 'cancelled' from the last_activity_outcome CHECK: no code path reads or writes that value ('failed' via fail_starting and 'completed' via the idle CAS are the only writers). CREATE TABLE IF NOT EXISTS leaves existing databases untouched; PR 799 above regenerates its DDL manifest from the binary. The DDL string is edited minimally, no reformatting. --- .../core/coordination/agent_org_runs/mod.rs | 2 +- .../core/coordination/agent_org_runs/store.rs | 26 +++++++++++++++++++ .../agent_org_watchdog/recover.rs | 17 ++---------- .../agent-core/src/core/session/scheduler.rs | 16 +----------- src-tauri/crates/agent-core/src/lifecycle.rs | 15 +---------- .../session/org_tasks/plan_approval.rs | 18 +------------ 6 files changed, 32 insertions(+), 62 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs index ca8a59a972..3e37e7d01d 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs @@ -484,7 +484,7 @@ pub fn init_schema(conn: &Connection) -> SqliteResult<()> { last_error TEXT, failure_json TEXT, last_activity_outcome TEXT CHECK(last_activity_outcome IN ( - 'completed', 'failed', 'cancelled' + 'completed', 'failed' )), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index fd9bca9f9d..bab60be2db 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -121,6 +121,12 @@ impl AgentOrgRunStore { .map_err(|err| err.to_string()) } + /// Test-fixture constructor: seeds a run row in an arbitrary status + /// without the Starting construction envelope. Production launch goes + /// exclusively through [`Self::create_starting`]; unit tests and the + /// `#![cfg(debug_assertions)]` /test endpoints need arbitrary-status + /// seeding, so this is compiled only for those builds. + #[cfg(any(test, debug_assertions))] pub fn create(params: CreateAgentOrgRunParams) -> Result { let entry_mode = validate_entry_mode(params.entry_mode.as_str())?; let status = validate_status(params.status.as_str())?; @@ -858,6 +864,26 @@ impl AgentOrgRunStore { Ok(assessment) } + /// Assess quiescence and, when the certificate allows, present its exact + /// generation and work-revision facts to the atomic Working→Idle CAS. + /// Shared by every post-turn / lifecycle / watchdog reconcile site so + /// they cannot drift on the certificate protocol. + pub fn try_reconcile_to_idle(run_id: &str) -> Result { + let assessment = Self::assess_run_quiescence(run_id)?; + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + Self::try_transition_working_to_idle(run_id, generation, work_revision) + } + /// Atomically commit the only automatic lifecycle transition owned by /// PR 1. Both snapshot certificates are required so a stale finalizer or /// watchdog pass cannot idle a newer activation or newer work graph. diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index ae3e18629d..723aabedee 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -187,21 +187,8 @@ fn execute_stall_recovery_plan( // left to wake or repair. When reconciliation declines (e.g. the // coordinator root session is still open), fall through and deliver // the wakes so pending inbox rows still reach their recipients. - if plan.terminal_candidate { - let assessment = AgentOrgRunStore::assess_run_quiescence(run_id)?; - if let (Some(generation), Some(work_revision)) = ( - assessment.facts.activation_generation, - assessment - .facts - .progress - .as_ref() - .map(|progress| progress.work_revision), - ) { - if AgentOrgRunStore::try_transition_working_to_idle(run_id, generation, work_revision)? - { - return Ok(plan); - } - } + if plan.terminal_candidate && AgentOrgRunStore::try_reconcile_to_idle(run_id)? { + return Ok(plan); } // Analyzer output is advisory. Every derived inbox row is revalidated diff --git a/src-tauri/crates/agent-core/src/core/session/scheduler.rs b/src-tauri/crates/agent-core/src/core/session/scheduler.rs index 75a4ab51f6..eb23d5fa1f 100644 --- a/src-tauri/crates/agent-core/src/core/session/scheduler.rs +++ b/src-tauri/crates/agent-core/src/core/session/scheduler.rs @@ -500,22 +500,8 @@ impl WorkerTask { if let Some(run_id) = org_run_id { let reconcile_run_id = run_id.clone(); match tokio::task::spawn_blocking(move || { - let assessment = crate::coordination::agent_org_runs::AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; - let Some(generation) = assessment.facts.activation_generation else { - return Ok(false); - }; - let Some(work_revision) = assessment - .facts - .progress - .as_ref() - .map(|progress| progress.work_revision) - else { - return Ok(false); - }; - crate::coordination::agent_org_runs::AgentOrgRunStore::try_transition_working_to_idle( + crate::coordination::agent_org_runs::AgentOrgRunStore::try_reconcile_to_idle( &reconcile_run_id, - generation, - work_revision, ) }) .await diff --git a/src-tauri/crates/agent-core/src/lifecycle.rs b/src-tauri/crates/agent-core/src/lifecycle.rs index a9430a96c4..7474f4db08 100644 --- a/src-tauri/crates/agent-core/src/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/lifecycle.rs @@ -476,20 +476,7 @@ pub fn finalize_agent_org_member_turn( // The store re-checks tasks, inbox, interventions and queued turn // intents in one IMMEDIATE transaction before committing quiescence. if let Some(run_id) = reconcile_run_id { - let transition = AgentOrgRunStore::assess_run_quiescence(&run_id).and_then(|assessment| { - let Some(generation) = assessment.facts.activation_generation else { - return Ok(false); - }; - let Some(work_revision) = assessment - .facts - .progress - .as_ref() - .map(|progress| progress.work_revision) - else { - return Ok(false); - }; - AgentOrgRunStore::try_transition_working_to_idle(&run_id, generation, work_revision) - }); + let transition = AgentOrgRunStore::try_reconcile_to_idle(&run_id); match transition { Ok(true) => { tracing::info!(run_id = %run_id, "[lifecycle] idled quiescent Agent Org run"); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs index adf4e2cee0..510730a06a 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs @@ -171,23 +171,7 @@ pub async fn agent_org_plan_approval_respond( let reconcile_run_id = run_id.clone(); tokio::spawn(async move { match tokio::task::spawn_blocking(move || { - let assessment = AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; - let Some(generation) = assessment.facts.activation_generation else { - return Ok(false); - }; - let Some(work_revision) = assessment - .facts - .progress - .as_ref() - .map(|progress| progress.work_revision) - else { - return Ok(false); - }; - AgentOrgRunStore::try_transition_working_to_idle( - &reconcile_run_id, - generation, - work_revision, - ) + AgentOrgRunStore::try_reconcile_to_idle(&reconcile_run_id) }) .await { From 46ce2f70cf29e0f3f6de013453676e28c0ae06a0 Mon Sep 17 00:00:00 2001 From: Neonforge <48338160+Neonforge98@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:12:06 -0700 Subject: [PATCH 11/11] fix(agent-org): send-fence identity probe no longer depends on needs_db resolve_session_identity computed has_persisted_agent_org_identity only when the unrelated needs_db condition forced a DB read; a fully cache-hit send (model/account/workspace/harness all in the runtime cache) skipped the probe and therefore the Agent Org lifecycle fence entirely. The probe is now unconditional: the loaded DB record answers when present; a runtime already carrying its org context answers directly; and ordinary cache-hit sessions pay one targeted indexed point read of agent_sessions.org_member_id (new session_persistence::get_org_member_id) instead of silently skipping the fence. DB errors propagate like the existing identity resolution errors. --- .../src/core/session/persistence/crud/mod.rs | 3 +- .../src/core/session/persistence/crud/ops.rs | 16 +++++++++- .../src/core/session/persistence/mod.rs | 2 +- .../src/state/commands/session/identity.rs | 32 ++++++++++++++++--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/crud/mod.rs b/src-tauri/crates/agent-core/src/core/session/persistence/crud/mod.rs index 9019cbfda6..a4ff383960 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/crud/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/crud/mod.rs @@ -24,7 +24,8 @@ mod workspace; pub use migration::ensure_unified_schema; pub use ops::{ backfill_agent_definition_id, delete_session, finalize_terminal_turn_status, - get_child_sessions, get_parent_session, get_session, link_bootstrap_work_item, list_sessions, + get_child_sessions, get_org_member_id, get_parent_session, get_session, + link_bootstrap_work_item, list_sessions, mark_stale_running_sessions_abandoned, reconcile_sessions_with_terminal_turn_markers, register_session_delete_mirror_hook, register_session_mirror_hook, update_account_id, update_agent_exec_mode, update_draft_text, update_mode_axes, update_model, diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/crud/ops.rs b/src-tauri/crates/agent-core/src/core/session/persistence/crud/ops.rs index 26f2754f39..9668fafd3d 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/crud/ops.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/crud/ops.rs @@ -4,7 +4,7 @@ //! mapper in [`super::record`] so the column list stays in one place. use chrono::Utc; -use rusqlite::{params, Result as SqliteResult}; +use rusqlite::{params, OptionalExtension, Result as SqliteResult}; use tracing::warn; use crate::persistence::db_helpers as shared; @@ -504,6 +504,20 @@ pub fn link_bootstrap_work_item(session_id: &str, work_item_id: &str) -> SqliteR }) } +/// Targeted point read of the canonical Agent Org roster member id. +/// The send-boundary Agent Org fence probes this for cache-hit sessions +/// without pulling the whole session record. +pub fn get_org_member_id(session_id: &str) -> SqliteResult> { + let conn = get_connection()?; + conn.query_row( + "SELECT org_member_id FROM agent_sessions WHERE session_id = ?1", + params![session_id], + |row| row.get::<_, Option>(0), + ) + .optional() + .map(Option::flatten) +} + /// Set the canonical Agent Org roster member id for a session. pub fn update_org_member_id(session_id: &str, org_member_id: &str) -> SqliteResult { let changed = with_sessions_writer(|| -> SqliteResult { diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs index c886c9bf16..59ebe81b96 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs @@ -25,7 +25,7 @@ mod sidebar; // doesn't need to be re-exported either. pub use crud::{ backfill_agent_definition_id, clear_worktree_metadata, delete_session, - finalize_terminal_turn_status, get_child_sessions, get_parent_session, get_session, + finalize_terminal_turn_status, get_child_sessions, get_org_member_id, get_parent_session, get_session, link_bootstrap_work_item, list_sessions, load_workspace, mark_stale_running_sessions_abandoned, reconcile_sessions_with_terminal_turn_markers, register_session_delete_mirror_hook, register_session_mirror_hook, save_workspace, save_worktree_metadata, session_type, diff --git a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs index 844b5db7d0..aeb3c7e4f0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs @@ -175,10 +175,34 @@ pub(super) async fn resolve_session_identity( }) .transpose()?; let native_harness_type = native_harness_after_l2.or(native_harness_from_db); - let has_persisted_agent_org_identity = db_record - .as_ref() - .and_then(|record| record.org_member_id.as_deref()) - .is_some(); + // The Agent Org send fence keys off this probe, so its reachability must + // not depend on the unrelated `needs_db` condition: a fully cache-hit + // send would otherwise silently skip the Team lifecycle fence. When the + // DB row was not loaded, a runtime that already carries its org context + // answers directly; ordinary cache-hit sessions pay one indexed point + // read of `org_member_id`. + let has_persisted_agent_org_identity = match db_record.as_ref() { + Some(record) => record.org_member_id.is_some(), + None if agent_org_run_id_hint.is_some() => true, + None => { + let sid = session_id.to_string(); + tokio::task::spawn_blocking(move || session_persistence::get_org_member_id(&sid)) + .await + .map_err(|join_err| { + format!( + "Task panic probing Agent Org identity for {}: {}", + session_id, join_err + ) + })? + .map_err(|db_err| { + format!( + "DB error probing Agent Org identity for {}: {}", + session_id, db_err + ) + })? + .is_some() + } + }; // ── Workspace Root ─────────────────────────────────────────────────── //