diff --git a/crates/tinyhivemind-driver/src/conduct/README.md b/crates/tinyhivemind-driver/src/conduct/README.md index c9db27c3..43f764b8 100644 --- a/crates/tinyhivemind-driver/src/conduct/README.md +++ b/crates/tinyhivemind-driver/src/conduct/README.md @@ -30,7 +30,19 @@ The rules, each with the decision it comes from: conversation waits with it rather than concluding for want of a turn. Nothing due with a seat parked is a wait, not a stall; `parked` says who, and `resume_seat` puts the seat back in the next wave, owed a turn where - it parked. `Event::Parked` and `Event::Resumed` mark both. + it parked. `Event::Parked` and `Event::Resumed` mark both. A held seat + also keeps `finished` false even where its own work closed some other way, + because ending the episode there would strand whatever the host queued to + hold it: the operator answers an approval with no loop to return to. +- **Checkpointing**: `snapshot` carries the wave in progress as well as the + episode, so a host checkpoints after **every committed row** rather than + once per wave, and a crash replays at most the one row whose sequence had + not been reported yet. It answers `None` only while the host holds a + commit it has not reported through `committed`: the conductor cannot say + whether that row landed, so it writes no claim either way. `resume` + rebuilds from a snapshot on a driver and a routing the host supplies + again, and a caller drains `step` before proposing a new wave -- a + restored wave is dropped otherwise, because `begin_wave` resets the phase. - **Sorting**: a broadcast or an ask made inside a conversation is desk work; only a post or a completion is a row of the conversation. - **Refusals**: a completion the ledger refuses is explained to the seat on diff --git a/crates/tinyhivemind-driver/src/conduct/child.rs b/crates/tinyhivemind-driver/src/conduct/child.rs index 5265da5f..9159c95a 100644 --- a/crates/tinyhivemind-driver/src/conduct/child.rs +++ b/crates/tinyhivemind-driver/src/conduct/child.rs @@ -1,5 +1,6 @@ //! One open conversation: a thread of the desk, run as its own episode. +use serde::{Deserialize, Serialize}; use tinyhivemind::Sequence; use crate::driver::{ConversationView, DriverState}; @@ -8,7 +9,7 @@ use crate::driver::{ConversationView, DriverState}; /// with the asker recorded here (ADR 0023). One question, one answer: the /// seat asked concludes with `complete_episode`, and its message is the /// answer; a follow-up is a further ask. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(super) struct Child { pub(super) root: Sequence, pub(super) asker: String, @@ -20,7 +21,9 @@ pub(super) struct Child { pub(super) last_by_askee: Option, /// Whether the seat asked has been told once that it has not answered. pub(super) nudged: bool, - /// Whether the seat asked took a turn in this wave. + /// Whether the seat asked took a turn in this wave. Carried across a + /// snapshot: a wave can be resumed mid-flight, and dropping this would + /// lose the nudge owed to a seat that was asked and said nothing. pub(super) turned: bool, } @@ -82,7 +85,7 @@ impl Child { /// A conversation that concluded, kept for the context of the seats that had /// it: shown whole once to each, on its next desk turn. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub(super) struct Concluded { pub(super) root: Sequence, pub(super) asker: String, diff --git a/crates/tinyhivemind-driver/src/conduct/mod.rs b/crates/tinyhivemind-driver/src/conduct/mod.rs index c3799e7b..1b841bee 100644 --- a/crates/tinyhivemind-driver/src/conduct/mod.rs +++ b/crates/tinyhivemind-driver/src/conduct/mod.rs @@ -47,6 +47,8 @@ mod wave; use std::collections::{BTreeMap, BTreeSet}; +use serde::{Deserialize, Serialize}; + use tinyhivemind::speech::{ToolCall, Utterance}; use tinyhivemind::{Conversation, Sequence}; use tinyhivemind_embed::RoutingPlan; @@ -110,6 +112,66 @@ pub fn starters(plan: &RoutingPlan, fallback: &str) -> Vec { } } +/// Everything a conductor needs to be rebuilt: the episode as the driver +/// folds it, the conversations open and concluded, and what each seat has +/// been shown, nudged for, or held on. +/// +/// Carries the wave in progress too, so a snapshot is exact rather than +/// per-wave: a host checkpoints after every committed row, and a crash +/// replays at most the one row whose sequence had not been reported yet. +/// The only point a snapshot cannot be taken is while the host holds a +/// commit it has not reported -- the conductor does not know whether that +/// row landed -- and [`Conductor::snapshot`] answers `None` there. +/// +/// The driver, the routing and the policy are **not** here. They are the +/// host's to supply again on resume, exactly as they were on open: a router +/// is a live object, and a policy the operator changed between restarts +/// should be the new one. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct ConductorState { + /// The desk's id, as every tool call names it. + pub chat: String, + /// The desk's display name. + pub desk_name: String, + /// The episode the driver folds. + pub state: DriverState, + /// The conversations still open, by their ask row. + children: Vec<(Sequence, Child)>, + /// The conversations that concluded, oldest first. + concluded: Vec, + /// How many concluded conversations each seat has been shown. + shown: BTreeMap, + /// The assignment each seat was last nudged for on the desk. + desk_nudged: BTreeMap, + /// Seats held on the host, by the thread they parked in. + parked: BTreeMap>, + /// Turns run so far. + turns: u64, + /// Waves proposed so far. + waves: u64, + /// Seats completed with their work for a spent broadcast budget. + discharged: u64, + /// The wave in progress: what has been said and not yet committed, and + /// the steps the host has not taken. Empty between waves. + wave: Wave, +} + +impl ConductorState { + /// Whether this snapshot was taken between waves, with nothing said and + /// nothing left for the host to do. + /// + /// A host does not need this -- [`resume_episode`] drains whatever the + /// wave holds either way -- but it is the difference between a restart + /// that lost a whole wave and one that lost a row. + /// + /// [`resume_episode`]: https://docs.rs/tinyhivemind-openhuman + #[must_use] + pub fn mid_wave_is_empty(&self) -> bool { + self.wave.is_idle() + } +} + /// The desk episode, its conversations, and the rules between them. pub struct Conductor<'a, A: BoundAgent> { driver: &'a CompletionDriver<'a, A>, @@ -214,7 +276,12 @@ impl<'a, A: BoundAgent> Conductor<'a, A> { /// Over: the desk is quiescent and no conversation is open. #[must_use] pub fn finished(&self) -> bool { - self.state.quiescent() && self.children.is_empty() + // A parked seat is not a finished one, even where its work closed + // some other way -- a broadcast it made in the same turn spending + // the budget, say. Ending the episode there would strand whatever + // the host queued to hold it: the operator answers an approval that + // no longer has a loop to return to. + self.state.quiescent() && self.children.is_empty() && self.parked.is_empty() } /// The desk episode's state. @@ -358,6 +425,134 @@ impl<'a, A: BoundAgent> Conductor<'a, A> { Ok(turns) } + /// Everything needed to rebuild this conductor, or `None` while the + /// host holds a commit it has not reported. + /// + /// A host checkpoints after every committed row. The wave in progress + /// travels with the snapshot, so a crash replays at most that one row: + /// the conductor comes back mid-wave with the same seats having spoken + /// and the same steps still to take. + /// + /// `None` means the host is holding a commit whose sequence it has not + /// reported through [`committed`](Self::committed). The conductor cannot + /// say whether that row reached the journal, so it will not write down a + /// claim either way; the caller reports the sequence and asks again. + #[must_use] + pub fn snapshot(&self) -> Option { + if !self.wave.recordable() { + return None; + } + Some(ConductorState { + chat: self.chat.clone(), + desk_name: self.desk_name.clone(), + state: self.state.clone(), + children: self + .children + .iter() + .map(|(root, child)| (*root, child.clone())) + .collect(), + concluded: self.concluded.clone(), + shown: self.shown.clone(), + desk_nudged: self.desk_nudged.clone(), + parked: self.parked.clone(), + turns: self.turns, + waves: self.waves, + discharged: self.discharged, + wave: self.wave.clone(), + }) + } + + /// Rebuild a conductor from a snapshot, on a driver and a routing the + /// host supplies again. + /// + /// The episode carries on where it paused: the same conversations are + /// open, the same seats are held, a seat already nudged for its + /// assignment is not nudged again for it, and a wave that was in + /// progress resumes mid-wave. A caller drains + /// [`step`](Self::step) before proposing a new wave, or the restored + /// steps are dropped. + /// + /// # Errors + /// + /// The driver refusing the episode or any of its conversations -- a + /// state naming a seat or a desk this hive does not have -- and + /// [`Error::InconsistentSnapshot`] for a snapshot that disagrees with + /// itself: a conversation filed under the wrong root, a cursor past the + /// conversations it points into, or a seat held that this desk does not + /// seat. + pub fn resume( + driver: &'a CompletionDriver<'a, A>, + routing: BroadcastRouting<'a>, + policy: ConductPolicy, + snapshot: ConductorState, + ) -> Result { + let state = driver.resume(snapshot.state)?; + // A snapshot is the host's file, not the conductor's memory: every + // part of it is validated here rather than trusted, because the + // alternative is an episode that resumes and then misbehaves waves + // later with nothing left to say why. + let mut children = BTreeMap::new(); + for (root, mut child) in snapshot.children { + if root != child.root { + return Err(Error::InconsistentSnapshot { + reason: format!( + "a conversation filed under {} calls itself {}", + root.0, child.root.0 + ), + }); + } + // The same validation the desk episode gets: the conversation + // names this desk, and its two seats are seats of this hive. + child.state = driver.resume(child.state)?; + children.insert(root, child); + } + // A cursor into the concluded list, for a seat that has been shown + // some of them. Past the end it would panic the first time that + // seat is briefed. + for (seat, cursor) in &snapshot.shown { + if *cursor > snapshot.concluded.len() { + return Err(Error::InconsistentSnapshot { + reason: format!( + "@{seat} has been shown {cursor} conversations of {}", + snapshot.concluded.len() + ), + }); + } + } + // A seat held on the host that this desk does not seat would be held + // for ever: it is never proposed, and nothing can release it into a + // wave. + for seat in snapshot.parked.keys() { + if !state + .episode() + .participants + .iter() + .any(|participant| &participant.agent_id == seat) + { + return Err(Error::InconsistentSnapshot { + reason: format!("@{seat} is held but is not seated at this desk"), + }); + } + } + Ok(Self { + driver, + routing, + chat: snapshot.chat, + desk_name: snapshot.desk_name, + policy, + state, + children, + concluded: snapshot.concluded, + shown: snapshot.shown, + desk_nudged: snapshot.desk_nudged, + parked: snapshot.parked, + turns: snapshot.turns, + waves: snapshot.waves, + discharged: snapshot.discharged, + wave: snapshot.wave, + }) + } + /// The seats held on the host, in seat order. An empty wave with any of /// these is the host's to end: it releases one with /// [`resume_seat`](Self::resume_seat), or gives up. diff --git a/crates/tinyhivemind-driver/src/conduct/test/parked.rs b/crates/tinyhivemind-driver/src/conduct/test/parked.rs index 8e891f21..91e90a3a 100644 --- a/crates/tinyhivemind-driver/src/conduct/test/parked.rs +++ b/crates/tinyhivemind-driver/src/conduct/test/parked.rs @@ -4,11 +4,13 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] use super::support::{ - Journal, ask, broadcast, complete, hive, policy, seats, two_seat, wave, wave_parking, + Journal, ask, broadcast, complete, hive, policy, run, seats, two_seat, wave, wave_parking, }; use crate::conduct::{ConductPolicy, Conductor, Event}; use crate::driver::BroadcastRouting; +use crate::test_support::Seat; use crate::{CompletionDriver, Error}; +use serde_json::{Value, json}; use tinyhivemind::Sequence; #[test] @@ -200,3 +202,228 @@ fn what_a_seat_said_before_it_parked_is_recorded() { ); assert_eq!(conductor.parked(), vec!["one".to_owned()]); } + +#[test] +fn a_snapshot_is_taken_between_waves_and_carries_the_episode_on() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(1); + let routing = BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = Journal::default(); + let mut conductor = two_seat(&driver, routing, ConductPolicy::default(), &journal); + // One asks two, and two parks: a conversation open and a seat held is + // the most state a snapshot has to carry. + wave( + &mut conductor, + &journal, + &[("one", vec![ask("two", "may I ship?")])], + ) + .expect("wave"); + wave_parking(&mut conductor, &journal, &[], &["two"]).expect("wave"); + let root = Sequence(2); + let snapshot = conductor.snapshot().expect("the wave settled"); + assert_eq!(snapshot.chat, "engineering"); + assert_eq!(conductor.parked(), vec!["two".to_owned()]); + let turns_before = conductor.turns_run(); + + // Through the wire and back, onto a driver the host supplies again. + let wire = serde_json::to_string(&snapshot).expect("serializes"); + let restored: crate::ConductorState = serde_json::from_str(&wire).expect("deserializes"); + let routing = BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let mut resumed = + Conductor::resume(&driver, routing, ConductPolicy::default(), restored).expect("resumes"); + assert_eq!(resumed.turns_run(), turns_before, "the count carries"); + assert_eq!( + resumed.parked(), + vec!["two".to_owned()], + "the held seat is still held" + ); + // Released, it answers in the conversation that was open before the + // restart, and the episode concludes it. + resumed.resume_seat("two"); + let answered = wave( + &mut resumed, + &journal, + &[("two", vec![complete("ship it")])], + ) + .expect("wave"); + assert_eq!(seats(&answered.turns)[0], ("two", Some(root))); + assert_eq!( + resumed.conversations(), + 1, + "the conversation survived the restart and concluded" + ); +} + +#[test] +fn a_snapshot_is_refused_only_while_the_host_holds_an_unreported_commit() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(1); + let routing = BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = Journal::default(); + let mut conductor = two_seat(&driver, routing, ConductPolicy::default(), &journal); + assert!( + conductor.snapshot().is_some(), + "a fresh conductor is between waves" + ); + for step in conductor.begin_wave() { + let _ = step; + } + let turns = conductor.turns().expect("turns"); + conductor.record( + &turns[0], + [tinyhivemind::speech::ToolCall::Speak(complete("done"))], + ); + // Mid-wave, before any commit leaves: the wave travels with the + // snapshot, so this is recordable. + let held = conductor + .snapshot() + .expect("a wave in progress is still recordable"); + assert!( + !held.mid_wave_is_empty(), + "the snapshot carries what the wave has not committed yet" + ); + // The host now holds a commit whose sequence it has not reported. The + // conductor cannot say whether that row landed, so it will not write a + // claim either way. + let step = conductor.step().expect("a step"); + assert!(matches!(step, Some(crate::Step::Commit(_)))); + assert!( + conductor.snapshot().is_none(), + "a commit is out there and unreported" + ); + // Reported, and it is recordable again. + let sequence = journal.append("one", "COMPLETE: done", None, None); + run(conductor.committed(sequence)).expect("committed"); + assert!(conductor.snapshot().is_some()); +} + +#[test] +fn a_held_seat_keeps_the_episode_open_even_where_its_work_closed() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(1); + let routing = BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = Journal::default(); + let mut conductor = two_seat(&driver, routing, ConductPolicy::default(), &journal); + // One completes and parks in the same turn: its work is closed, so the + // desk is quiescent, but the host is still holding it. + wave_parking( + &mut conductor, + &journal, + &[("one", vec![complete("done")])], + &["one"], + ) + .expect("wave"); + assert_eq!(conductor.parked(), vec!["one".to_owned()]); + assert!( + !conductor.finished(), + "a held seat is not a finished one: the operator's answer needs a \ + loop to come back to" + ); + // Released, there is nothing left to hold and the episode is over. + conductor.resume_seat("one"); + assert!(conductor.parked().is_empty()); + assert!(conductor.finished()); +} + +/// A snapshot with one conversation open and one seat held, as a value a +/// test can bend before handing it back. +fn snapshot_with_a_conversation(journal: &Journal, conductor: &mut Conductor<'_, Seat>) -> Value { + wave( + conductor, + journal, + &[("one", vec![ask("two", "which port?")])], + ) + .expect("wave"); + serde_json::to_value(conductor.snapshot().expect("recordable")).expect("serializes") +} + +#[test] +fn a_snapshot_that_disagrees_with_itself_is_refused_rather_than_resumed() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(1); + let routing = || BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = Journal::default(); + let mut conductor = two_seat(&driver, routing(), ConductPolicy::default(), &journal); + let good = snapshot_with_a_conversation(&journal, &mut conductor); + // The unbent snapshot resumes, so each refusal below is about the bend. + let restored: crate::ConductorState = + serde_json::from_value(good.clone()).expect("deserializes"); + assert!(Conductor::resume(&driver, routing(), ConductPolicy::default(), restored).is_ok()); + + // A conversation filed under a root it does not call its own. + let mut bent = good.clone(); + bent["children"][0][0] = json!(99); + let restored: crate::ConductorState = serde_json::from_value(bent).expect("deserializes"); + let refused = Conductor::resume(&driver, routing(), ConductPolicy::default(), restored); + assert!( + matches!(&refused, Err(Error::InconsistentSnapshot { reason }) if reason.contains("99")), + "{refused:?}" + ); + + // A cursor past the conversations it points into: shown_conversations + // would index off the end of the list the first time that seat spoke. + let mut bent = good.clone(); + bent["shown"] = json!({ "one": 7 }); + let restored: crate::ConductorState = serde_json::from_value(bent).expect("deserializes"); + let refused = Conductor::resume(&driver, routing(), ConductPolicy::default(), restored); + assert!( + matches!(&refused, Err(Error::InconsistentSnapshot { reason }) if reason.contains("@one")), + "{refused:?}" + ); + + // A seat held that this desk does not seat: nothing could ever release + // it into a wave. + let mut bent = good.clone(); + bent["parked"] = json!({ "nobody": null }); + let restored: crate::ConductorState = serde_json::from_value(bent).expect("deserializes"); + let refused = Conductor::resume(&driver, routing(), ConductPolicy::default(), restored); + assert!( + matches!(&refused, Err(Error::InconsistentSnapshot { reason }) if reason.contains("nobody")), + "{refused:?}" + ); + + // A conversation whose own state names another desk is refused by the + // driver, exactly as the desk episode's state would be. + let mut bent = good; + bent["children"][0][1]["state"]["episode"]["conversation"]["desk_id"] = json!("marketing"); + let restored: crate::ConductorState = serde_json::from_value(bent).expect("deserializes"); + let refused = Conductor::resume(&driver, routing(), ConductPolicy::default(), restored); + assert!( + matches!(&refused, Err(Error::OutOfHiveEpisode { .. })), + "{refused:?}" + ); +} diff --git a/crates/tinyhivemind-driver/src/conduct/test/wire.rs b/crates/tinyhivemind-driver/src/conduct/test/wire.rs index 25fb0d33..dced3b42 100644 --- a/crates/tinyhivemind-driver/src/conduct/test/wire.rs +++ b/crates/tinyhivemind-driver/src/conduct/test/wire.rs @@ -246,3 +246,80 @@ fn every_event_and_refusal_survives_the_wire() { json!({"kind": "not_yet_shown"}) ); } + +#[test] +fn a_conductor_snapshot_names_every_field_a_restart_reads_back() { + let hive = super::support::hive(&["one", "two"]); + let driver = crate::CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = super::support::policy(1); + let routing = crate::driver::BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = super::support::Journal::default(); + let mut conductor = + super::support::two_seat(&driver, routing, crate::ConductPolicy::default(), &journal); + // A conversation open and a seat held: the two pieces of state a + // restart most needs, so both are on the wire. + super::support::wave( + &mut conductor, + &journal, + &[("one", vec![super::support::ask("two", "which port?")])], + ) + .expect("wave"); + let snapshot = conductor.snapshot().expect("recordable"); + let wire = serde_json::to_value(&snapshot).expect("serializes"); + let object = wire.as_object().expect("an object"); + + // The spelling a stored snapshot is read back by. Renaming any of these + // orphans every snapshot written by the build before it, which is the + // failure the whole checkpoint exists to prevent. + let mut keys: Vec<&str> = object.keys().map(String::as_str).collect(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "chat", + "children", + "concluded", + "desk_name", + "desk_nudged", + "discharged", + "parked", + "shown", + "state", + "turns", + "wave", + "waves", + ] + ); + assert_eq!(object["chat"], json!("engineering")); + + // A conversation on the wire is its root paired with its record: the + // key is the root a host files it under, and `resume` refuses a pair + // whose two halves disagree. + let pair = &wire["children"][0]; + let child = &pair[1]; + assert_eq!( + pair[0], child["root"], + "the key a conversation is filed under is its own root" + ); + for field in [ + "root", "asker", "askee", "state", "turns", "nudged", "turned", + ] { + assert!( + child.get(field).is_some(), + "a conversation must carry `{field}`: {child}" + ); + } + assert_eq!(child["asker"], json!("one")); + assert_eq!(child["askee"], json!("two")); + + // And it decodes back to the same thing. + let back: crate::ConductorState = serde_json::from_value(wire).expect("deserializes"); + assert_eq!(back.chat, snapshot.chat); + assert_eq!(back.mid_wave_is_empty(), snapshot.mid_wave_is_empty()); +} diff --git a/crates/tinyhivemind-driver/src/conduct/wave.rs b/crates/tinyhivemind-driver/src/conduct/wave.rs index c680f121..6d1233b2 100644 --- a/crates/tinyhivemind-driver/src/conduct/wave.rs +++ b/crates/tinyhivemind-driver/src/conduct/wave.rs @@ -10,6 +10,8 @@ use std::collections::VecDeque; +use serde::{Deserialize, Serialize}; + use tinyhivemind::Sequence; use tinyhivemind::speech::Utterance; use tinyhivemind_hive::CompletionEpisodeState; @@ -22,7 +24,8 @@ use tinyhivemind::Conversation; use crate::{BoundAgent, Error, Result}; -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] enum Phase { /// No wave in progress. #[default] @@ -40,7 +43,8 @@ enum Phase { } /// One wave's bookkeeping. -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] pub(super) struct Wave { phase: Phase, /// Nothing was due: every open conversation concludes without an answer. @@ -59,6 +63,28 @@ pub(super) struct Wave { } impl Wave { + /// Whether this wave can be written down truthfully. + /// + /// Everything here is the conductor's own: a queued step or commit has + /// not reached the host, so a snapshot carrying it is recoverable by + /// re-issuing it. The one exception is `outstanding` -- a commit the + /// host holds and has not reported the sequence of. The conductor does + /// not know whether that row landed, so it is the one point a snapshot + /// cannot describe the journal, and the caller waits for the report. + /// Nothing said, nothing queued, nothing out: between waves. + pub(super) fn is_idle(&self) -> bool { + matches!(self.phase, Phase::Idle) + && self.thread.is_empty() + && self.desk.is_empty() + && self.steps.is_empty() + && self.commits.is_empty() + && self.outstanding.is_none() + } + + pub(super) fn recordable(&self) -> bool { + self.outstanding.is_none() + } + pub(super) fn begin(&mut self, nothing_due: bool) { self.phase = Phase::Threads; self.force_conclusions = nothing_due; diff --git a/crates/tinyhivemind-driver/src/error/mod.rs b/crates/tinyhivemind-driver/src/error/mod.rs index a889712e..57dad79a 100644 --- a/crates/tinyhivemind-driver/src/error/mod.rs +++ b/crates/tinyhivemind-driver/src/error/mod.rs @@ -156,6 +156,19 @@ pub enum Error { /// Number of committed receipts represented by the state. receipt_count: usize, }, + /// A restored conversation disagrees with the snapshot that carries it: + /// its key is not its root, its seats are not the two it names, or a + /// cursor points past what it points into. + /// + /// A snapshot is host-stored data, so it is validated on the way in + /// rather than trusted: the alternative is an episode that resumes and + /// then panics or misbehaves several waves later, where the cause is + /// unrecoverable. + #[error("restored episode state is inconsistent: {reason}")] + InconsistentSnapshot { + /// What disagreed. + reason: String, + }, /// A serialized freshness floor disagrees with its episode and receipts. #[error("driver freshness floor {stored} does not match derived floor {derived}")] InvalidFreshnessFloor { diff --git a/crates/tinyhivemind-driver/src/lib.rs b/crates/tinyhivemind-driver/src/lib.rs index 1ae94c6b..4136c298 100644 --- a/crates/tinyhivemind-driver/src/lib.rs +++ b/crates/tinyhivemind-driver/src/lib.rs @@ -78,7 +78,8 @@ pub mod graph; mod test_support; pub use conduct::{ - Commit, ConductPolicy, Conductor, Door, Event, Note, Refusal, Step, Turn, starters, + Commit, ConductPolicy, Conductor, ConductorState, Door, Event, Note, Refusal, Step, Turn, + starters, }; pub use driver::{ AssignmentSpend, BroadcastRouting, Channel, CommittedUtterance, CompletionDriver, diff --git a/crates/tinyhivemind-openhuman/src/episode/README.md b/crates/tinyhivemind-openhuman/src/episode/README.md index c69c9f6f..3117220f 100644 --- a/crates/tinyhivemind-openhuman/src/episode/README.md +++ b/crates/tinyhivemind-openhuman/src/episode/README.md @@ -9,11 +9,11 @@ conductor's. `Journal` is what a host implements: its `SessionLog`, `commit` and `note` to append the conductor's rows and return the sequence a commit was given, -and five optional hooks -- `event` to show what the episode did, `compose` +and six optional hooks -- `event` to show what the episode did, `compose` to put its own context in front of the brief, `turn_done` to see a turn's reply, refusals and recorded calls, `channels` to name the seat's other -conversations, and `released` to say which parked seats the host has -settled. `Report` is what an episode came to. +conversations, `released` to say which parked seats the host has settled, +and `checkpoint` to keep the snapshot a restart resumes from. `Report` is what an episode came to. A turn that comes back `TurnResult::Parked` is recorded with whatever it called and then held: the conductor stops proposing that seat. When a wave @@ -21,6 +21,18 @@ has nothing to run and seats are parked, the loop asks `released`, which is where a host blocks on its own approval queue; a host that releases nobody ends the episode with `Error::Parked` rather than spinning. +`checkpoint` is handed a `ConductorState` after every committed row, and +again at the end of each wave so a wave that only parked or nudged a seat +is durable too. `resume_episode` carries an episode on from the newest one: +the same conversations open, the same seats held, and a wave that was in +progress resumed mid-wave. A host that keeps nothing loses a running +episode to a restart. + +The residual window is one row: a crash between a row landing and +`checkpoint` returning leaves the journal ahead of the snapshot, so the +seat that wrote that row runs again. A host that cannot tolerate a +duplicate keys its appends and drops one it has already written. + `channels` names every conversation the seat is in that this episode does not run. Their newest rows are read through `gather_elsewhere`, bounded by the same wave watermark as every other read, and carried in diff --git a/crates/tinyhivemind-openhuman/src/episode/mod.rs b/crates/tinyhivemind-openhuman/src/episode/mod.rs index 24d0b31e..3f3fd156 100644 --- a/crates/tinyhivemind-openhuman/src/episode/mod.rs +++ b/crates/tinyhivemind-openhuman/src/episode/mod.rs @@ -28,8 +28,8 @@ use tinyhivemind::{ SessionMessage, SessionQuery, gather_elsewhere, project_session, }; use tinyhivemind_driver::{ - BoundAgent, BroadcastRouting, Commit, CompletionDriver, ConductPolicy, Conductor, Door, - ElsewhereView, EpisodeBrief, Event, Note, Step, Turn, + BoundAgent, BroadcastRouting, Commit, CompletionDriver, ConductPolicy, Conductor, + ConductorState, Door, ElsewhereView, EpisodeBrief, Event, Note, Step, Turn, }; use tinyhivemind_tools::{Dispatch, Refusal}; @@ -80,6 +80,29 @@ pub trait Journal: Send + Sync { Vec::new() } + /// The episode is exactly where this snapshot says. The default keeps + /// nothing, and such a host loses a running episode to a restart. + /// + /// Called after **every committed row**, not once per wave: the row is + /// in the journal and the conductor has folded it, so the two agree. + /// A host stores the snapshot beside its rows -- ideally in the same + /// journal, so the ordering is the journal's own -- and hands the + /// newest one to [`resume_episode`] on boot. + /// + /// A crash between a row landing and this returning replays that one + /// row: the resumed conductor has not folded it, so the seat that wrote + /// it runs again. A host that cannot tolerate a duplicate row keys its + /// appends and drops one it has already written. + /// + /// # Errors + /// + /// The host failing to store it, which ends the episode: an episode + /// that cannot be checkpointed is one a restart would lose silently. + fn checkpoint(&self, state: &ConductorState) -> Result<()> { + let _ = state; + Ok(()) + } + /// Nothing is due and these seats are parked: the seats the host has /// settled, waiting until it has one. /// @@ -161,7 +184,59 @@ where desk_name: door.desk_name.clone(), thread_root: None, }; - let mut conductor = Conductor::open(driver, routing, policy, door)?; + let conductor = Conductor::open(driver, routing, policy, door)?; + drive(journal, runner, conductor, desk).await +} + +/// Carry on an episode from a snapshot the host stored. +/// +/// The same loop as [`run_episode`], opened from +/// [`Conductor::resume`](tinyhivemind_driver::Conductor::resume) rather than +/// from a door: the same conversations are open, the same seats are held, +/// and the rows already committed are already in the host's journal. +/// +/// # Errors +/// +/// Whatever [`run_episode`] errors on, plus the driver refusing the snapshot +/// -- one naming a seat or a desk this hive does not have. +pub async fn resume_episode( + journal: &J, + runner: &R, + driver: &CompletionDriver<'_, A>, + routing: BroadcastRouting<'_>, + policy: ConductPolicy, + snapshot: ConductorState, +) -> Result +where + A: BoundAgent, + J: Journal, + R: SeatRunner, +{ + let desk = Conversation { + desk_id: snapshot.chat.clone(), + desk_name: snapshot.desk_name.clone(), + thread_root: None, + }; + let conductor = Conductor::resume(driver, routing, policy, snapshot)?; + drive(journal, runner, conductor, desk).await +} + +/// The loop itself, however the conductor was opened. +async fn drive( + journal: &J, + runner: &R, + mut conductor: Conductor<'_, A>, + desk: Conversation, +) -> Result +where + A: BoundAgent, + J: Journal, + R: SeatRunner, +{ + // A snapshot taken mid-wave comes back mid-wave: drain what it restored + // before proposing anything, because `begin_wave` resets the phase and + // would drop those steps on the floor. + settle_wave(journal, &mut conductor).await?; loop { if conductor.finished() { break; @@ -206,8 +281,12 @@ where } } } - while let Some(step) = conductor.step()? { - settle(journal, &mut conductor, step).await?; + settle_wave(journal, &mut conductor).await?; + // And once the wave is over, whether or not it committed anything: + // a wave that only parked a seat or nudged one moved state no row + // records, and a restart would otherwise lose it. + if let Some(snapshot) = conductor.snapshot() { + journal.checkpoint(&snapshot)?; } } Ok(Report { @@ -219,6 +298,26 @@ where }) } +/// Take every step the wave has left, checkpointing after each committed +/// row so a crash replays at most that one row. +async fn settle_wave( + journal: &J, + conductor: &mut Conductor<'_, A>, +) -> Result<()> { + while let Some(step) = conductor.step()? { + let committed = matches!(step, Step::Commit(_)); + settle(journal, conductor, step).await?; + // After a commit the row is in the journal and the conductor has + // folded it, so the snapshot and the journal agree. A note or an + // event moves nothing a resume would double-count, so neither is + // worth a write. + if committed && let Some(snapshot) = conductor.snapshot() { + journal.checkpoint(&snapshot)?; + } + } + Ok(()) +} + /// One step taken: a commit appended and its sequence reported, a note /// appended, an event shown. async fn settle( @@ -285,6 +384,13 @@ async fn wait_for_release( for seat in &released { conductor.resume_seat(seat); } + // Releasing the last held seat can be the thing that finishes the + // episode -- its work may have closed while it was held. Ask again only + // if there is still an episode to ask about; `turns` would read an + // empty wave with nothing parked as a stall. + if conductor.finished() { + return Ok(Vec::new()); + } Ok(conductor.turns()?) } diff --git a/crates/tinyhivemind-openhuman/src/episode/test/support.rs b/crates/tinyhivemind-openhuman/src/episode/test/support.rs index 23c60a1e..18c22089 100644 --- a/crates/tinyhivemind-openhuman/src/episode/test/support.rs +++ b/crates/tinyhivemind-openhuman/src/episode/test/support.rs @@ -11,6 +11,7 @@ use serde_json::{Value, json}; use tinyhivemind::desk::{Desk, ResponderMode}; use tinyhivemind::responder::Probability; use tinyhivemind::{Conversation, Sequence, SessionFuture, SessionLog}; +use tinyhivemind_driver::ConductorState; use tinyhivemind_driver::{ AgentBinding, BoundAgent, BoundHive, Commit, Door, EpisodeBrief, Event, HiveGraph, Note, }; @@ -146,6 +147,8 @@ pub(super) struct TestJournal { pub(super) release: Mutex>>, /// Every set of parked seats the loop asked about. pub(super) asked: Mutex>>, + /// Every snapshot the loop handed over, in order. + pub(super) checkpoints: Mutex>, } impl TestJournal { @@ -162,6 +165,7 @@ impl TestJournal { channels: Mutex::new(Vec::new()), release: Mutex::new(VecDeque::new()), asked: Mutex::new(Vec::new()), + checkpoints: Mutex::new(Vec::new()), } } @@ -178,6 +182,14 @@ impl Journal for TestJournal { &self.log } + fn checkpoint(&self, state: &ConductorState) -> Result<()> { + self.checkpoints + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(state.clone()); + Ok(()) + } + fn channels(&self, _seat: &str) -> Vec { self.channels .lock() diff --git a/crates/tinyhivemind-openhuman/src/episode/test/watermark.rs b/crates/tinyhivemind-openhuman/src/episode/test/watermark.rs index bb19444e..2a3b2e76 100644 --- a/crates/tinyhivemind-openhuman/src/episode/test/watermark.rs +++ b/crates/tinyhivemind-openhuman/src/episode/test/watermark.rs @@ -8,8 +8,12 @@ use std::sync::atomic::AtomicBool; use tinyhivemind::Sequence; use tinyhivemind_driver::{BroadcastRouting, CompletionDriver, ConductPolicy, Door}; -use super::super::run_episode; -use super::support::{GrowingLog, ScriptRunner, TestJournal, complete, door, hive, policy, run}; +use super::super::{resume_episode, run_episode}; +use serde_json::json; + +use super::support::{ + GrowingLog, ScriptRunner, TestJournal, ask, complete, door, hive, policy, run, +}; use crate::journal::MemoryLog; #[test] @@ -106,3 +110,106 @@ fn a_row_the_host_appends_above_the_wave_watermark_is_shown_once_and_later() { assert!(prompts[1].2.contains("one more thing"), "{}", prompts[1].2); assert_eq!(runner.since(), vec![None, Some(opened_at)]); } + +#[test] +fn an_episode_is_checkpointed_every_wave_and_carries_on_from_one() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(); + let routing = || BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = TestJournal::new(); + // One parks on its first turn; with nobody released the episode ends + // parked, leaving a checkpoint behind with the seat still held. + let runner = ScriptRunner::new(&["one", "two"], &[("one", vec![vec![("park", json!({}))]])]); + let entrance = door(&journal, &["one", "two"], &["one"]); + let stopped = run(run_episode( + &journal, + &runner, + &driver, + routing(), + ConductPolicy::default(), + entrance, + )); + assert!(stopped.is_err(), "{stopped:?}"); + let snapshot = journal + .checkpoints + .lock() + .unwrap() + .last() + .cloned() + .expect("a row was committed before the episode stopped"); + assert_eq!(snapshot.chat, "engineering"); + + // A fresh process: the same journal, a new runner, resumed from the + // snapshot. The seat is still held, and released it completes. + let after = ScriptRunner::new( + &["one", "two"], + &[("one", vec![vec![complete("approved", None)]])], + ); + let report = run(resume_episode( + &journal, + &after, + &driver, + routing(), + ConductPolicy::default(), + snapshot, + )); + // Nobody released it, so it stops parked again rather than silently + // starting over: the hold survived the restart, which is the point. + assert!(report.is_err(), "{report:?}"); + assert!( + after.prompts().is_empty(), + "a held seat is not proposed on resume: {:?}", + after.prompts() + ); +} + +#[test] +fn a_crash_after_a_row_lands_replays_that_row_and_no_more() { + let hive = hive(&["one", "two"]); + let driver = CompletionDriver::new(&hive, 4).expect("driver"); + let route_policy = policy(); + let routing = || BroadcastRouting { + primary: None, + reasoning: None, + policy: &route_policy, + roster_version: 1, + thread_context: &[], + }; + let journal = TestJournal::new(); + // One asks two: the ask is a committed row, and the wave that carries + // it has more to do afterwards. + let runner = ScriptRunner::new( + &["one", "two"], + &[("one", vec![vec![ask("two", "which port?", None)]])], + ); + let entrance = door(&journal, &["one", "two"], &["one"]); + // Stop the episode the moment the ask has been committed, by refusing + // to release anybody once nothing is due. + let _ = run(run_episode( + &journal, + &runner, + &driver, + routing(), + ConductPolicy::default(), + entrance, + )); + let checkpoints = journal.checkpoints.lock().unwrap().clone(); + assert!( + checkpoints.len() > 1, + "a checkpoint per committed row, not one per wave: {}", + checkpoints.len() + ); + // The first checkpoint is taken mid-wave: the wave it carries is not + // idle, which is what bounds a crash to one row. + assert!( + checkpoints.iter().any(|state| !state.mid_wave_is_empty()), + "at least one checkpoint was taken with the wave still in progress" + ); +} diff --git a/crates/tinyhivemind-openhuman/src/lib.rs b/crates/tinyhivemind-openhuman/src/lib.rs index 2e598a80..f42ac2f5 100644 --- a/crates/tinyhivemind-openhuman/src/lib.rs +++ b/crates/tinyhivemind-openhuman/src/lib.rs @@ -124,7 +124,7 @@ pub mod raw; pub mod runner; pub use embed::{EmbedRunner, EmbedSeat}; -pub use episode::{Journal, Released, Report, run_episode}; +pub use episode::{Journal, Released, Report, resume_episode, run_episode}; pub use error::{Error, Result}; pub use hosted::{Disposition, EpisodeBelt, EpisodeHost, HostedRunner, HostedSeat, HostedTurn}; pub use journal::MemoryLog;