diff --git a/crates/tinytools-jev/README.md b/crates/tinytools-jev/README.md index fda20a6..9c76926 100644 --- a/crates/tinytools-jev/README.md +++ b/crates/tinytools-jev/README.md @@ -46,3 +46,19 @@ Implement `JevEvaluator` in the host by translating `JevRequest` into the client's wire request and translating its answer into `JevDecision`. Pass that implementation to `JevRanker::new`. This keeps transport and runtime choices at the host boundary where their policy belongs. + +## Strategies + +`JevRankerConfig::with_strategy` picks how the catalogue is narrowed before +the evaluator decides: + +- `RetrieveThenDecide` (default): the retriever shortlists `retrieval_k` + candidates, one evaluation decides. Bounded by the retriever's recall. +- `FamilyThenDecide`: one evaluation over the candidates' families (a + toolkit, a pack; candidates without one form `core`), then one evaluation + per chosen family (`max_families`, default 2, run concurrently) over all + its members. No retrieval for a family that fits one choice, so a paraphrase + is judged semantically at both steps; a larger family is cut to + `MAX_CANDIDATES` by the retriever. The family stage sets + `JevRequest::instructions` so the evaluator asks "which group" rather than + "which tool"; `JevRanking::families` reports what it chose. diff --git a/crates/tinytools-jev/src/family.rs b/crates/tinytools-jev/src/family.rs new file mode 100644 index 0000000..142d188 --- /dev/null +++ b/crates/tinytools-jev/src/family.rs @@ -0,0 +1,343 @@ +//! [`JevStrategy::FamilyThenDecide`]: the evaluator picks the family first, +//! then decides among every member of the top families. +//! +//! One evaluation over the families (a toolkit, a pack — a small choice), +//! then one evaluation per chosen family over all its members, run +//! concurrently. No retrieval for a family that fits one choice, so a +//! paraphrase ("ping alex" for `SLACK_SEND_MESSAGE`) is judged semantically +//! at both steps. A family larger than one choice is cut to fit by the +//! configured retriever — the one place recall can still be lost, and the +//! reason a host should give the ranker a semantic retriever. + +use std::collections::{BTreeMap, HashSet}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context as TaskContext, Poll}; + +use tinytools::{RankCandidate, RankContext, RankError, RankHit}; + +use crate::{ + JevDecision, JevOption, JevRanker, JevRankerConfig, JevRanking, JevRequest, JevStrategy, + NONE_OPTION, option_text_clipped, validate_candidates, +}; + +/// Shorter clip for a whole-family choice, which can hold 254 members and +/// has to stay under a provider's per-request token and cost ceilings. +const FAMILY_SUMMARY_CHARS: usize = 150; +/// The family candidates without one are grouped into. +const CORE_FAMILY: &str = "core"; +/// Member names shown per family in the first stage. +const FAMILY_SAMPLE: usize = 12; +/// A decision whose `needs_tool` is below this abstains, as `decode` does. +const NEEDS_TOOL_FLOOR: f64 = 0.5; + +type Families<'a> = BTreeMap<&'a str, Vec<&'a RankCandidate>>; + +/// The strategy's entry point; see the module docs. +pub(crate) async fn rank( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, +) -> Result { + debug_assert_eq!(ranker.config().strategy(), JevStrategy::FamilyThenDecide); + validate_candidates(candidates)?; + let started = std::time::Instant::now(); + let mut families: Families<'_> = BTreeMap::new(); + for candidate in candidates { + families + .entry(candidate.family.as_deref().unwrap_or(CORE_FAMILY)) + .or_default() + .push(candidate); + } + let chosen = match choose_families(ranker, intent, context, &families).await? { + Chosen::Families(chosen) => chosen, + Chosen::Nothing(mut empty) => { + empty.latency = started.elapsed(); + return Ok(empty); + } + }; + + // Second stage: every chosen family at once. + let mut requests: Vec<(String, f64, Vec<&RankCandidate>)> = Vec::new(); + for (family, p_family) in &chosen { + let Some(members) = families.get(family.as_str()) else { + continue; + }; + let members = fit_one_choice(ranker, intent, context, members).await?; + requests.push((family.clone(), *p_family, members)); + } + let decisions = join_all( + requests + .iter() + .map(|(family, _, members)| { + let request = family_request(ranker, intent, context, family, members); + async move { ranker.evaluator().evaluate(&request).await } + }) + .collect(), + ) + .await; + + let mut ranking = JevRanking::empty(); + ranking.families = chosen; + for ((_, p_family, members), decision) in requests.iter().zip(decisions) { + let decision = decision?; + merge( + &mut ranking, + &decision, + *p_family, + members, + ranker.config().min_probability, + ); + ranking.shortlisted += members.len(); + ranking.input_tokens = match (ranking.input_tokens, decision.input_tokens) { + (Some(a), Some(b)) => Some(a + b), + (a, b) => a.or(b), + }; + ranking.attempts = ranking.attempts.max(decision.attempts); + ranking.needs_tool = match (ranking.needs_tool, decision.needs_tool) { + (Some(a), Some(b)) => Some(a.max(b)), + (a, b) => a.or(b), + }; + } + if ranking.needs_tool.is_some_and(|p| p < NEEDS_TOOL_FLOOR) { + ranking.hits.clear(); + } + ranking + .hits + .sort_by(|a, b| b.score.total_cmp(&a.score).then_with(|| a.key.cmp(&b.key))); + ranking.hits.truncate(limit); + ranking.latency = started.elapsed(); + Ok(ranking) +} + +enum Chosen { + Families(Vec<(String, f64)>), + Nothing(JevRanking), +} + +/// First stage: which families could answer, best first, at most +/// `max_families`, each above `min_probability`. +async fn choose_families( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + families: &Families<'_>, +) -> Result { + if families.len() == 1 { + return Ok(Chosen::Families( + families.keys().map(|f| ((*f).to_owned(), 1.0)).collect(), + )); + } + if families.len() > JevRankerConfig::MAX_CANDIDATES { + return Err(RankError::invalid_input("too many families for one choice")); + } + let mut options: Vec = families + .iter() + .map(|(family, members)| { + if *family == NONE_OPTION { + return Err(RankError::invalid_input("family `none` is reserved")); + } + Ok(JevOption { + key: (*family).to_owned(), + description: family_summary(family, members), + }) + }) + .collect::>()?; + options.push(JevOption { + key: NONE_OPTION.into(), + description: "No listed group of tools is relevant to the request.".into(), + }); + let request = JevRequest { + intent: intent.into(), + recent_turns: context.recent_turns.clone(), + options, + model: ranker.config().model.clone(), + instructions: Some( + "Which group of tools would accomplish the user's request? Each option names \ + a service or a category and lists what its tools do. Pick `none` when no \ + group applies." + .into(), + ), + }; + let decision = ranker.evaluator().evaluate(&request).await?; + let none = decision + .probabilities + .get(NONE_OPTION) + .copied() + .unwrap_or(0.0); + let mut ordered: Vec<(String, f64)> = decision + .probabilities + .iter() + .filter(|(name, _)| name.as_str() != NONE_OPTION && families.contains_key(name.as_str())) + .map(|(name, p)| (name.clone(), *p)) + .collect(); + ordered.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + ordered.truncate(ranker.config().max_families); + ordered.retain(|(_, p)| *p >= ranker.config().min_probability); + let abstained = ordered.first().is_none_or(|(_, best)| none >= *best) + || decision.needs_tool.is_some_and(|p| p < NEEDS_TOOL_FLOOR); + if abstained { + let mut empty = JevRanking::empty(); + empty.attempts = decision.attempts; + empty.input_tokens = decision.input_tokens; + empty.none_probability = none; + empty.needs_tool = decision.needs_tool; + empty.choice_confidence = decision.choice_confidence; + return Ok(Chosen::Nothing(empty)); + } + Ok(Chosen::Families(ordered)) +} + +/// The second-stage request for one family: every member (already cut to +/// fit) plus `none`. +fn family_request( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + family: &str, + members: &[&RankCandidate], +) -> JevRequest { + let mut options: Vec = members + .iter() + .map(|m| JevOption { + key: m.key.clone(), + description: option_text_clipped(m, FAMILY_SUMMARY_CHARS), + }) + .collect(); + options.push(JevOption { + key: NONE_OPTION.into(), + description: "No listed tool accomplishes the request.".into(), + }); + JevRequest { + intent: intent.into(), + recent_turns: context.recent_turns.clone(), + options, + model: ranker.config().model.clone(), + instructions: Some(format!( + "Which `{family}` tool accomplishes the user's request? Judge by what each \ + tool does, not by shared words. Pick `none` when no listed tool does it." + )), + } +} + +/// A family's members, cut to one choice by the retriever when larger; +/// caller order otherwise. +async fn fit_one_choice<'a>( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + members: &[&'a RankCandidate], +) -> Result, RankError> { + let room = JevRankerConfig::MAX_CANDIDATES; + if members.len() <= room { + return Ok(members.to_vec()); + } + let owned: Vec = members.iter().map(|m| (*m).clone()).collect(); + let hits = ranker + .config() + .retriever + .rank(intent, context, &owned, room) + .await?; + let keep: HashSet<&str> = hits.iter().map(|h| h.key.as_str()).collect(); + let kept: Vec<&RankCandidate> = members + .iter() + .copied() + .filter(|m| keep.contains(m.key.as_str())) + .take(room) + .collect(); + if kept.is_empty() { + return Ok(members.iter().copied().take(room).collect()); + } + Ok(kept) +} + +/// Folds one family's decision into `ranking` as `P(family) · P(member)`; +/// a family whose `none` beats its best member contributes nothing. +fn merge( + ranking: &mut JevRanking, + decision: &JevDecision, + p_family: f64, + members: &[&RankCandidate], + floor: f64, +) { + let none = decision + .probabilities + .get(NONE_OPTION) + .copied() + .unwrap_or(0.0); + ranking.none_probability = ranking.none_probability.max(none); + ranking.choice_confidence = ranking + .choice_confidence + .max(decision.choice_confidence * p_family); + let best = members + .iter() + .filter_map(|m| decision.probabilities.get(&m.key).copied()) + .fold(0.0_f64, f64::max); + if none >= best { + return; + } + for member in members { + let Some(p) = decision.probabilities.get(&member.key).copied() else { + continue; + }; + let joint = p * p_family; + if joint >= floor { + ranking.hits.push(RankHit { + key: member.key.clone(), + score: joint, + confidence: Some(joint), + }); + } + } +} + +/// What a family is, for the first stage: its name, its size, and a sample +/// of member names so a toolkit reads as what it does. +fn family_summary(family: &str, members: &[&RankCandidate]) -> String { + let sample: Vec = members + .iter() + .take(FAMILY_SAMPLE) + .map(|m| m.key.to_ascii_lowercase().replace('_', " ")) + .collect(); + let mut text = format!( + "{family}: {} tool(s), e.g. {}", + members.len(), + sample.join("; ") + ); + if text.chars().count() > 600 { + text = text.chars().take(600).collect::() + "…"; + } + text +} + +/// Awaits every future, in order, without an executor dependency. +async fn join_all(futures: Vec) -> Vec { + let mut pending: Vec>>> = + futures.into_iter().map(|f| Some(Box::pin(f))).collect(); + let mut outputs: Vec> = (0..pending.len()).map(|_| None).collect(); + std::future::poll_fn(|cx: &mut TaskContext<'_>| { + let mut all_done = true; + for (slot, out) in pending.iter_mut().zip(outputs.iter_mut()) { + if let Some(future) = slot.as_mut() { + match future.as_mut().poll(cx) { + Poll::Ready(value) => { + *out = Some(value); + *slot = None; + } + Poll::Pending => all_done = false, + } + } + } + if all_done { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + // Every slot was filled before `poll_fn` resolved; flattening is the + // panic-free way to say so. + outputs.into_iter().flatten().collect() +} diff --git a/crates/tinytools-jev/src/lib.rs b/crates/tinytools-jev/src/lib.rs index 18bb197..8e06229 100644 --- a/crates/tinytools-jev/src/lib.rs +++ b/crates/tinytools-jev/src/lib.rs @@ -3,10 +3,13 @@ //! A host supplies [`JevEvaluator`], retaining ownership of transport, //! authentication, retry, and deadline policy. +mod family; #[cfg(test)] mod test; mod types; -pub use types::{JevDecision, JevEvaluator, JevOption, JevRankerConfig, JevRanking, JevRequest}; +pub use types::{ + JevDecision, JevEvaluator, JevOption, JevRankerConfig, JevRanking, JevRequest, JevStrategy, +}; use std::{ collections::{BTreeMap, BTreeSet}, @@ -14,7 +17,7 @@ use std::{ time::Duration, }; use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; -const NONE_OPTION: &str = "none"; +pub(crate) const NONE_OPTION: &str = "none"; const MAX_SUMMARY_CHARS: usize = 240; /// Ranks tools using a host-provided evaluator. @@ -36,6 +39,11 @@ impl JevRanker { pub fn config(&self) -> &JevRankerConfig { &self.config } + /// The evaluator this ranker decides with. + #[must_use] + pub fn evaluator(&self) -> &Arc { + &self.evaluator + } /// Ranks and returns all decision metadata. /// /// # Errors @@ -54,6 +62,9 @@ impl JevRanker { if candidates.is_empty() || limit == 0 { return Ok(JevRanking::empty()); } + if self.config.strategy == JevStrategy::FamilyThenDecide { + return family::rank(self, intent, context, candidates, limit).await; + } let shortlist = self.shortlist(intent, context, candidates).await?; if shortlist.is_empty() { return Ok(JevRanking::empty()); @@ -119,6 +130,7 @@ impl JevRanker { recent_turns: context.recent_turns.clone(), options, model: self.config.model.clone(), + instructions: None, }) } } @@ -140,20 +152,21 @@ impl ToolRanker for JevRanker { } } impl JevRanking { - fn empty() -> Self { + pub(crate) fn empty() -> Self { Self { hits: vec![], choice_confidence: 0.0, needs_tool: None, none_probability: 0.0, shortlisted: 0, + families: vec![], input_tokens: None, latency: Duration::ZERO, attempts: 0, } } } -fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { +pub(crate) fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { let mut keys = BTreeSet::new(); for c in candidates { if c.key == NONE_OPTION { @@ -165,9 +178,12 @@ fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { } Ok(()) } -fn option_text(candidate: &RankCandidate) -> String { - let mut summary: String = candidate.summary.chars().take(MAX_SUMMARY_CHARS).collect(); - if candidate.summary.chars().count() > MAX_SUMMARY_CHARS { +pub(crate) fn option_text(candidate: &RankCandidate) -> String { + option_text_clipped(candidate, MAX_SUMMARY_CHARS) +} +pub(crate) fn option_text_clipped(candidate: &RankCandidate, max_chars: usize) -> String { + let mut summary: String = candidate.summary.chars().take(max_chars).collect(); + if candidate.summary.chars().count() > max_chars { summary.push('…'); } candidate.family.as_ref().map_or(summary.clone(), |family| { @@ -207,6 +223,7 @@ fn decode(decision: &JevDecision, shortlist: &[&RankCandidate], floor: f64) -> J needs_tool: decision.needs_tool, none_probability: none, shortlisted: shortlist.len(), + families: vec![], input_tokens: decision.input_tokens, latency: Duration::ZERO, attempts: decision.attempts, diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index 5ba6355..e48a096 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -207,3 +207,155 @@ fn configuration_builders_are_observable() { assert!((config.min_probability - 1.0).abs() < f64::EPSILON); assert_eq!(config.model, "jev-pinned"); } + +/// Answers the family stage and each family stage from a script keyed by +/// the request's instructions, so the two-stage flow is observable. +#[derive(Debug)] +struct FamilyEvaluator { + seen: Mutex>, +} +#[async_trait::async_trait] +impl JevEvaluator for FamilyEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + if let Ok(mut seen) = self.seen.lock() { + seen.push(request.clone()); + } + let instructions = request.instructions.clone().unwrap_or_default(); + let probabilities: BTreeMap = if instructions.starts_with("Which group") { + [ + ("slack", 0.7), + ("gmail", 0.2), + ("core", 0.05), + ("none", 0.05), + ] + } else if instructions.contains("`slack`") { + [ + ("SLACK_SEND_MESSAGE", 0.8), + ("SLACK_LIST", 0.1), + ("none", 0.1), + ("_", 0.0), + ] + } else { + [ + ("GMAIL_SEND_EMAIL", 0.6), + ("GMAIL_FETCH", 0.3), + ("none", 0.1), + ("_", 0.0), + ] + } + .into_iter() + .filter(|(k, _)| *k != "_") + .map(|(k, v)| (k.to_owned(), v)) + .collect(); + Ok(JevDecision { + probabilities, + choice_confidence: 0.7, + needs_tool: Some(0.9), + input_tokens: Some(100), + attempts: 1, + }) + } +} + +fn family_catalogue() -> Vec { + vec![ + RankCandidate::new("SLACK_SEND_MESSAGE", "Send a message").with_family("slack"), + RankCandidate::new("SLACK_LIST", "List channels").with_family("slack"), + RankCandidate::new("GMAIL_SEND_EMAIL", "Send an email").with_family("gmail"), + RankCandidate::new("GMAIL_FETCH", "Fetch emails").with_family("gmail"), + RankCandidate::new("file_read", "Read a file"), + ] +} + +#[tokio::test] +async fn family_then_decide_asks_the_family_first_then_each_chosen_family() { + let evaluator = Arc::new(FamilyEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &family_catalogue(), 3) + .await + .ok(); + + let seen: Vec = evaluator + .seen + .lock() + .map(|seen| seen.clone()) + .unwrap_or_default(); + assert_eq!(seen.len(), 3, "one family stage, then slack and gmail"); + assert_eq!( + seen.first().map(|r| r.options.len()), + Some(4), + "slack, gmail, core, none" + ); + assert!(seen.first().is_some_and(|r| { + r.options + .iter() + .any(|o| o.key == "core" && o.description.contains("file read")) + })); + assert!( + seen.get(1) + .and_then(|r| r.instructions.as_deref()) + .is_some_and(|i| i.contains("`slack`")) + ); + assert!( + seen.get(2) + .and_then(|r| r.instructions.as_deref()) + .is_some_and(|i| i.contains("`gmail`")) + ); + + assert_eq!( + ranking.as_ref().map(|r| r.families.clone()), + Some(vec![("slack".to_owned(), 0.7), ("gmail".to_owned(), 0.2)]) + ); + let keys: Option> = ranking + .as_ref() + .map(|r| r.hits.iter().map(|h| h.key.as_str()).collect()); + assert_eq!( + keys, + Some(vec!["SLACK_SEND_MESSAGE", "GMAIL_SEND_EMAIL", "SLACK_LIST"]) + ); + assert!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .is_some_and(|h| (h.score - 0.56).abs() < 1e-9) + ); + assert_eq!(ranking.as_ref().map(|r| r.shortlisted), Some(4)); + assert_eq!(ranking.as_ref().and_then(|r| r.input_tokens), Some(200)); +} + +#[tokio::test] +async fn family_then_decide_with_one_family_skips_the_family_stage() { + let evaluator = Arc::new(FamilyEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await + .ok(); + assert_eq!(evaluator.seen.lock().map_or(0, |s| s.len()), 1); + assert_eq!( + ranking.as_ref().map(|r| r.families.clone()), + Some(vec![("slack".to_owned(), 1.0)]) + ); + assert_eq!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .map(|h| h.key.as_str()), + Some("SLACK_SEND_MESSAGE") + ); +} diff --git a/crates/tinytools-jev/src/types.rs b/crates/tinytools-jev/src/types.rs index bd5b3d3..fe03eeb 100644 --- a/crates/tinytools-jev/src/types.rs +++ b/crates/tinytools-jev/src/types.rs @@ -23,6 +23,28 @@ pub struct JevRequest { pub options: Vec, /// Model identifier configured by the host. pub model: String, + /// What the options are and how to choose among them, when the ranker + /// asks something other than "which tool accomplishes the request" — the + /// family stage of [`JevStrategy::FamilyThenDecide`] asks which *group* + /// of tools applies. `None` is the evaluator's default tool wording. + pub instructions: Option, +} + +/// How [`JevRanker`][crate::JevRanker] narrows a catalogue before deciding. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum JevStrategy { + /// The retriever shortlists `retrieval_k` candidates, one evaluation + /// decides. Bounded by the retriever's recall — a paraphrase it cannot + /// bridge never reaches the evaluator. + #[default] + RetrieveThenDecide, + /// The evaluator first picks the candidates' *family* (a toolkit, a + /// pack), then decides among every member of the top families, one + /// evaluation per family. No retrieval for a family that fits one + /// choice, so a paraphrase is only ever judged semantically; a larger + /// family is cut to [`JevRankerConfig::MAX_CANDIDATES`] by the retriever. + /// Candidates without a family form one `core` family. + FamilyThenDecide, } /// The decision returned by a host-provided evaluator. @@ -53,6 +75,8 @@ pub trait JevEvaluator: Send + Sync + fmt::Debug { /// How [`JevRanker`][crate::JevRanker] retrieves and decides. #[derive(Clone)] pub struct JevRankerConfig { + pub(crate) strategy: JevStrategy, + pub(crate) max_families: usize, pub(crate) retriever: Arc, pub(crate) retrieval_k: usize, pub(crate) min_probability: f64, @@ -68,12 +92,32 @@ impl JevRankerConfig { #[must_use] pub fn new() -> Self { Self { + strategy: JevStrategy::default(), + max_families: 2, retriever: Arc::new(Bm25Ranker), retrieval_k: 20, min_probability: 0.05, model: "jev-latest".into(), } } + /// Sets the narrowing strategy. + #[must_use] + pub fn with_strategy(mut self, value: JevStrategy) -> Self { + self.strategy = value; + self + } + /// The narrowing strategy in force. + #[must_use] + pub fn strategy(&self) -> JevStrategy { + self.strategy + } + /// Sets how many top families the second stage decides among, clamped + /// to `1..=8`. + #[must_use] + pub fn with_max_families(mut self, value: usize) -> Self { + self.max_families = value.clamp(1, 8); + self + } /// Replaces the retriever. #[must_use] pub fn with_retriever(mut self, value: Arc) -> Self { @@ -109,6 +153,8 @@ impl Default for JevRankerConfig { impl fmt::Debug for JevRankerConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("JevRankerConfig") + .field("strategy", &self.strategy) + .field("max_families", &self.max_families) .field("retriever", &self.retriever.kind()) .field("retrieval_k", &self.retrieval_k) .field("min_probability", &self.min_probability) @@ -130,6 +176,9 @@ pub struct JevRanking { pub none_probability: f64, /// Candidate count shown. pub shortlisted: usize, + /// [`JevStrategy::FamilyThenDecide`] only: the families the first stage + /// chose, best first, with their probabilities. + pub families: Vec<(String, f64)>, /// Input tokens billed, when reported. pub input_tokens: Option, /// Evaluator wall time.