From fbd7bd4bbada687d354d26be7c1bb0ba11191417 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 08:04:45 +0530 Subject: [PATCH] feat(jev): FamilyThenDecide strategy over the evaluator seam The evaluator first picks the candidates' family (one small choice over toolkits and packs), then decides among every member of the top families, one evaluation per family run concurrently, so a paraphrase is judged semantically at both steps instead of being lost by a lexical shortlist. A family larger than one choice is cut to fit by the configured retriever. JevRequest gains optional instructions for the family question; JevRanking reports the chosen families. Default strategy unchanged. Co-authored-by: Medulla --- crates/tinytools-jev/README.md | 16 ++ crates/tinytools-jev/src/family.rs | 343 +++++++++++++++++++++++++++++ crates/tinytools-jev/src/lib.rs | 31 ++- crates/tinytools-jev/src/test.rs | 152 +++++++++++++ crates/tinytools-jev/src/types.rs | 49 +++++ 5 files changed, 584 insertions(+), 7 deletions(-) create mode 100644 crates/tinytools-jev/src/family.rs diff --git a/crates/tinytools-jev/README.md b/crates/tinytools-jev/README.md index fda20a6..9c76926 100644 --- a/crates/tinytools-jev/README.md +++ b/crates/tinytools-jev/README.md @@ -46,3 +46,19 @@ Implement `JevEvaluator` in the host by translating `JevRequest` into the client's wire request and translating its answer into `JevDecision`. Pass that implementation to `JevRanker::new`. This keeps transport and runtime choices at the host boundary where their policy belongs. + +## Strategies + +`JevRankerConfig::with_strategy` picks how the catalogue is narrowed before +the evaluator decides: + +- `RetrieveThenDecide` (default): the retriever shortlists `retrieval_k` + candidates, one evaluation decides. Bounded by the retriever's recall. +- `FamilyThenDecide`: one evaluation over the candidates' families (a + toolkit, a pack; candidates without one form `core`), then one evaluation + per chosen family (`max_families`, default 2, run concurrently) over all + its members. No retrieval for a family that fits one choice, so a paraphrase + is judged semantically at both steps; a larger family is cut to + `MAX_CANDIDATES` by the retriever. The family stage sets + `JevRequest::instructions` so the evaluator asks "which group" rather than + "which tool"; `JevRanking::families` reports what it chose. diff --git a/crates/tinytools-jev/src/family.rs b/crates/tinytools-jev/src/family.rs new file mode 100644 index 0000000..142d188 --- /dev/null +++ b/crates/tinytools-jev/src/family.rs @@ -0,0 +1,343 @@ +//! [`JevStrategy::FamilyThenDecide`]: the evaluator picks the family first, +//! then decides among every member of the top families. +//! +//! One evaluation over the families (a toolkit, a pack — a small choice), +//! then one evaluation per chosen family over all its members, run +//! concurrently. No retrieval for a family that fits one choice, so a +//! paraphrase ("ping alex" for `SLACK_SEND_MESSAGE`) is judged semantically +//! at both steps. A family larger than one choice is cut to fit by the +//! configured retriever — the one place recall can still be lost, and the +//! reason a host should give the ranker a semantic retriever. + +use std::collections::{BTreeMap, HashSet}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context as TaskContext, Poll}; + +use tinytools::{RankCandidate, RankContext, RankError, RankHit}; + +use crate::{ + JevDecision, JevOption, JevRanker, JevRankerConfig, JevRanking, JevRequest, JevStrategy, + NONE_OPTION, option_text_clipped, validate_candidates, +}; + +/// Shorter clip for a whole-family choice, which can hold 254 members and +/// has to stay under a provider's per-request token and cost ceilings. +const FAMILY_SUMMARY_CHARS: usize = 150; +/// The family candidates without one are grouped into. +const CORE_FAMILY: &str = "core"; +/// Member names shown per family in the first stage. +const FAMILY_SAMPLE: usize = 12; +/// A decision whose `needs_tool` is below this abstains, as `decode` does. +const NEEDS_TOOL_FLOOR: f64 = 0.5; + +type Families<'a> = BTreeMap<&'a str, Vec<&'a RankCandidate>>; + +/// The strategy's entry point; see the module docs. +pub(crate) async fn rank( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, +) -> Result { + debug_assert_eq!(ranker.config().strategy(), JevStrategy::FamilyThenDecide); + validate_candidates(candidates)?; + let started = std::time::Instant::now(); + let mut families: Families<'_> = BTreeMap::new(); + for candidate in candidates { + families + .entry(candidate.family.as_deref().unwrap_or(CORE_FAMILY)) + .or_default() + .push(candidate); + } + let chosen = match choose_families(ranker, intent, context, &families).await? { + Chosen::Families(chosen) => chosen, + Chosen::Nothing(mut empty) => { + empty.latency = started.elapsed(); + return Ok(empty); + } + }; + + // Second stage: every chosen family at once. + let mut requests: Vec<(String, f64, Vec<&RankCandidate>)> = Vec::new(); + for (family, p_family) in &chosen { + let Some(members) = families.get(family.as_str()) else { + continue; + }; + let members = fit_one_choice(ranker, intent, context, members).await?; + requests.push((family.clone(), *p_family, members)); + } + let decisions = join_all( + requests + .iter() + .map(|(family, _, members)| { + let request = family_request(ranker, intent, context, family, members); + async move { ranker.evaluator().evaluate(&request).await } + }) + .collect(), + ) + .await; + + let mut ranking = JevRanking::empty(); + ranking.families = chosen; + for ((_, p_family, members), decision) in requests.iter().zip(decisions) { + let decision = decision?; + merge( + &mut ranking, + &decision, + *p_family, + members, + ranker.config().min_probability, + ); + ranking.shortlisted += members.len(); + ranking.input_tokens = match (ranking.input_tokens, decision.input_tokens) { + (Some(a), Some(b)) => Some(a + b), + (a, b) => a.or(b), + }; + ranking.attempts = ranking.attempts.max(decision.attempts); + ranking.needs_tool = match (ranking.needs_tool, decision.needs_tool) { + (Some(a), Some(b)) => Some(a.max(b)), + (a, b) => a.or(b), + }; + } + if ranking.needs_tool.is_some_and(|p| p < NEEDS_TOOL_FLOOR) { + ranking.hits.clear(); + } + ranking + .hits + .sort_by(|a, b| b.score.total_cmp(&a.score).then_with(|| a.key.cmp(&b.key))); + ranking.hits.truncate(limit); + ranking.latency = started.elapsed(); + Ok(ranking) +} + +enum Chosen { + Families(Vec<(String, f64)>), + Nothing(JevRanking), +} + +/// First stage: which families could answer, best first, at most +/// `max_families`, each above `min_probability`. +async fn choose_families( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + families: &Families<'_>, +) -> Result { + if families.len() == 1 { + return Ok(Chosen::Families( + families.keys().map(|f| ((*f).to_owned(), 1.0)).collect(), + )); + } + if families.len() > JevRankerConfig::MAX_CANDIDATES { + return Err(RankError::invalid_input("too many families for one choice")); + } + let mut options: Vec = families + .iter() + .map(|(family, members)| { + if *family == NONE_OPTION { + return Err(RankError::invalid_input("family `none` is reserved")); + } + Ok(JevOption { + key: (*family).to_owned(), + description: family_summary(family, members), + }) + }) + .collect::>()?; + options.push(JevOption { + key: NONE_OPTION.into(), + description: "No listed group of tools is relevant to the request.".into(), + }); + let request = JevRequest { + intent: intent.into(), + recent_turns: context.recent_turns.clone(), + options, + model: ranker.config().model.clone(), + instructions: Some( + "Which group of tools would accomplish the user's request? Each option names \ + a service or a category and lists what its tools do. Pick `none` when no \ + group applies." + .into(), + ), + }; + let decision = ranker.evaluator().evaluate(&request).await?; + let none = decision + .probabilities + .get(NONE_OPTION) + .copied() + .unwrap_or(0.0); + let mut ordered: Vec<(String, f64)> = decision + .probabilities + .iter() + .filter(|(name, _)| name.as_str() != NONE_OPTION && families.contains_key(name.as_str())) + .map(|(name, p)| (name.clone(), *p)) + .collect(); + ordered.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + ordered.truncate(ranker.config().max_families); + ordered.retain(|(_, p)| *p >= ranker.config().min_probability); + let abstained = ordered.first().is_none_or(|(_, best)| none >= *best) + || decision.needs_tool.is_some_and(|p| p < NEEDS_TOOL_FLOOR); + if abstained { + let mut empty = JevRanking::empty(); + empty.attempts = decision.attempts; + empty.input_tokens = decision.input_tokens; + empty.none_probability = none; + empty.needs_tool = decision.needs_tool; + empty.choice_confidence = decision.choice_confidence; + return Ok(Chosen::Nothing(empty)); + } + Ok(Chosen::Families(ordered)) +} + +/// The second-stage request for one family: every member (already cut to +/// fit) plus `none`. +fn family_request( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + family: &str, + members: &[&RankCandidate], +) -> JevRequest { + let mut options: Vec = members + .iter() + .map(|m| JevOption { + key: m.key.clone(), + description: option_text_clipped(m, FAMILY_SUMMARY_CHARS), + }) + .collect(); + options.push(JevOption { + key: NONE_OPTION.into(), + description: "No listed tool accomplishes the request.".into(), + }); + JevRequest { + intent: intent.into(), + recent_turns: context.recent_turns.clone(), + options, + model: ranker.config().model.clone(), + instructions: Some(format!( + "Which `{family}` tool accomplishes the user's request? Judge by what each \ + tool does, not by shared words. Pick `none` when no listed tool does it." + )), + } +} + +/// A family's members, cut to one choice by the retriever when larger; +/// caller order otherwise. +async fn fit_one_choice<'a>( + ranker: &JevRanker, + intent: &str, + context: &RankContext, + members: &[&'a RankCandidate], +) -> Result, RankError> { + let room = JevRankerConfig::MAX_CANDIDATES; + if members.len() <= room { + return Ok(members.to_vec()); + } + let owned: Vec = members.iter().map(|m| (*m).clone()).collect(); + let hits = ranker + .config() + .retriever + .rank(intent, context, &owned, room) + .await?; + let keep: HashSet<&str> = hits.iter().map(|h| h.key.as_str()).collect(); + let kept: Vec<&RankCandidate> = members + .iter() + .copied() + .filter(|m| keep.contains(m.key.as_str())) + .take(room) + .collect(); + if kept.is_empty() { + return Ok(members.iter().copied().take(room).collect()); + } + Ok(kept) +} + +/// Folds one family's decision into `ranking` as `P(family) · P(member)`; +/// a family whose `none` beats its best member contributes nothing. +fn merge( + ranking: &mut JevRanking, + decision: &JevDecision, + p_family: f64, + members: &[&RankCandidate], + floor: f64, +) { + let none = decision + .probabilities + .get(NONE_OPTION) + .copied() + .unwrap_or(0.0); + ranking.none_probability = ranking.none_probability.max(none); + ranking.choice_confidence = ranking + .choice_confidence + .max(decision.choice_confidence * p_family); + let best = members + .iter() + .filter_map(|m| decision.probabilities.get(&m.key).copied()) + .fold(0.0_f64, f64::max); + if none >= best { + return; + } + for member in members { + let Some(p) = decision.probabilities.get(&member.key).copied() else { + continue; + }; + let joint = p * p_family; + if joint >= floor { + ranking.hits.push(RankHit { + key: member.key.clone(), + score: joint, + confidence: Some(joint), + }); + } + } +} + +/// What a family is, for the first stage: its name, its size, and a sample +/// of member names so a toolkit reads as what it does. +fn family_summary(family: &str, members: &[&RankCandidate]) -> String { + let sample: Vec = members + .iter() + .take(FAMILY_SAMPLE) + .map(|m| m.key.to_ascii_lowercase().replace('_', " ")) + .collect(); + let mut text = format!( + "{family}: {} tool(s), e.g. {}", + members.len(), + sample.join("; ") + ); + if text.chars().count() > 600 { + text = text.chars().take(600).collect::() + "…"; + } + text +} + +/// Awaits every future, in order, without an executor dependency. +async fn join_all(futures: Vec) -> Vec { + let mut pending: Vec>>> = + futures.into_iter().map(|f| Some(Box::pin(f))).collect(); + let mut outputs: Vec> = (0..pending.len()).map(|_| None).collect(); + std::future::poll_fn(|cx: &mut TaskContext<'_>| { + let mut all_done = true; + for (slot, out) in pending.iter_mut().zip(outputs.iter_mut()) { + if let Some(future) = slot.as_mut() { + match future.as_mut().poll(cx) { + Poll::Ready(value) => { + *out = Some(value); + *slot = None; + } + Poll::Pending => all_done = false, + } + } + } + if all_done { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + // Every slot was filled before `poll_fn` resolved; flattening is the + // panic-free way to say so. + outputs.into_iter().flatten().collect() +} diff --git a/crates/tinytools-jev/src/lib.rs b/crates/tinytools-jev/src/lib.rs index 18bb197..8e06229 100644 --- a/crates/tinytools-jev/src/lib.rs +++ b/crates/tinytools-jev/src/lib.rs @@ -3,10 +3,13 @@ //! A host supplies [`JevEvaluator`], retaining ownership of transport, //! authentication, retry, and deadline policy. +mod family; #[cfg(test)] mod test; mod types; -pub use types::{JevDecision, JevEvaluator, JevOption, JevRankerConfig, JevRanking, JevRequest}; +pub use types::{ + JevDecision, JevEvaluator, JevOption, JevRankerConfig, JevRanking, JevRequest, JevStrategy, +}; use std::{ collections::{BTreeMap, BTreeSet}, @@ -14,7 +17,7 @@ use std::{ time::Duration, }; use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; -const NONE_OPTION: &str = "none"; +pub(crate) const NONE_OPTION: &str = "none"; const MAX_SUMMARY_CHARS: usize = 240; /// Ranks tools using a host-provided evaluator. @@ -36,6 +39,11 @@ impl JevRanker { pub fn config(&self) -> &JevRankerConfig { &self.config } + /// The evaluator this ranker decides with. + #[must_use] + pub fn evaluator(&self) -> &Arc { + &self.evaluator + } /// Ranks and returns all decision metadata. /// /// # Errors @@ -54,6 +62,9 @@ impl JevRanker { if candidates.is_empty() || limit == 0 { return Ok(JevRanking::empty()); } + if self.config.strategy == JevStrategy::FamilyThenDecide { + return family::rank(self, intent, context, candidates, limit).await; + } let shortlist = self.shortlist(intent, context, candidates).await?; if shortlist.is_empty() { return Ok(JevRanking::empty()); @@ -119,6 +130,7 @@ impl JevRanker { recent_turns: context.recent_turns.clone(), options, model: self.config.model.clone(), + instructions: None, }) } } @@ -140,20 +152,21 @@ impl ToolRanker for JevRanker { } } impl JevRanking { - fn empty() -> Self { + pub(crate) fn empty() -> Self { Self { hits: vec![], choice_confidence: 0.0, needs_tool: None, none_probability: 0.0, shortlisted: 0, + families: vec![], input_tokens: None, latency: Duration::ZERO, attempts: 0, } } } -fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { +pub(crate) fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { let mut keys = BTreeSet::new(); for c in candidates { if c.key == NONE_OPTION { @@ -165,9 +178,12 @@ fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { } Ok(()) } -fn option_text(candidate: &RankCandidate) -> String { - let mut summary: String = candidate.summary.chars().take(MAX_SUMMARY_CHARS).collect(); - if candidate.summary.chars().count() > MAX_SUMMARY_CHARS { +pub(crate) fn option_text(candidate: &RankCandidate) -> String { + option_text_clipped(candidate, MAX_SUMMARY_CHARS) +} +pub(crate) fn option_text_clipped(candidate: &RankCandidate, max_chars: usize) -> String { + let mut summary: String = candidate.summary.chars().take(max_chars).collect(); + if candidate.summary.chars().count() > max_chars { summary.push('…'); } candidate.family.as_ref().map_or(summary.clone(), |family| { @@ -207,6 +223,7 @@ fn decode(decision: &JevDecision, shortlist: &[&RankCandidate], floor: f64) -> J needs_tool: decision.needs_tool, none_probability: none, shortlisted: shortlist.len(), + families: vec![], input_tokens: decision.input_tokens, latency: Duration::ZERO, attempts: decision.attempts, diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index 5ba6355..e48a096 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -207,3 +207,155 @@ fn configuration_builders_are_observable() { assert!((config.min_probability - 1.0).abs() < f64::EPSILON); assert_eq!(config.model, "jev-pinned"); } + +/// Answers the family stage and each family stage from a script keyed by +/// the request's instructions, so the two-stage flow is observable. +#[derive(Debug)] +struct FamilyEvaluator { + seen: Mutex>, +} +#[async_trait::async_trait] +impl JevEvaluator for FamilyEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + if let Ok(mut seen) = self.seen.lock() { + seen.push(request.clone()); + } + let instructions = request.instructions.clone().unwrap_or_default(); + let probabilities: BTreeMap = if instructions.starts_with("Which group") { + [ + ("slack", 0.7), + ("gmail", 0.2), + ("core", 0.05), + ("none", 0.05), + ] + } else if instructions.contains("`slack`") { + [ + ("SLACK_SEND_MESSAGE", 0.8), + ("SLACK_LIST", 0.1), + ("none", 0.1), + ("_", 0.0), + ] + } else { + [ + ("GMAIL_SEND_EMAIL", 0.6), + ("GMAIL_FETCH", 0.3), + ("none", 0.1), + ("_", 0.0), + ] + } + .into_iter() + .filter(|(k, _)| *k != "_") + .map(|(k, v)| (k.to_owned(), v)) + .collect(); + Ok(JevDecision { + probabilities, + choice_confidence: 0.7, + needs_tool: Some(0.9), + input_tokens: Some(100), + attempts: 1, + }) + } +} + +fn family_catalogue() -> Vec { + vec![ + RankCandidate::new("SLACK_SEND_MESSAGE", "Send a message").with_family("slack"), + RankCandidate::new("SLACK_LIST", "List channels").with_family("slack"), + RankCandidate::new("GMAIL_SEND_EMAIL", "Send an email").with_family("gmail"), + RankCandidate::new("GMAIL_FETCH", "Fetch emails").with_family("gmail"), + RankCandidate::new("file_read", "Read a file"), + ] +} + +#[tokio::test] +async fn family_then_decide_asks_the_family_first_then_each_chosen_family() { + let evaluator = Arc::new(FamilyEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &family_catalogue(), 3) + .await + .ok(); + + let seen: Vec = evaluator + .seen + .lock() + .map(|seen| seen.clone()) + .unwrap_or_default(); + assert_eq!(seen.len(), 3, "one family stage, then slack and gmail"); + assert_eq!( + seen.first().map(|r| r.options.len()), + Some(4), + "slack, gmail, core, none" + ); + assert!(seen.first().is_some_and(|r| { + r.options + .iter() + .any(|o| o.key == "core" && o.description.contains("file read")) + })); + assert!( + seen.get(1) + .and_then(|r| r.instructions.as_deref()) + .is_some_and(|i| i.contains("`slack`")) + ); + assert!( + seen.get(2) + .and_then(|r| r.instructions.as_deref()) + .is_some_and(|i| i.contains("`gmail`")) + ); + + assert_eq!( + ranking.as_ref().map(|r| r.families.clone()), + Some(vec![("slack".to_owned(), 0.7), ("gmail".to_owned(), 0.2)]) + ); + let keys: Option> = ranking + .as_ref() + .map(|r| r.hits.iter().map(|h| h.key.as_str()).collect()); + assert_eq!( + keys, + Some(vec!["SLACK_SEND_MESSAGE", "GMAIL_SEND_EMAIL", "SLACK_LIST"]) + ); + assert!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .is_some_and(|h| (h.score - 0.56).abs() < 1e-9) + ); + assert_eq!(ranking.as_ref().map(|r| r.shortlisted), Some(4)); + assert_eq!(ranking.as_ref().and_then(|r| r.input_tokens), Some(200)); +} + +#[tokio::test] +async fn family_then_decide_with_one_family_skips_the_family_stage() { + let evaluator = Arc::new(FamilyEvaluator { + seen: Mutex::new(vec![]), + }); + let ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_strategy(JevStrategy::FamilyThenDecide), + ); + let only_slack: Vec = family_catalogue() + .into_iter() + .filter(|c| c.family.as_deref() == Some("slack")) + .collect(); + let ranking = ranker + .rank_detailed("ping alex", &RankContext::empty(), &only_slack, 3) + .await + .ok(); + assert_eq!(evaluator.seen.lock().map_or(0, |s| s.len()), 1); + assert_eq!( + ranking.as_ref().map(|r| r.families.clone()), + Some(vec![("slack".to_owned(), 1.0)]) + ); + assert_eq!( + ranking + .as_ref() + .and_then(|r| r.hits.first()) + .map(|h| h.key.as_str()), + Some("SLACK_SEND_MESSAGE") + ); +} diff --git a/crates/tinytools-jev/src/types.rs b/crates/tinytools-jev/src/types.rs index bd5b3d3..fe03eeb 100644 --- a/crates/tinytools-jev/src/types.rs +++ b/crates/tinytools-jev/src/types.rs @@ -23,6 +23,28 @@ pub struct JevRequest { pub options: Vec, /// Model identifier configured by the host. pub model: String, + /// What the options are and how to choose among them, when the ranker + /// asks something other than "which tool accomplishes the request" — the + /// family stage of [`JevStrategy::FamilyThenDecide`] asks which *group* + /// of tools applies. `None` is the evaluator's default tool wording. + pub instructions: Option, +} + +/// How [`JevRanker`][crate::JevRanker] narrows a catalogue before deciding. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum JevStrategy { + /// The retriever shortlists `retrieval_k` candidates, one evaluation + /// decides. Bounded by the retriever's recall — a paraphrase it cannot + /// bridge never reaches the evaluator. + #[default] + RetrieveThenDecide, + /// The evaluator first picks the candidates' *family* (a toolkit, a + /// pack), then decides among every member of the top families, one + /// evaluation per family. No retrieval for a family that fits one + /// choice, so a paraphrase is only ever judged semantically; a larger + /// family is cut to [`JevRankerConfig::MAX_CANDIDATES`] by the retriever. + /// Candidates without a family form one `core` family. + FamilyThenDecide, } /// The decision returned by a host-provided evaluator. @@ -53,6 +75,8 @@ pub trait JevEvaluator: Send + Sync + fmt::Debug { /// How [`JevRanker`][crate::JevRanker] retrieves and decides. #[derive(Clone)] pub struct JevRankerConfig { + pub(crate) strategy: JevStrategy, + pub(crate) max_families: usize, pub(crate) retriever: Arc, pub(crate) retrieval_k: usize, pub(crate) min_probability: f64, @@ -68,12 +92,32 @@ impl JevRankerConfig { #[must_use] pub fn new() -> Self { Self { + strategy: JevStrategy::default(), + max_families: 2, retriever: Arc::new(Bm25Ranker), retrieval_k: 20, min_probability: 0.05, model: "jev-latest".into(), } } + /// Sets the narrowing strategy. + #[must_use] + pub fn with_strategy(mut self, value: JevStrategy) -> Self { + self.strategy = value; + self + } + /// The narrowing strategy in force. + #[must_use] + pub fn strategy(&self) -> JevStrategy { + self.strategy + } + /// Sets how many top families the second stage decides among, clamped + /// to `1..=8`. + #[must_use] + pub fn with_max_families(mut self, value: usize) -> Self { + self.max_families = value.clamp(1, 8); + self + } /// Replaces the retriever. #[must_use] pub fn with_retriever(mut self, value: Arc) -> Self { @@ -109,6 +153,8 @@ impl Default for JevRankerConfig { impl fmt::Debug for JevRankerConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("JevRankerConfig") + .field("strategy", &self.strategy) + .field("max_families", &self.max_families) .field("retriever", &self.retriever.kind()) .field("retrieval_k", &self.retrieval_k) .field("min_probability", &self.min_probability) @@ -130,6 +176,9 @@ pub struct JevRanking { pub none_probability: f64, /// Candidate count shown. pub shortlisted: usize, + /// [`JevStrategy::FamilyThenDecide`] only: the families the first stage + /// chose, best first, with their probabilities. + pub families: Vec<(String, f64)>, /// Input tokens billed, when reported. pub input_tokens: Option, /// Evaluator wall time.