From a512a77aa9a6137c3b8402b9e8b8e5f21a2b100d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:20:52 +0530 Subject: [PATCH 1/7] feat(rank): ToolRanker vocabulary, Bm25Ranker, and Tool::family Add the rank module: the ToolRanker trait a harness asks through without knowing which kind of ranker answers, the RankCandidate/RankHit/RankContext vocabulary, and Bm25Ranker built on the BM25 index moved here from the tinyagents harness's discovery module so both crates rank with one arithmetic. Tool gains a defaulted family() declaration so a search index can say which pack, toolkit or server a hit came from. Co-authored-by: Medulla --- README.md | 6 +- crates/tinytools/src/lib.rs | 4 + crates/tinytools/src/rank/README.md | 40 ++++ crates/tinytools/src/rank/bm25.rs | 283 ++++++++++++++++++++++++++++ crates/tinytools/src/rank/mod.rs | 81 ++++++++ crates/tinytools/src/rank/test.rs | 114 +++++++++++ crates/tinytools/src/rank/types.rs | 128 +++++++++++++ crates/tinytools/src/tool/types.rs | 11 ++ 8 files changed, 666 insertions(+), 1 deletion(-) create mode 100644 crates/tinytools/src/rank/README.md create mode 100644 crates/tinytools/src/rank/bm25.rs create mode 100644 crates/tinytools/src/rank/mod.rs create mode 100644 crates/tinytools/src/rank/test.rs create mode 100644 crates/tinytools/src/rank/types.rs diff --git a/README.md b/README.md index 866171b..408d764 100644 --- a/README.md +++ b/README.md @@ -65,11 +65,15 @@ compiles neither the harness nor the host. | `context` | `ToolRunContext` — the narrow seam onto a live run | | `workspace` | `WorkspaceDescriptor`, `SandboxMode` — the root a tool may touch, and how strictly it is sandboxed | | `naming` | `humanize_tool_name`, `context_detail_from_args` — rendering a call for a human | +| `rank` | `ToolRanker`, `RankCandidate`, `RankHit`, `Bm25Ranker` — ranking a catalogue of tools against an intent, with the lexical ranker built in | The workspace also contains `tinytools-agent`, a separate crate for model-facing tool-call parsing, dialects, catalogue/result rendering, and transcript replay. It builds on `ToolSpec` without adding agent-loop or provider -dependencies to the base `tinytools` vocabulary crate. +dependencies to the base `tinytools` vocabulary crate. `tinytools-jev` is a +second sibling: a `ToolRanker` backed by TypeSafe's Jev decision model through +`tinyjevclient`, kept out of the vocabulary crate because it carries an HTTP +transport. ## What is deliberately not here diff --git a/crates/tinytools/src/lib.rs b/crates/tinytools/src/lib.rs index 22e1d5a..f9fb058 100644 --- a/crates/tinytools/src/lib.rs +++ b/crates/tinytools/src/lib.rs @@ -32,6 +32,8 @@ //! - [`context`] — [`ToolRunContext`], the narrow seam onto a live run. //! - [`workspace`] — [`WorkspaceDescriptor`], the root a tool may touch. //! - [`naming`] — rendering a call for a human. +//! - [`rank`] — [`ToolRanker`], ranking a catalogue of tools against an +//! intent, and the lexical [`Bm25Ranker`] every host gets for free. //! //! # What is deliberately not here //! @@ -104,6 +106,7 @@ pub mod context; pub mod naming; pub mod permission; pub mod policy; +pub mod rank; pub mod result; pub mod spec; pub mod tool; @@ -124,6 +127,7 @@ pub use permission::PermissionLevel; pub use policy::{ ToolAccess, ToolDisplay, ToolPolicy, ToolReplay, ToolRuntime, ToolSideEffects, WorkspaceAccess, }; +pub use rank::{Bm25Ranker, RankCandidate, RankContext, RankError, RankHit, ToolRanker}; pub use result::{FileData, ImageData, ToolContent, ToolControl, ToolErrorKind, ToolResult}; pub use spec::ToolSpec; pub use tool::{Tool, ToolExposure}; diff --git a/crates/tinytools/src/rank/README.md b/crates/tinytools/src/rank/README.md new file mode 100644 index 0000000..3fa8e7f --- /dev/null +++ b/crates/tinytools/src/rank/README.md @@ -0,0 +1,40 @@ +# `rank` + +Ranking a catalogue of tools against an intent: the `ToolRanker` trait, the +`RankCandidate` / `RankHit` / `RankContext` vocabulary, and `Bm25Ranker`, the +lexical implementation every host gets without a network. + +## Design + +A host that registers more tools than a model should see on every request +advertises a few and lets the model search for the rest. That search is a +ranking problem, and the interesting rankers (a decision model such as +TypeSafe's Jev, an embedding index) talk to a service this crate must never +depend on. So the crate owns the *question* — here are candidates, here is an +intent, order them — and one free answer, BM25. A harness asks through the +trait and never learns which kind answered; a host composes them, typically +BM25 to retrieve a shortlist and a model to decide (see `tinytools-jev`). + +`Bm25Index` and `tokenize` moved here from the `tinyagents` harness's +discovery module so both crates rank with one arithmetic. + +## Contract + +- `rank` returns at most `limit` hits, best first, and only candidates it + considers relevant. Empty means "nothing fits", never "padded to `limit`". +- Every returned key names a candidate the caller passed. +- Failure is a `RankError`; the caller falls back. Never a panic on input. +- `RankHit::confidence` is a calibrated probability or `None`. BM25 returns + `None`: a BM25 score is not a probability, and a caller gating on + confidence must treat `None` as unknown rather than zero. +- `RankContext` is deliberately small. A model-backed ranker pays for every + byte on every search, and unrelated context is a distractor. + +## Files + +| File | Owns | +|------------|-------------------------------------------------------------| +| `mod.rs` | `ToolRanker`, the `Arc` blanket impl, re-exports | +| `types.rs` | `RankCandidate`, `RankHit`, `RankContext`, `RankError` | +| `bm25.rs` | `tokenize`, `Bm25Index`, `Bm25Ranker` | +| `test.rs` | Unit tests | diff --git a/crates/tinytools/src/rank/bm25.rs b/crates/tinytools/src/rank/bm25.rs new file mode 100644 index 0000000..4fd4441 --- /dev/null +++ b/crates/tinytools/src/rank/bm25.rs @@ -0,0 +1,283 @@ +//! BM25 ranking over short documents, and the [`ToolRanker`] built on it. +//! +//! Takes `(sort_key, text)` pairs and returns ranked indices, so the +//! tool-specific half (what text is searchable, what a hit looks like) stays +//! with the caller. Hand-rolled rather than the `bm25` crate: the arithmetic +//! is ~80 lines, and this crate is the dependency floor of every tool author. +//! +//! Moved here from the `tinyagents` harness's discovery module so a host can +//! rank with the same arithmetic the harness uses, and so a model-backed +//! ranker can retrieve a shortlist with it before deciding. + +use std::collections::HashMap; + +use super::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; + +/// Words that carry no capability meaning, dropped from a query before ranking. +/// +/// A document-frequency threshold alone cannot do this on a small corpus: +/// with three deferred tools, "a" may appear in exactly one description and +/// so rank as the *most* distinguishing term in "send a calendar invite". +/// Deliberately short and English-only — it can only remove terms, so a +/// description in another language ranks exactly as it would without it. +/// Words that could name a capability ("up", as in "look up") are left in. +const STOPWORDS: &[&str] = &[ + "a", "an", "and", "are", "as", "at", "be", "but", "by", "can", "do", "for", "from", "how", "i", + "if", "in", "into", "is", "it", "its", "me", "my", "of", "on", "or", "our", "so", "that", + "the", "their", "them", "then", "there", "these", "they", "this", "to", "was", "we", "were", + "what", "when", "which", "who", "will", "with", "would", "you", "your", +]; + +/// BM25 term-frequency saturation (the standard default). +const K1: f64 = 1.2; +/// BM25 length normalisation (the standard default). +const B: f64 = 0.75; + +/// Splits text into search terms. +/// +/// Splits on non-alphanumerics **and** on a lower→upper transition, so +/// `memory_hybrid_search` and `readWorkflowResource` both yield the words a +/// person would type. Without the camel-case rule a query for "workflow" +/// misses a tool whose only mention of it is inside an identifier. +#[must_use] +pub fn tokenize(text: &str) -> Vec { + let mut out = Vec::new(); + let mut current = String::new(); + let mut previous_lower = false; + for ch in text.chars() { + if ch.is_alphanumeric() { + if ch.is_uppercase() && previous_lower && !current.is_empty() { + out.push(std::mem::take(&mut current)); + } + current.extend(ch.to_lowercase()); + previous_lower = ch.is_lowercase() || ch.is_numeric(); + } else if !current.is_empty() { + out.push(std::mem::take(&mut current)); + previous_lower = false; + } + } + if !current.is_empty() { + out.push(current); + } + out +} + +/// A ranked corpus of short documents, identified by their index into the +/// slice the caller built the index from. +#[derive(Default, Debug, Clone)] +pub struct Bm25Index { + documents: Vec, + document_frequency: HashMap, + average_length: f64, +} + +#[derive(Debug, Clone)] +struct Document { + /// Used only to break score ties deterministically. + sort_key: String, + tokens: Vec, +} + +impl Bm25Index { + /// Builds from `(sort_key, searchable_text)` pairs, in caller order. + /// + /// `sort_key` breaks ties; make it the id the caller would print, so two + /// identical queries produce identical output. An unstable order would make + /// a model's transcript non-reproducible for no benefit. + pub fn build<'a>(documents: impl IntoIterator) -> Self { + let documents: Vec = documents + .into_iter() + .map(|(sort_key, text)| Document { + sort_key: sort_key.to_string(), + tokens: tokenize(text), + }) + .collect(); + + let mut document_frequency: HashMap = HashMap::new(); + for doc in &documents { + let mut seen: Vec<&str> = Vec::new(); + for token in &doc.tokens { + if !seen.contains(&token.as_str()) { + seen.push(token); + *document_frequency.entry(token.clone()).or_insert(0) += 1; + } + } + } + + let total: usize = documents.iter().map(|d| d.tokens.len()).sum(); + #[allow(clippy::cast_precision_loss)] + let average_length = if documents.is_empty() { + 0.0 + } else { + total as f64 / documents.len() as f64 + }; + + Self { + documents, + document_frequency, + average_length, + } + } + + /// `true` when the corpus holds no documents. + #[must_use] + pub fn is_empty(&self) -> bool { + self.documents.is_empty() + } + + /// Number of documents in the corpus. + #[must_use] + pub fn len(&self) -> usize { + self.documents.len() + } + + /// Ranks against `query`, best first, returning **indices** into the corpus. + /// + /// Only documents scoring above zero are returned. Padding the list out to + /// `limit` with unrelated entries would spend exactly the tokens deferral + /// exists to save, and would invite the model to call something unrelated + /// to what it asked for. + #[must_use] + pub fn search(&self, query: &str, limit: usize) -> Vec { + self.search_scored(query, limit) + .into_iter() + .map(|(_, index)| index) + .collect() + } + + /// [`Self::search`], keeping each hit's score. + #[must_use] + pub fn search_scored(&self, query: &str, limit: usize) -> Vec<(f64, usize)> { + let terms = self.significant(tokenize(query)); + if terms.is_empty() || self.documents.is_empty() { + return Vec::new(); + } + let mut scored: Vec<(f64, usize)> = self + .documents + .iter() + .enumerate() + .map(|(index, doc)| (self.score(doc, &terms), index)) + .filter(|(score, _)| *score > 0.0) + .collect(); + + scored.sort_by(|a, b| { + b.0.partial_cmp(&a.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + self.documents[a.1] + .sort_key + .cmp(&self.documents[b.1].sort_key) + }) + }); + scored.truncate(limit); + scored + } + + /// Drops query terms too common in this corpus to mean anything. + /// + /// The IDF below carries the standard `+ 1`, which keeps a term present in + /// every document at a small **positive** weight — without it a corpus of + /// one document scores every term at zero and nothing is ever findable. + /// The cost is that "a" and "the" score, so a document-frequency filter + /// runs first: drop the term when `df >= max(2, ceil(0.8 * n))`. The floor + /// of 2 makes it inert on a one-document corpus; [`STOPWORDS`] covers the + /// small-corpus case the threshold cannot. + fn significant(&self, terms: Vec) -> Vec { + let n = self.documents.len(); + if n == 0 { + return Vec::new(); + } + #[allow( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss + )] + let threshold = std::cmp::max(2, (0.8 * n as f64).ceil() as usize); + terms + .into_iter() + .filter(|term| !STOPWORDS.contains(&term.as_str())) + .filter(|term| *self.document_frequency.get(term).unwrap_or(&0) < threshold) + .collect() + } + + #[allow(clippy::cast_precision_loss)] + fn score(&self, doc: &Document, terms: &[String]) -> f64 { + let length = doc.tokens.len() as f64; + let count = self.documents.len() as f64; + terms + .iter() + .map(|term| { + let frequency = doc.tokens.iter().filter(|t| *t == term).count() as f64; + if frequency == 0.0 { + return 0.0; + } + let df = *self.document_frequency.get(term).unwrap_or(&0) as f64; + // Standard BM25 IDF with the +1 that keeps a term present in + // every document at a small positive weight (see `significant`). + let idf = ((count - df + 0.5) / (df + 0.5) + 1.0).ln(); + let normalised = frequency * (K1 + 1.0) + / (frequency + K1 * (1.0 - B + B * length / self.average_length.max(1.0))); + idf * normalised + }) + .sum() + } +} + +/// The lexical [`ToolRanker`]: BM25 over each candidate's summary and family. +/// +/// Builds a fresh [`Bm25Index`] per call. A catalogue of a few hundred short +/// summaries indexes in microseconds, and a per-call index means the ranker +/// holds no state to invalidate when the caller's catalogue changes. Returns +/// no [`RankHit::confidence`]: a BM25 score is not a probability. +#[derive(Debug, Default, Clone, Copy)] +pub struct Bm25Ranker; + +impl Bm25Ranker { + /// The stable [`ToolRanker::kind`] of this ranker. + pub const KIND: &'static str = "bm25"; + + /// Ranks synchronously; [`ToolRanker::rank`] delegates here. + #[must_use] + pub fn rank_sync(candidates: &[RankCandidate], intent: &str, limit: usize) -> Vec { + let texts: Vec = candidates + .iter() + .map(|candidate| match &candidate.family { + Some(family) => format!("{} {}", candidate.summary, family), + None => candidate.summary.clone(), + }) + .collect(); + let index = Bm25Index::build( + candidates + .iter() + .zip(&texts) + .map(|(candidate, text)| (candidate.key.as_str(), text.as_str())), + ); + index + .search_scored(intent, limit) + .into_iter() + .map(|(score, i)| RankHit::new(candidates[i].key.clone(), score)) + .collect() + } +} + +#[async_trait::async_trait] +impl ToolRanker for Bm25Ranker { + fn kind(&self) -> &'static str { + Self::KIND + } + + async fn rank( + &self, + intent: &str, + _context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError> { + if intent.trim().is_empty() { + return Err(RankError::InvalidInput { + reason: "intent is empty".to_owned(), + }); + } + Ok(Self::rank_sync(candidates, intent, limit)) + } +} diff --git a/crates/tinytools/src/rank/mod.rs b/crates/tinytools/src/rank/mod.rs new file mode 100644 index 0000000..7b82213 --- /dev/null +++ b/crates/tinytools/src/rank/mod.rs @@ -0,0 +1,81 @@ +//! Ranking a catalogue of tools against an intent. +//! +//! A host that registers more tools than a model should see on every request +//! advertises a few and lets the model *search* for the rest. The search is a +//! ranking problem — "which of these tools is the one for *send Alex a note +//! that I'm late*?" — and this module is the vocabulary for it: the +//! [`ToolRanker`] trait, the [`RankCandidate`] a ranker reads, the +//! [`RankHit`] it returns, and one implementation every host can use without +//! a network, [`Bm25Ranker`]. +//! +//! # Why a trait +//! +//! Lexical ranking is free and answers most queries whose wording overlaps a +//! tool's description. It misses the paraphrase — "ping" for a tool described +//! as "send a message" — and it has no notion of confidence, so a host cannot +//! tell a strong match from the least-bad one. A decision model (`TypeSafe`'s +//! Jev, or an embedding index) answers both, at the cost of a network call the +//! vocabulary crate must not make. The trait lets a harness ask "rank these" +//! without knowing which kind is answering, and lets a host compose them: +//! retrieve a shortlist lexically, then let the model decide. +//! +//! # Contract +//! +//! - `rank` returns at most `limit` hits, best first, and only candidates it +//! considers relevant: an empty result means "nothing here fits", never +//! "I padded to `limit`". +//! - Every returned key names a candidate the caller passed. A ranker never +//! invents one. +//! - A ranker that cannot answer returns [`RankError`]; the caller decides +//! what to fall back to. It never panics on caller input. +//! - `rank` is `async` because the interesting implementations talk to a +//! service; [`Bm25Ranker`] completes without yielding. + +mod bm25; +#[cfg(test)] +mod test; +mod types; + +pub use bm25::{Bm25Index, Bm25Ranker, tokenize}; +pub use types::{RankCandidate, RankContext, RankError, RankHit}; + +/// Ranks tool candidates against an intent. +/// +/// See the [module docs](self) for the contract every implementation keeps. +#[async_trait::async_trait] +pub trait ToolRanker: Send + Sync { + /// A short stable name for logs and telemetry: `"bm25"`, `"jev"`. + fn kind(&self) -> &'static str; + + /// Ranks `candidates` against `intent`, best first, at most `limit` hits. + /// + /// # Errors + /// + /// Returns [`RankError`] when the ranker cannot produce an answer. A + /// caller should treat every variant as "fall back", and read the variant + /// only to say why in a log line. + async fn rank( + &self, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError>; +} + +#[async_trait::async_trait] +impl ToolRanker for std::sync::Arc { + fn kind(&self) -> &'static str { + (**self).kind() + } + + async fn rank( + &self, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError> { + (**self).rank(intent, context, candidates, limit).await + } +} diff --git a/crates/tinytools/src/rank/test.rs b/crates/tinytools/src/rank/test.rs new file mode 100644 index 0000000..3c81149 --- /dev/null +++ b/crates/tinytools/src/rank/test.rs @@ -0,0 +1,114 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::*; + +fn candidates() -> Vec { + vec![ + RankCandidate::new( + "calendar_invite", + "calendar_invite Send a calendar invite to attendees. title start attendees", + ) + .with_family("calendar"), + RankCandidate::new( + "pdf_read", + "pdf_read Read the text of a PDF document. path pages", + ) + .with_family("documents"), + RankCandidate::new( + "stock_quote", + "stock_quote Fetch the latest price for a ticker symbol. symbol", + ) + .with_family("finance"), + ] +} + +#[test] +fn tokenize_splits_identifiers_and_camel_case() { + assert_eq!( + tokenize("memory_hybrid_search readWorkflowResource v2"), + vec![ + "memory", "hybrid", "search", "read", "workflow", "resource", "v2" + ] + ); +} + +#[test] +fn index_ranks_by_description_and_breaks_ties_by_key() { + let index = Bm25Index::build([ + ("b_tool", "send a message"), + ("a_tool", "send a message"), + ("c_tool", "read a file"), + ]); + assert_eq!(index.len(), 3); + assert_eq!(index.search("send message", 5), vec![1, 0]); +} + +#[test] +fn index_on_one_document_corpus_still_finds_it() { + let index = Bm25Index::build([("only", "fetch the latest price for a ticker symbol")]); + assert_eq!(index.search("ticker price", 5), vec![0]); + assert!(index.search("calendar", 5).is_empty()); +} + +#[test] +fn index_returns_nothing_for_stopword_only_queries() { + let index = Bm25Index::build([("a", "send a message"), ("b", "read a file")]); + assert!(index.search("the a of", 5).is_empty()); + assert!(Bm25Index::default().search("anything", 5).is_empty()); + assert!(Bm25Index::default().is_empty()); +} + +#[test] +fn bm25_ranker_returns_positive_hits_only_best_first() { + let hits = Bm25Ranker::rank_sync(&candidates(), "read the text of a pdf", 5); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].key, "pdf_read"); + assert!(hits[0].score > 0.0); + assert!(hits[0].confidence.is_none()); +} + +#[test] +fn bm25_ranker_matches_on_family() { + let hits = Bm25Ranker::rank_sync(&candidates(), "finance", 5); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].key, "stock_quote"); +} + +#[test] +fn bm25_ranker_honours_limit() { + let hits = Bm25Ranker::rank_sync(&candidates(), "send read fetch", 2); + assert_eq!(hits.len(), 2); +} + +#[tokio::test] +async fn bm25_ranker_rejects_empty_intent() { + let err = Bm25Ranker + .rank(" ", &RankContext::empty(), &candidates(), 3) + .await + .err() + .map(|e| e.to_string()); + assert_eq!( + err.as_deref(), + Some("invalid ranking input: intent is empty") + ); +} + +#[tokio::test] +async fn ranker_is_object_safe_behind_an_arc() { + let ranker: std::sync::Arc = std::sync::Arc::new(Bm25Ranker); + assert_eq!(ranker.kind(), "bm25"); + let hits = ranker + .rank("calendar invite", &RankContext::empty(), &candidates(), 3) + .await + .unwrap(); + assert_eq!(hits[0].key, "calendar_invite"); +} + +#[test] +fn rank_error_displays_without_credentials() { + let err = RankError::Backend { + reason: "status 401".to_owned(), + }; + assert_eq!(err.to_string(), "ranker backend failed: status 401"); + assert_eq!(RankError::Timeout.to_string(), "ranker timed out"); +} diff --git a/crates/tinytools/src/rank/types.rs b/crates/tinytools/src/rank/types.rs new file mode 100644 index 0000000..44382d2 --- /dev/null +++ b/crates/tinytools/src/rank/types.rs @@ -0,0 +1,128 @@ +//! The vocabulary of tool ranking: what a ranker is given and what it returns. + +use std::fmt; + +/// One tool as a ranker sees it: an opaque key the caller maps back to the +/// tool, an optional family (the pack, toolkit, or server it belongs to), and +/// a short summary — the text a ranker reads. +/// +/// The summary is what the ranker judges *on*. Callers should keep it to the +/// tool's name, its first sentence or two, and its argument names: a decision +/// model degrades as its input fills with content unrelated to the choice. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RankCandidate { + /// Caller-owned identifier, returned verbatim in [`RankHit::key`]. Must be + /// unique within one `rank` call. + pub key: String, + /// The group this tool belongs to, when the caller has one — a toolpack, + /// a connector toolkit, an MCP server. Used as ranking text and echoed + /// back so a caller can render it; a ranker never requires it. + pub family: Option, + /// Name plus a one-line description plus argument names: the searchable + /// text. + pub summary: String, +} + +impl RankCandidate { + /// A candidate with no family. + #[must_use] + pub fn new(key: impl Into, summary: impl Into) -> Self { + Self { + key: key.into(), + family: None, + summary: summary.into(), + } + } + + /// Sets the family. + #[must_use] + pub fn with_family(mut self, family: impl Into) -> Self { + self.family = Some(family.into()); + self + } +} + +/// One ranked tool. +#[derive(Clone, Debug, PartialEq)] +pub struct RankHit { + /// The [`RankCandidate::key`] this hit refers to. + pub key: String, + /// Ranker-specific relevance, higher is better. Comparable only within one + /// `rank` call and one ranker: BM25 scores and calibrated probabilities do + /// not share a scale. + pub score: f64, + /// A calibrated probability that this is the right tool, in `0.0..=1.0`, + /// when the ranker produces one. Lexical rankers return `None`; a caller + /// gating on confidence treats `None` as "unknown", not as zero. + pub confidence: Option, +} + +impl RankHit { + /// A hit with no confidence estimate. + #[must_use] + pub fn new(key: impl Into, score: f64) -> Self { + Self { + key: key.into(), + score, + confidence: None, + } + } +} + +/// Context a ranker may use beyond the intent itself. +/// +/// Deliberately small. Rankers that consult a decision model pay for every +/// byte of this on every search, and unrelated context is a distractor, not +/// a help. The intent is always passed separately and is always what the +/// candidates are judged against. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RankContext { + /// The most recent user turns, oldest first, each already clipped by the + /// caller. Lets a ranker resolve "do it again for Bob" against the turn + /// before. Empty when the caller has nothing worth adding. + pub recent_turns: Vec, +} + +impl RankContext { + /// A context with nothing beyond the intent. + #[must_use] + pub fn empty() -> Self { + Self::default() + } +} + +/// Why a ranker could not rank. +/// +/// A caller treats every variant the same way — fall back to a cheaper ranker +/// or to the candidates' declared order — so the variants exist for logs, not +/// for control flow. +#[derive(Debug)] +#[non_exhaustive] +pub enum RankError { + /// The ranker's backing service refused or failed the request. The + /// message never carries a credential. + Backend { + /// Stable, log-safe description. + reason: String, + }, + /// The request could not be built: too many candidates for the ranker, + /// an empty intent, a duplicate key. + InvalidInput { + /// Stable description of the rejected input. + reason: String, + }, + /// The ranker did not answer within its deadline. + Timeout, +} + +impl fmt::Display for RankError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Backend { reason } => write!(f, "ranker backend failed: {reason}"), + Self::InvalidInput { reason } => write!(f, "invalid ranking input: {reason}"), + Self::Timeout => f.write_str("ranker timed out"), + } + } +} + +impl std::error::Error for RankError {} diff --git a/crates/tinytools/src/tool/types.rs b/crates/tinytools/src/tool/types.rs index 9185fc8..cce2f4f 100644 --- a/crates/tinytools/src/tool/types.rs +++ b/crates/tinytools/src/tool/types.rs @@ -189,6 +189,17 @@ pub trait Tool: Send + Sync { ToolExposure::Direct } + /// The group this tool belongs to, when it has one: a toolpack, a + /// connector toolkit, an MCP server. + /// + /// Read by a tool-search index so a model can find "the Slack one" and so + /// a hit can say where it came from. Purely descriptive — a host neither + /// gates nor routes on it. Most tools have no family and keep the + /// default. + fn family(&self) -> Option<&str> { + None + } + /// Whether two concurrent invocations are safe to run in parallel within a /// single model turn. /// From e2544bc844f803abaaa0ee57dc2306c5b1f15dcf Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:20:52 +0530 Subject: [PATCH 2/7] feat(jev): tinytools-jev, a ToolRanker backed by TypeSafe's Jev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retrieve-then-decide: a retriever (Bm25Ranker by default) narrows the catalogue to retrieval_k candidates, and one Jev request — a Choice over the shortlist plus a none option and a needs_tool Noul — decides among them with calibrated probabilities. Kept out of the vocabulary crate because tinyjevclient carries an HTTP transport; consumed by pinned revision as that crate asks. Co-authored-by: Medulla --- Cargo.lock | 1200 ++++++++++++++++++++++++++++- Cargo.toml | 9 +- crates/tinytools-jev/Cargo.toml | 29 + crates/tinytools-jev/README.md | 57 ++ crates/tinytools-jev/src/lib.rs | 381 +++++++++ crates/tinytools-jev/src/test.rs | 370 +++++++++ crates/tinytools-jev/src/types.rs | 127 +++ 7 files changed, 2138 insertions(+), 35 deletions(-) create mode 100644 crates/tinytools-jev/Cargo.toml create mode 100644 crates/tinytools-jev/README.md create mode 100644 crates/tinytools-jev/src/lib.rs create mode 100644 crates/tinytools-jev/src/test.rs create mode 100644 crates/tinytools-jev/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index acb14a9..d6d44e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,24 +28,463 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -55,6 +494,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -64,6 +559,38 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + [[package]] name = "regex" version = "1.13.1" @@ -77,46 +604,247 @@ dependencies = [ ] [[package]] -name = "regex-automata" -version = "0.4.18" +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "libc", + "windows-sys 0.61.2", ] [[package]] -name = "regex-syntax" -version = "0.8.11" +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "serde" -version = "1.0.229" +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" dependencies = [ - "serde_core", - "serde_derive", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "serde_core" -version = "1.0.229" +name = "sync_wrapper" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ - "serde_derive", + "futures-core", ] [[package]] -name = "serde_derive" -version = "1.0.229" +name = "synstructure" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", @@ -124,27 +852,46 @@ dependencies = [ ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "thiserror" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "thiserror-impl", ] [[package]] -name = "syn" -version = "3.0.4" +name = "thiserror-impl" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", +] + +[[package]] +name = "tinyjevclient" +version = "0.2.1" +source = "git+https://github.com/tinyhumansai/tinyjevclient?rev=e53d5f088ff03fa38c53bac219dab7697b5016c9#e53d5f088ff03fa38c53bac219dab7697b5016c9" +dependencies = [ + "httpdate", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", ] [[package]] @@ -169,14 +916,37 @@ dependencies = [ "tracing", ] +[[package]] +name = "tinytools-jev" +version = "0.3.0" +dependencies = [ + "async-trait", + "serde_json", + "tinyjevclient", + "tinytools", + "tokio", + "tracing", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "socket2", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -190,6 +960,61 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -205,6 +1030,15 @@ name = "tracing-core" version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" @@ -212,6 +1046,306 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index c8b237f..776bb2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,8 +38,13 @@ tracing = { version = "0.1", default-features = false } # `Tool::execute` returns `anyhow::Result` because a tool body calls arbitrary # host code and has no useful closed error set of its own. anyhow = "1" -# Unit tests for the async trait defaults drive a real executor. -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +# `tinytools-jev` ranks tools with TypeSafe's Jev decision model. Consumed by +# pinned revision, as that crate's README asks; only the ranker crate links it, +# never the vocabulary crate. +tinyjevclient = { git = "https://github.com/tinyhumansai/tinyjevclient", rev = "e53d5f088ff03fa38c53bac219dab7697b5016c9" } +# Unit tests for the async trait defaults drive a real executor; `tinytools-jev` +# also bounds its decision call with a `time` deadline of its own. +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } # Lints apply to every member that opts in with `[lints] workspace = true`, and # to every target of that member. CI runs clippy with `-D warnings`, so anything diff --git a/crates/tinytools-jev/Cargo.toml b/crates/tinytools-jev/Cargo.toml new file mode 100644 index 0000000..a25989d --- /dev/null +++ b/crates/tinytools-jev/Cargo.toml @@ -0,0 +1,29 @@ +[package] +publish = false +name = "tinytools-jev" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "A tinytools ToolRanker backed by TypeSafe's Jev decision model: retrieve a shortlist lexically, let Jev decide." +documentation = "https://docs.rs/tinytools-jev" +readme = "README.md" + +[dependencies] +async-trait = { workspace = true } +serde_json = { workspace = true } +tinyjevclient = { workspace = true } +tinytools = { path = "../tinytools", version = "0.3.0" } +tokio = { workspace = true } +tracing = { workspace = true, optional = true } + +[dev-dependencies] +tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] } + +[features] +default = [] +tracing = ["dep:tracing"] + +[lints] +workspace = true diff --git a/crates/tinytools-jev/README.md b/crates/tinytools-jev/README.md new file mode 100644 index 0000000..3f9de2b --- /dev/null +++ b/crates/tinytools-jev/README.md @@ -0,0 +1,57 @@ +# tinytools-jev + +A `tinytools::ToolRanker` backed by TypeSafe's Jev decision model, through +[`tinyjevclient`](https://github.com/tinyhumansai/tinyjevclient). + +## Retrieve, then decide + +Jev answers a `Choice` question with a calibrated probability for every +option in ~150 ms, accepts at most 255 options, and loses accuracy as the +option list fills with entries unrelated to the request. So `JevRanker` +never shows it a whole catalogue: + +1. A retriever (`Bm25Ranker` unless the host supplies one) narrows the + catalogue to `retrieval_k` candidates (20 by default). Skipped when the + catalogue already fits. When the retriever finds nothing and the catalogue + fits one Choice, Jev sees all of it — a lexical miss on a paraphrase is + exactly the case a decision model is for. +2. One request: a `Choice` over the shortlist plus a `none` option, and a + `Noul` asking whether the request needs a tool at all. +3. Hits are the options by probability, `none` removed, anything below + `min_probability` dropped. Each hit's `confidence` is its probability; + `rank_detailed` also returns the Choice confidence, the `needs_tool` + probability, the `none` probability, tokens, latency and attempts. + +Every failure — transport, a rejected request, the deadline — is a +`RankError` the caller falls back from. The API key never appears in an error +or a log line. + +## Limits that shape the design + +| Limit | Value | Consequence | +|------------------------------|--------|-----------------------------------------------| +| Options per Choice | 255 | `retrieval_k` is clamped to it | +| Summary shown per option | 240 ch | clipped, with the family named after | +| Context per request | 64k | `RankContext` stays to a few recent turns | +| Pricing (jev-1.13) | $0.042 / M input, output free | a search is ~1–2k tokens | + +## Wording + +Jev reads literally. The instructions name the user's `request` and ask which +tool accomplishes it "by what each tool does, not by shared words"; each +option is `name: first sentence (from family)`; the state carries the request +and at most the caller's few recent turns. + +## Building a ranker + +```rust,no_run +use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; + +let client = ClientConfig::tinyhumans_openrouter(""); +let ranker = JevRanker::from_config(client, JevRankerConfig::new())?; +# Ok::<(), tinytools::RankError>(()) +``` + +`ClientConfig::new` targets TypeSafe directly, `::openrouter` OpenRouter's +compatible endpoint, and `::tinyhumans_openrouter` the TinyHumans proxy; the +`base_url` field is public for a self-hosted proxy. diff --git a/crates/tinytools-jev/src/lib.rs b/crates/tinytools-jev/src/lib.rs new file mode 100644 index 0000000..8ae850e --- /dev/null +++ b/crates/tinytools-jev/src/lib.rs @@ -0,0 +1,381 @@ +//! A [`ToolRanker`] backed by `TypeSafe`'s Jev decision model. +//! +//! # Retrieve, then decide +//! +//! Jev answers a `Choice` question — "which of these options fits this +//! state?" — with a calibrated probability for every option, in one round +//! trip of ~150 ms. It accepts at most 255 options, and its accuracy falls as +//! the option list fills with entries unrelated to the request. So this +//! ranker never shows Jev the whole catalogue. A cheap retriever (BM25 unless +//! the host supplies something better) narrows the catalogue to a shortlist +//! of `retrieval_k` candidates, and one Jev request decides among them: +//! +//! 1. `retriever.rank(intent, catalogue, retrieval_k)` → shortlist. When the +//! catalogue already fits, the retriever is skipped and Jev sees all of +//! it. When the retriever finds *nothing* — the paraphrase it cannot +//! bridge — and the catalogue is small enough, Jev still sees all of it, +//! because that is precisely the case a decision model exists for. +//! 2. One request: a `Choice` over the shortlist plus a `none` option, and a +//! `Noul` asking whether the request needs a tool at all. +//! 3. Hits are the options ordered by probability, `none` removed, below +//! [`JevRankerConfig::min_probability`] dropped. Each hit's `confidence` +//! is its probability. +//! +//! Anything that stops the decision — transport, a rejected request, the +//! deadline — is a [`RankError`] the caller falls back from. The API key is +//! never in an error or a log line; `tinyjevclient` redacts it. +//! +//! # Wording +//! +//! Jev reads literally. The instructions name the *user's request* and ask +//! for the tool that accomplishes it, each option is `name: first sentence`, +//! and the state carries the request and at most a few recent turns. Extra +//! context is a distractor, not a help. + +#[cfg(test)] +mod test; +mod types; + +pub use tinyjevclient::{Client, ClientConfig, Provider, RetryPolicy}; +pub use types::{JevRankerConfig, JevRanking}; + +use std::{collections::BTreeMap, time::Duration}; + +use serde_json::{Value, json}; +use tinyjevclient::{ + Answer, Choice, Error as JevError, EvaluationFailure, EvaluationRequest, Noul, NoulCriteria, + Question, +}; +use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; + +/// Question id of the tool `Choice`. +const TOOL_QUESTION: &str = "tool"; +/// Question id of the needs-a-tool `Noul`. +const NEEDS_TOOL_QUESTION: &str = "needs_tool"; +/// The option every Choice carries so an off-catalogue request has somewhere +/// to go other than the least-bad tool. +const NONE_OPTION: &str = "none"; +/// Longest summary Jev is shown per option. Descriptions past this are +/// clipped at a character boundary; the model decides on the opening +/// sentence anyway, and the request has to fit under the provider's body cap. +const MAX_SUMMARY_CHARS: usize = 240; + +/// Ranks tools with Jev. See the [crate docs](crate) for how. +#[derive(Debug, Clone)] +pub struct JevRanker { + client: Client, + config: JevRankerConfig, +} + +impl JevRanker { + /// The stable [`ToolRanker::kind`] of this ranker. + pub const KIND: &'static str = "jev"; + + /// A ranker over an already-built client. + #[must_use] + pub fn new(client: Client, config: JevRankerConfig) -> Self { + Self { client, config } + } + + /// A ranker over a client built from `client_config`. + /// + /// # Errors + /// + /// Returns the client's configuration error (empty key, bad base URL, + /// zero timeout) as [`RankError::InvalidInput`]. + pub fn from_config( + client_config: ClientConfig, + config: JevRankerConfig, + ) -> Result { + let client = Client::new(client_config).map_err(|error| RankError::InvalidInput { + reason: error.to_string(), + })?; + Ok(Self::new(client, config)) + } + + /// The configuration in force. + #[must_use] + pub fn config(&self) -> &JevRankerConfig { + &self.config + } + + /// Ranks and returns everything the decision learned, not only the hits. + /// + /// # Errors + /// + /// [`RankError::InvalidInput`] for an empty intent or a duplicate + /// candidate key; [`RankError::Timeout`] past the configured deadline; + /// [`RankError::Backend`] for anything the provider or transport did. + pub async fn rank_detailed( + &self, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result { + let intent = intent.trim(); + if intent.is_empty() { + return Err(RankError::InvalidInput { + reason: "intent is empty".to_owned(), + }); + } + if candidates.is_empty() || limit == 0 { + return Ok(JevRanking::empty()); + } + let shortlist = self.shortlist(intent, context, candidates).await?; + if shortlist.is_empty() { + return Ok(JevRanking::empty()); + } + + let request = self.build_request(intent, context, &shortlist)?; + let started = std::time::Instant::now(); + let evaluated = tokio::time::timeout(self.config.timeout, self.client.evaluate(&request)) + .await + .map_err(|_elapsed| RankError::Timeout)?; + let result = evaluated.map_err(map_failure)?; + let mut ranking = decode(&result.response, &shortlist, self.config.min_probability)?; + ranking.hits.truncate(limit); + ranking.shortlisted = shortlist.len(); + ranking.latency = started.elapsed(); + ranking.attempts = result.attempts; + ranking.input_tokens = result.response.usage.input_tokens; + #[cfg(feature = "tracing")] + tracing::debug!( + target: "tinytools_jev", + shortlisted = shortlist.len(), + hits = ranking.hits.len(), + choice_confidence = ranking.choice_confidence, + needs_tool = ?ranking.needs_tool, + latency_ms = ranking.latency.as_millis() as u64, + attempts = ranking.attempts, + "jev tool ranking" + ); + Ok(ranking) + } + + /// Narrows `candidates` to what Jev will be shown. + async fn shortlist<'a>( + &self, + intent: &str, + context: &RankContext, + candidates: &'a [RankCandidate], + ) -> Result, RankError> { + let fits_without_retrieval = candidates.len() <= self.config.retrieval_k; + if fits_without_retrieval { + return Ok(candidates.iter().collect()); + } + let hits = self + .config + .retriever + .rank(intent, context, candidates, self.config.retrieval_k) + .await?; + let by_key: BTreeMap<&str, &RankCandidate> = candidates + .iter() + .map(|candidate| (candidate.key.as_str(), candidate)) + .collect(); + if by_key.len() != candidates.len() { + return Err(RankError::InvalidInput { + reason: "duplicate candidate key".to_owned(), + }); + } + let shortlist: Vec<&RankCandidate> = hits + .iter() + .filter_map(|hit| by_key.get(hit.key.as_str()).copied()) + .collect(); + // A lexical retriever that finds nothing has met a paraphrase. If the + // whole catalogue fits one Choice, let the decision model see it. + if shortlist.is_empty() && candidates.len() <= JevRankerConfig::MAX_OPTIONS { + return Ok(candidates.iter().collect()); + } + Ok(shortlist) + } + + fn build_request( + &self, + intent: &str, + context: &RankContext, + shortlist: &[&RankCandidate], + ) -> Result { + let mut criteria: BTreeMap> = BTreeMap::new(); + for candidate in shortlist { + if candidate.key == NONE_OPTION { + return Err(RankError::InvalidInput { + reason: format!("candidate key `{NONE_OPTION}` is reserved"), + }); + } + if criteria + .insert(candidate.key.clone(), Some(option_text(candidate))) + .is_some() + { + return Err(RankError::InvalidInput { + reason: "duplicate candidate key".to_owned(), + }); + } + } + criteria.insert( + NONE_OPTION.to_owned(), + Some(json!("No listed tool accomplishes the request.")), + ); + + let mut state = json!({ "request": intent }); + if !context.recent_turns.is_empty() + && let Some(object) = state.as_object_mut() + { + object.insert( + "recent_user_turns".to_owned(), + Value::Array( + context + .recent_turns + .iter() + .map(|turn| Value::String(turn.clone())) + .collect(), + ), + ); + } + + let questions = BTreeMap::from([ + ( + TOOL_QUESTION.to_owned(), + Question::Choice(Choice { + instructions: json!( + "Which tool accomplishes the user's `request`? Judge by what \ + each tool does, not by shared words. Pick `none` when no \ + listed tool does it." + ), + criteria, + }), + ), + ( + NEEDS_TOOL_QUESTION.to_owned(), + Question::Noul(Noul { + instructions: json!( + "Does fulfilling the user's `request` require calling a tool \ + — an action or a lookup outside the assistant's own knowledge?" + ), + criteria: Some(NoulCriteria { + r#true: json!( + "The request asks for an action or for information that \ + must be fetched." + ), + r#false: json!("The request can be answered by replying, with no tool."), + }), + }), + ), + ]); + + Ok(EvaluationRequest { + state, + model: self.config.model.clone(), + questions, + }) + } +} + +#[async_trait::async_trait] +impl ToolRanker for JevRanker { + fn kind(&self) -> &'static str { + Self::KIND + } + + async fn rank( + &self, + intent: &str, + context: &RankContext, + candidates: &[RankCandidate], + limit: usize, + ) -> Result, RankError> { + self.rank_detailed(intent, context, candidates, limit) + .await + .map(|ranking| ranking.hits) + } +} + +impl JevRanking { + fn empty() -> Self { + Self { + hits: Vec::new(), + choice_confidence: 0.0, + needs_tool: None, + none_probability: 0.0, + shortlisted: 0, + input_tokens: None, + latency: Duration::ZERO, + attempts: 0, + } + } +} + +/// `name: summary`, clipped, with the family named so "the Slack one" ranks. +fn option_text(candidate: &RankCandidate) -> Value { + let mut summary: String = candidate.summary.chars().take(MAX_SUMMARY_CHARS).collect(); + if summary.len() < candidate.summary.len() { + summary.push('…'); + } + match &candidate.family { + Some(family) => json!(format!("{summary} (from {family})")), + None => json!(summary), + } +} + +fn map_failure(failure: EvaluationFailure) -> RankError { + match failure.error { + JevError::InvalidRequest { reason } | JevError::InvalidConfig { reason } => { + RankError::InvalidInput { reason } + } + JevError::Timeout => RankError::Timeout, + other => RankError::Backend { + // `Display` on every variant is credential-free by the client's + // contract; the transport source is dropped, not printed. + reason: format!("{other} after {} attempt(s)", failure.attempts), + }, + } +} + +/// Turns the provider's answers into hits, best first. +fn decode( + response: &tinyjevclient::EvaluationResponse, + shortlist: &[&RankCandidate], + min_probability: f64, +) -> Result { + let Some(Answer::Choice(choice)) = response.answers.get(TOOL_QUESTION) else { + return Err(RankError::Backend { + reason: "response has no choice answer for `tool`".to_owned(), + }); + }; + let needs_tool = match response.answers.get(NEEDS_TOOL_QUESTION) { + Some(Answer::Noul(noul)) => Some(noul.noul), + _ => None, + }; + let none_probability = choice + .probabilities + .get(NONE_OPTION) + .copied() + .unwrap_or(0.0); + let mut hits: Vec = shortlist + .iter() + .filter_map(|candidate| { + let probability = *choice.probabilities.get(&candidate.key)?; + (probability >= min_probability).then(|| RankHit { + key: candidate.key.clone(), + score: probability, + confidence: Some(probability), + }) + }) + .collect(); + hits.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.key.cmp(&b.key)) + }); + Ok(JevRanking { + hits, + choice_confidence: choice.confidence, + needs_tool, + none_probability, + shortlisted: shortlist.len(), + input_tokens: None, + latency: Duration::ZERO, + attempts: 0, + }) +} diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs new file mode 100644 index 0000000..d8352f5 --- /dev/null +++ b/crates/tinytools-jev/src/test.rs @@ -0,0 +1,370 @@ +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::float_cmp, + clippy::needless_pass_by_value +)] + +use std::{sync::Arc, time::Duration}; + +use serde_json::{Value, json}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::Mutex, +}; + +use super::*; + +fn candidates() -> Vec { + vec![ + RankCandidate::new( + "SLACK_SEND_MESSAGE", + "SLACK_SEND_MESSAGE Send a message to a Slack channel or user. channel text", + ) + .with_family("slack"), + RankCandidate::new( + "GMAIL_SEND_EMAIL", + "GMAIL_SEND_EMAIL Send an email from the connected Gmail account. to subject body", + ) + .with_family("gmail"), + RankCandidate::new( + "stock_quote", + "stock_quote Fetch the latest price for a ticker symbol. symbol", + ), + ] +} + +fn response(status: u16, body: &str) -> String { + let reason = if status == 200 { "OK" } else { "Error" }; + format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) +} + +fn answer(probabilities: Value, confidence: f64, needs_tool: Option) -> String { + let choice = probabilities + .as_object() + .unwrap() + .iter() + .max_by(|a, b| a.1.as_f64().partial_cmp(&b.1.as_f64()).unwrap()) + .map(|(k, _)| k.clone()) + .unwrap(); + let mut answers = json!({ + "tool": { + "type": "choice", + "choice": choice, + "probabilities": probabilities, + "confidence": confidence + } + }); + if let Some(p) = needs_tool { + answers["needs_tool"] = json!({"type": "noul", "noul": p}); + } + json!({ + "model": "typesafe/jev-1.13", + "answers": answers, + "usage": {"input_tokens": 321, "output_tokens": 4} + }) + .to_string() +} + +/// One-shot loopback server: answers each connection with the next canned +/// response and records the request bodies it saw. +async fn server(responses: Vec) -> (String, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = Arc::clone(&seen); + tokio::spawn(async move { + for canned in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = vec![0_u8; 65_536]; + let mut raw = Vec::new(); + loop { + let read = socket.read(&mut buffer).await.unwrap(); + raw.extend_from_slice(&buffer[..read]); + let text = String::from_utf8_lossy(&raw); + if let Some((head, body)) = text.split_once("\r\n\r\n") { + let length: usize = head + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if body.len() >= length { + recorder + .lock() + .await + .push(serde_json::from_str(body).unwrap()); + break; + } + } + if read == 0 { + break; + } + } + socket.write_all(canned.as_bytes()).await.unwrap(); + socket.shutdown().await.unwrap(); + } + }); + (format!("http://{address}"), seen) +} + +fn ranker(base_url: String, config: JevRankerConfig) -> JevRanker { + let mut client_config = ClientConfig::openrouter("test-key"); + client_config.base_url = base_url; + client_config.retry = RetryPolicy { + max_retries: 0, + ..RetryPolicy::default() + }; + JevRanker::from_config(client_config, config).unwrap() +} + +#[tokio::test] +async fn small_catalogue_goes_straight_to_jev_with_a_none_option() { + let (url, seen) = server(vec![response( + 200, + &answer( + json!({"SLACK_SEND_MESSAGE": 0.83, "GMAIL_SEND_EMAIL": 0.12, "stock_quote": 0.01, "none": 0.04}), + 0.79, + Some(0.97), + ), + )]) + .await; + let ranker = ranker(url, JevRankerConfig::new()); + + let ranking = ranker + .rank_detailed( + "ping alex on slack that I'm ten minutes late", + &RankContext::empty(), + &candidates(), + 3, + ) + .await + .unwrap(); + + assert_eq!( + ranking + .hits + .iter() + .map(|h| h.key.as_str()) + .collect::>(), + vec!["SLACK_SEND_MESSAGE", "GMAIL_SEND_EMAIL"], + "stock_quote sits below min_probability and none is never a hit" + ); + assert_eq!(ranking.hits[0].confidence, Some(0.83)); + assert_eq!(ranking.choice_confidence, 0.79); + assert_eq!(ranking.needs_tool, Some(0.97)); + assert_eq!(ranking.none_probability, 0.04); + assert_eq!(ranking.shortlisted, 3); + assert_eq!(ranking.input_tokens, Some(321)); + assert_eq!(ranking.attempts, 1); + + let request = &seen.lock().await[0]; + assert_eq!(request["model"], "jev-latest"); + assert_eq!( + request["state"]["request"], + "ping alex on slack that I'm ten minutes late" + ); + let criteria = request["questions"]["tool"]["criteria"] + .as_object() + .unwrap(); + assert_eq!(criteria.len(), 4, "three candidates plus `none`"); + assert!( + criteria["SLACK_SEND_MESSAGE"] + .as_str() + .unwrap() + .ends_with("(from slack)") + ); + assert_eq!(request["questions"]["needs_tool"]["type"], "noul"); +} + +#[tokio::test] +async fn large_catalogue_is_retrieved_first_then_decided() { + let (url, seen) = server(vec![response( + 200, + &answer( + json!({"t_send_7": 0.9, "t_send_3": 0.06, "none": 0.04}), + 0.85, + Some(0.9), + ), + )]) + .await; + let mut catalogue: Vec = (0..40) + .map(|i| { + RankCandidate::new( + format!("t_read_{i}"), + format!("t_read_{i} Read record {i}."), + ) + }) + .collect(); + catalogue.push(RankCandidate::new( + "t_send_7", + "t_send_7 Send a message to a person.", + )); + catalogue.push(RankCandidate::new( + "t_send_3", + "t_send_3 Send a message to a channel.", + )); + let ranker = ranker(url, JevRankerConfig::new().with_retrieval_k(5)); + + let ranking = ranker + .rank_detailed("send a message", &RankContext::empty(), &catalogue, 3) + .await; + let ranking = ranking.unwrap(); + + assert_eq!(ranking.hits[0].key, "t_send_7"); + assert_eq!(ranking.needs_tool, Some(0.9)); + let request = &seen.lock().await[0]; + let criteria = request["questions"]["tool"]["criteria"] + .as_object() + .unwrap(); + assert!( + criteria.len() <= 6, + "shortlist of at most 5 plus `none`, got {}", + criteria.len() + ); + assert!(criteria.contains_key("t_send_7")); + assert!(criteria.contains_key("t_send_3")); + assert_eq!(ranking.shortlisted, criteria.len() - 1); +} + +#[tokio::test] +async fn retriever_miss_on_a_small_catalogue_still_lets_jev_decide() { + let (url, seen) = server(vec![response( + 200, + &answer( + json!({"SLACK_SEND_MESSAGE": 0.7, "GMAIL_SEND_EMAIL": 0.2, "stock_quote": 0.05, "none": 0.05}), + 0.6, + Some(0.8), + ), + )]) + .await; + // retrieval_k of 1 forces retrieval; "ping" matches nothing lexically. + let ranker = ranker(url, JevRankerConfig::new().with_retrieval_k(1)); + + let hits = ranker + .rank("ping alex", &RankContext::empty(), &candidates(), 3) + .await + .unwrap(); + + assert_eq!(hits[0].key, "SLACK_SEND_MESSAGE"); + let request = &seen.lock().await[0]; + let criteria = request["questions"]["tool"]["criteria"] + .as_object() + .unwrap(); + assert_eq!( + criteria.len(), + 4, + "the whole catalogue was shown after the retriever missed" + ); +} + +#[tokio::test] +async fn empty_inputs_never_reach_the_network() { + let ranker = ranker("http://127.0.0.1:9".to_owned(), JevRankerConfig::new()); + assert!( + ranker + .rank("anything", &RankContext::empty(), &[], 3) + .await + .unwrap() + .is_empty() + ); + assert!( + ranker + .rank("anything", &RankContext::empty(), &candidates(), 0) + .await + .unwrap() + .is_empty() + ); + let err = ranker + .rank(" ", &RankContext::empty(), &candidates(), 3) + .await + .unwrap_err(); + assert!(matches!(err, RankError::InvalidInput { .. }), "{err}"); +} + +#[tokio::test] +async fn reserved_and_duplicate_keys_are_rejected_before_sending() { + let ranker = ranker("http://127.0.0.1:9".to_owned(), JevRankerConfig::new()); + let reserved = vec![RankCandidate::new("none", "none nothing")]; + let err = ranker + .rank("x", &RankContext::empty(), &reserved, 3) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + "invalid ranking input: candidate key `none` is reserved" + ); + let duplicate = vec![ + RankCandidate::new("a", "a one"), + RankCandidate::new("a", "a two"), + ]; + let err = ranker + .rank("x", &RankContext::empty(), &duplicate, 3) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + "invalid ranking input: duplicate candidate key" + ); +} + +#[tokio::test] +async fn provider_failures_become_backend_errors_without_the_key() { + let (url, _) = server(vec![response(401, r#"{"error":"nope"}"#)]).await; + let ranker = ranker(url, JevRankerConfig::new()); + let err = ranker + .rank("send a message", &RankContext::empty(), &candidates(), 3) + .await + .unwrap_err(); + let text = err.to_string(); + assert!(matches!(err, RankError::Backend { .. }), "{text}"); + assert!(text.contains("authentication failed"), "{text}"); + assert!( + !text.contains("test-key"), + "the key must never surface: {text}" + ); +} + +#[tokio::test] +async fn the_deadline_is_enforced() { + // A listener that accepts and never answers. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let (_socket, _) = listener.accept().await.unwrap(); + tokio::time::sleep(Duration::from_secs(30)).await; + }); + let ranker = ranker( + url, + JevRankerConfig::new().with_timeout(Duration::from_millis(200)), + ); + let err = ranker + .rank("send a message", &RankContext::empty(), &candidates(), 3) + .await + .unwrap_err(); + assert!(matches!(err, RankError::Timeout), "{err}"); +} + +#[test] +fn option_text_clips_long_summaries_and_names_the_family() { + let long = "x".repeat(400); + let candidate = RankCandidate::new("k", long).with_family("fam"); + let text = option_text(&candidate); + let text = text.as_str().unwrap(); + assert!(text.starts_with(&"x".repeat(MAX_SUMMARY_CHARS))); + assert!(text.ends_with("… (from fam)")); +} + +#[test] +fn config_debug_never_prints_a_client_and_clamps_knobs() { + let config = JevRankerConfig::new() + .with_retrieval_k(9_999) + .with_min_probability(7.0); + assert_eq!(config.retrieval_k, JevRankerConfig::MAX_OPTIONS); + assert_eq!(config.min_probability, 1.0); + assert!(format!("{config:?}").contains("bm25")); +} diff --git a/crates/tinytools-jev/src/types.rs b/crates/tinytools-jev/src/types.rs new file mode 100644 index 0000000..09cbd4f --- /dev/null +++ b/crates/tinytools-jev/src/types.rs @@ -0,0 +1,127 @@ +//! Configuration and the detailed answer a Jev ranking produces. + +use std::{fmt, sync::Arc, time::Duration}; + +use tinytools::{Bm25Ranker, RankHit, ToolRanker}; + +/// How [`JevRanker`][crate::JevRanker] retrieves and decides. +#[derive(Clone)] +pub struct JevRankerConfig { + /// Ranks the full catalogue down to a shortlist before Jev sees it. + /// [`Bm25Ranker`] by default; a host with an embedding index passes that. + pub retriever: Arc, + /// How many candidates the retriever hands to Jev. Twenty is the + /// documented sweet spot: small enough that one Choice question decides + /// in ~150 ms, large enough that a lexical retriever's recall is not the + /// bottleneck. Never above [`Self::MAX_OPTIONS`]. + pub retrieval_k: usize, + /// Hits whose probability falls below this are dropped, so a caller + /// never sees the long tail of a distribution as if it were a match. + pub min_probability: f64, + /// Deadline for the decision call, on top of the client's own per-attempt + /// timeout and retries. A tool search sits in a model's turn; a slow + /// answer is worse than a fallback. + pub timeout: Duration, + /// System One model id. `jev-latest` unless a host pins one. + pub model: String, +} + +impl JevRankerConfig { + /// The most options one Jev Choice question accepts. + pub const MAX_OPTIONS: usize = 255; + + /// Defaults: BM25 retrieval to 20, `min_probability` 0.05, 3 s deadline, + /// `jev-latest`. + #[must_use] + pub fn new() -> Self { + Self { + retriever: Arc::new(Bm25Ranker), + retrieval_k: 20, + min_probability: 0.05, + timeout: Duration::from_secs(3), + model: "jev-latest".to_owned(), + } + } + + /// Replaces the retriever. + #[must_use] + pub fn with_retriever(mut self, retriever: Arc) -> Self { + self.retriever = retriever; + self + } + + /// Sets the shortlist size, clamped to `1..=MAX_OPTIONS`. + #[must_use] + pub fn with_retrieval_k(mut self, k: usize) -> Self { + self.retrieval_k = k.clamp(1, Self::MAX_OPTIONS); + self + } + + /// Sets the probability floor, clamped to `0.0..=1.0`. + #[must_use] + pub fn with_min_probability(mut self, p: f64) -> Self { + self.min_probability = p.clamp(0.0, 1.0); + self + } + + /// Sets the decision deadline. + #[must_use] + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + /// Sets the model id. + #[must_use] + pub fn with_model(mut self, model: impl Into) -> Self { + self.model = model.into(); + self + } +} + +impl Default for JevRankerConfig { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for JevRankerConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("JevRankerConfig") + .field("retriever", &self.retriever.kind()) + .field("retrieval_k", &self.retrieval_k) + .field("min_probability", &self.min_probability) + .field("timeout", &self.timeout) + .field("model", &self.model) + .finish() + } +} + +/// Everything one Jev ranking learned, beyond the hits the trait returns. +/// +/// A caller that only wants the hits uses [`tinytools::ToolRanker::rank`]; +/// one that gates on "does this need a tool at all", or that reports cost, +/// calls [`JevRanker::rank_detailed`][crate::JevRanker::rank_detailed]. +#[derive(Clone, Debug, PartialEq)] +pub struct JevRanking { + /// Ranked hits, best first; each `confidence` is Jev's probability for + /// that option. + pub hits: Vec, + /// Jev's confidence in the Choice as a whole, `0.0..=1.0`. Low when the + /// distribution is flat — the signal to prefer asking over acting. + pub choice_confidence: f64, + /// Probability that the request needs a tool at all, from the `Noul` + /// asked alongside. `None` when Jev did not answer it. + pub needs_tool: Option, + /// Probability Jev put on "none of these", the option every request + /// carries so an off-catalogue intent is not forced onto a tool. + pub none_probability: f64, + /// How many candidates the retriever handed to Jev. + pub shortlisted: usize, + /// Input tokens billed, when the provider reports them. + pub input_tokens: Option, + /// Wall time of the decision call, including the client's retries. + pub latency: Duration, + /// Attempts the client made. + pub attempts: u32, +} From 574dc854501b79a8260e2c67e1b18ae10cb8d7af Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 04:59:49 +0300 Subject: [PATCH 3/7] chore: files changed Cargo.lock,Cargo.toml,crates/tinytools-jev/Cargo.toml,crates/tinytools-jev/READ Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 1124 ---------------------------- Cargo.toml | 7 +- crates/tinytools-jev/Cargo.toml | 5 +- crates/tinytools-jev/README.md | 23 +- crates/tinytools-jev/src/lib.rs | 357 +++------ crates/tinytools-jev/src/test.rs | 424 ++--------- crates/tinytools-jev/src/types.rs | 148 ++-- crates/tinytools/src/lib.rs | 4 +- crates/tinytools/src/rank/test.rs | 9 +- crates/tinytools/src/rank/types.rs | 18 + 10 files changed, 292 insertions(+), 1827 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6d44e7..93e26fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,463 +28,24 @@ dependencies = [ "syn", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cc" -version = "1.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core", -] - -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures-channel" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "rand_core", - "wasm-bindgen", -] - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "icu_collections" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" - -[[package]] -name = "icu_properties" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" - -[[package]] -name = "icu_provider" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "ipnet" -version = "2.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "js-sys" -version = "0.3.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "litemap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "lru-slab" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "mio" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "potential_utf" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" -dependencies = [ - "zerovec", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -494,62 +55,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quinn" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" -dependencies = [ - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - [[package]] name = "quote" version = "1.0.47" @@ -559,38 +64,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core", -] - [[package]] name = "regex" version = "1.13.1" @@ -620,111 +93,6 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64", - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustls" -version = "0.23.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "serde" version = "1.0.229" @@ -768,58 +136,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "3.0.4" @@ -831,69 +147,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tinyjevclient" -version = "0.2.1" -source = "git+https://github.com/tinyhumansai/tinyjevclient?rev=e53d5f088ff03fa38c53bac219dab7697b5016c9#e53d5f088ff03fa38c53bac219dab7697b5016c9" -dependencies = [ - "httpdate", - "reqwest", - "serde", - "serde_json", - "thiserror", - "tokio", -] - -[[package]] -name = "tinystr" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinytools" version = "0.3.0" @@ -921,32 +174,19 @@ name = "tinytools-jev" version = "0.3.0" dependencies = [ "async-trait", - "serde_json", - "tinyjevclient", "tinytools", "tokio", "tracing", ] -[[package]] -name = "tinyvec" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" - [[package]] name = "tokio" version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ - "bytes", - "libc", - "mio", "pin-project-lite", - "socket2", "tokio-macros", - "windows-sys 0.61.2", ] [[package]] @@ -960,61 +200,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-rustls" -version = "0.26.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.44" @@ -1030,15 +215,6 @@ name = "tracing-core" version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" @@ -1046,306 +222,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasm-bindgen" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "writeable" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 776bb2c..95366f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,12 +38,7 @@ tracing = { version = "0.1", default-features = false } # `Tool::execute` returns `anyhow::Result` because a tool body calls arbitrary # host code and has no useful closed error set of its own. anyhow = "1" -# `tinytools-jev` ranks tools with TypeSafe's Jev decision model. Consumed by -# pinned revision, as that crate's README asks; only the ranker crate links it, -# never the vocabulary crate. -tinyjevclient = { git = "https://github.com/tinyhumansai/tinyjevclient", rev = "e53d5f088ff03fa38c53bac219dab7697b5016c9" } -# Unit tests for the async trait defaults drive a real executor; `tinytools-jev` -# also bounds its decision call with a `time` deadline of its own. +# Unit tests for async traits drive a real executor. tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } # Lints apply to every member that opts in with `[lints] workspace = true`, and diff --git a/crates/tinytools-jev/Cargo.toml b/crates/tinytools-jev/Cargo.toml index a25989d..e029391 100644 --- a/crates/tinytools-jev/Cargo.toml +++ b/crates/tinytools-jev/Cargo.toml @@ -12,14 +12,11 @@ readme = "README.md" [dependencies] async-trait = { workspace = true } -serde_json = { workspace = true } -tinyjevclient = { workspace = true } tinytools = { path = "../tinytools", version = "0.3.0" } -tokio = { workspace = true } tracing = { workspace = true, optional = true } [dev-dependencies] -tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] } +tokio = { workspace = true } [features] default = [] diff --git a/crates/tinytools-jev/README.md b/crates/tinytools-jev/README.md index 3f9de2b..fda20a6 100644 --- a/crates/tinytools-jev/README.md +++ b/crates/tinytools-jev/README.md @@ -1,7 +1,7 @@ # tinytools-jev -A `tinytools::ToolRanker` backed by TypeSafe's Jev decision model, through -[`tinyjevclient`](https://github.com/tinyhumansai/tinyjevclient). +A `tinytools::ToolRanker` backed by a host-provided Jev evaluator. The host +owns its HTTP client, credentials, retry policy, and deadline. ## Retrieve, then decide @@ -22,9 +22,7 @@ never shows it a whole catalogue: `rank_detailed` also returns the Choice confidence, the `needs_tool` probability, the `none` probability, tokens, latency and attempts. -Every failure — transport, a rejected request, the deadline — is a -`RankError` the caller falls back from. The API key never appears in an error -or a log line. +Every evaluator failure is a `RankError` the caller falls back from. ## Limits that shape the design @@ -44,14 +42,7 @@ and at most the caller's few recent turns. ## Building a ranker -```rust,no_run -use tinytools_jev::{ClientConfig, JevRanker, JevRankerConfig}; - -let client = ClientConfig::tinyhumans_openrouter(""); -let ranker = JevRanker::from_config(client, JevRankerConfig::new())?; -# Ok::<(), tinytools::RankError>(()) -``` - -`ClientConfig::new` targets TypeSafe directly, `::openrouter` OpenRouter's -compatible endpoint, and `::tinyhumans_openrouter` the TinyHumans proxy; the -`base_url` field is public for a self-hosted proxy. +Implement `JevEvaluator` in the host by translating `JevRequest` into the +client's wire request and translating its answer into `JevDecision`. Pass that +implementation to `JevRanker::new`. This keeps transport and runtime choices +at the host boundary where their policy belongs. diff --git a/crates/tinytools-jev/src/lib.rs b/crates/tinytools-jev/src/lib.rs index 8ae850e..cff3a9b 100644 --- a/crates/tinytools-jev/src/lib.rs +++ b/crates/tinytools-jev/src/lib.rs @@ -1,111 +1,45 @@ -//! A [`ToolRanker`] backed by `TypeSafe`'s Jev decision model. +//! Dependency-free Jev-backed tool ranking. //! -//! # Retrieve, then decide -//! -//! Jev answers a `Choice` question — "which of these options fits this -//! state?" — with a calibrated probability for every option, in one round -//! trip of ~150 ms. It accepts at most 255 options, and its accuracy falls as -//! the option list fills with entries unrelated to the request. So this -//! ranker never shows Jev the whole catalogue. A cheap retriever (BM25 unless -//! the host supplies something better) narrows the catalogue to a shortlist -//! of `retrieval_k` candidates, and one Jev request decides among them: -//! -//! 1. `retriever.rank(intent, catalogue, retrieval_k)` → shortlist. When the -//! catalogue already fits, the retriever is skipped and Jev sees all of -//! it. When the retriever finds *nothing* — the paraphrase it cannot -//! bridge — and the catalogue is small enough, Jev still sees all of it, -//! because that is precisely the case a decision model exists for. -//! 2. One request: a `Choice` over the shortlist plus a `none` option, and a -//! `Noul` asking whether the request needs a tool at all. -//! 3. Hits are the options ordered by probability, `none` removed, below -//! [`JevRankerConfig::min_probability`] dropped. Each hit's `confidence` -//! is its probability. -//! -//! Anything that stops the decision — transport, a rejected request, the -//! deadline — is a [`RankError`] the caller falls back from. The API key is -//! never in an error or a log line; `tinyjevclient` redacts it. -//! -//! # Wording -//! -//! Jev reads literally. The instructions name the *user's request* and ask -//! for the tool that accomplishes it, each option is `name: first sentence`, -//! and the state carries the request and at most a few recent turns. Extra -//! context is a distractor, not a help. +//! A host supplies [`JevEvaluator`], retaining ownership of transport, +//! authentication, retry, and deadline policy. #[cfg(test)] mod test; mod types; +pub use types::{JevDecision, JevEvaluator, JevOption, JevRankerConfig, JevRanking, JevRequest}; -pub use tinyjevclient::{Client, ClientConfig, Provider, RetryPolicy}; -pub use types::{JevRankerConfig, JevRanking}; - -use std::{collections::BTreeMap, time::Duration}; - -use serde_json::{Value, json}; -use tinyjevclient::{ - Answer, Choice, Error as JevError, EvaluationFailure, EvaluationRequest, Noul, NoulCriteria, - Question, +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, + time::Duration, }; use tinytools::{RankCandidate, RankContext, RankError, RankHit, ToolRanker}; - -/// Question id of the tool `Choice`. -const TOOL_QUESTION: &str = "tool"; -/// Question id of the needs-a-tool `Noul`. -const NEEDS_TOOL_QUESTION: &str = "needs_tool"; -/// The option every Choice carries so an off-catalogue request has somewhere -/// to go other than the least-bad tool. const NONE_OPTION: &str = "none"; -/// Longest summary Jev is shown per option. Descriptions past this are -/// clipped at a character boundary; the model decides on the opening -/// sentence anyway, and the request has to fit under the provider's body cap. const MAX_SUMMARY_CHARS: usize = 240; -/// Ranks tools with Jev. See the [crate docs](crate) for how. +/// Ranks tools using a host-provided evaluator. #[derive(Debug, Clone)] pub struct JevRanker { - client: Client, + evaluator: Arc, config: JevRankerConfig, } - impl JevRanker { - /// The stable [`ToolRanker::kind`] of this ranker. + /// Stable ranker kind. pub const KIND: &'static str = "jev"; - - /// A ranker over an already-built client. + /// Creates a ranker. #[must_use] - pub fn new(client: Client, config: JevRankerConfig) -> Self { - Self { client, config } - } - - /// A ranker over a client built from `client_config`. - /// - /// # Errors - /// - /// Returns the client's configuration error (empty key, bad base URL, - /// zero timeout) as [`RankError::InvalidInput`]. - pub fn from_config( - client_config: ClientConfig, - config: JevRankerConfig, - ) -> Result { - let client = Client::new(client_config).map_err(|error| RankError::InvalidInput { - reason: error.to_string(), - })?; - Ok(Self::new(client, config)) + pub fn new(evaluator: Arc, config: JevRankerConfig) -> Self { + Self { evaluator, config } } - - /// The configuration in force. + /// Returns the active configuration. #[must_use] pub fn config(&self) -> &JevRankerConfig { &self.config } - - /// Ranks and returns everything the decision learned, not only the hits. + /// Ranks and returns all decision metadata. /// /// # Errors - /// - /// [`RankError::InvalidInput`] for an empty intent or a duplicate - /// candidate key; [`RankError::Timeout`] past the configured deadline; - /// [`RankError::Backend`] for anything the provider or transport did. + /// Returns invalid-input errors and forwards evaluator/retriever failures. pub async fn rank_detailed( &self, intent: &str, @@ -115,9 +49,7 @@ impl JevRanker { ) -> Result { let intent = intent.trim(); if intent.is_empty() { - return Err(RankError::InvalidInput { - reason: "intent is empty".to_owned(), - }); + return Err(RankError::invalid_input("intent is empty")); } if candidates.is_empty() || limit == 0 { return Ok(JevRanking::empty()); @@ -126,157 +58,75 @@ impl JevRanker { if shortlist.is_empty() { return Ok(JevRanking::empty()); } - let request = self.build_request(intent, context, &shortlist)?; let started = std::time::Instant::now(); - let evaluated = tokio::time::timeout(self.config.timeout, self.client.evaluate(&request)) - .await - .map_err(|_elapsed| RankError::Timeout)?; - let result = evaluated.map_err(map_failure)?; - let mut ranking = decode(&result.response, &shortlist, self.config.min_probability)?; + let decision = self.evaluator.evaluate(&request).await?; + let mut ranking = decode(decision, &shortlist, self.config.min_probability); ranking.hits.truncate(limit); - ranking.shortlisted = shortlist.len(); ranking.latency = started.elapsed(); - ranking.attempts = result.attempts; - ranking.input_tokens = result.response.usage.input_tokens; - #[cfg(feature = "tracing")] - tracing::debug!( - target: "tinytools_jev", - shortlisted = shortlist.len(), - hits = ranking.hits.len(), - choice_confidence = ranking.choice_confidence, - needs_tool = ?ranking.needs_tool, - latency_ms = ranking.latency.as_millis() as u64, - attempts = ranking.attempts, - "jev tool ranking" - ); Ok(ranking) } - - /// Narrows `candidates` to what Jev will be shown. async fn shortlist<'a>( &self, intent: &str, context: &RankContext, candidates: &'a [RankCandidate], ) -> Result, RankError> { - let fits_without_retrieval = candidates.len() <= self.config.retrieval_k; - if fits_without_retrieval { + validate_candidates(candidates)?; + let k = self.config.retrieval_k.min(JevRankerConfig::MAX_CANDIDATES); + if candidates.len() <= k { return Ok(candidates.iter().collect()); } let hits = self .config .retriever - .rank(intent, context, candidates, self.config.retrieval_k) + .rank(intent, context, candidates, k) .await?; - let by_key: BTreeMap<&str, &RankCandidate> = candidates + let by_key: BTreeMap<&str, &RankCandidate> = + candidates.iter().map(|c| (c.key.as_str(), c)).collect(); + let shortlist: Vec<_> = hits .iter() - .map(|candidate| (candidate.key.as_str(), candidate)) + .filter_map(|h| by_key.get(h.key.as_str()).copied()) + .take(JevRankerConfig::MAX_CANDIDATES) .collect(); - if by_key.len() != candidates.len() { - return Err(RankError::InvalidInput { - reason: "duplicate candidate key".to_owned(), - }); - } - let shortlist: Vec<&RankCandidate> = hits - .iter() - .filter_map(|hit| by_key.get(hit.key.as_str()).copied()) - .collect(); - // A lexical retriever that finds nothing has met a paraphrase. If the - // whole catalogue fits one Choice, let the decision model see it. - if shortlist.is_empty() && candidates.len() <= JevRankerConfig::MAX_OPTIONS { + if shortlist.is_empty() && candidates.len() <= JevRankerConfig::MAX_CANDIDATES { return Ok(candidates.iter().collect()); } Ok(shortlist) } - fn build_request( &self, intent: &str, context: &RankContext, shortlist: &[&RankCandidate], - ) -> Result { - let mut criteria: BTreeMap> = BTreeMap::new(); - for candidate in shortlist { - if candidate.key == NONE_OPTION { - return Err(RankError::InvalidInput { - reason: format!("candidate key `{NONE_OPTION}` is reserved"), - }); - } - if criteria - .insert(candidate.key.clone(), Some(option_text(candidate))) - .is_some() - { - return Err(RankError::InvalidInput { - reason: "duplicate candidate key".to_owned(), - }); - } + ) -> Result { + if shortlist.len() > JevRankerConfig::MAX_CANDIDATES { + return Err(RankError::invalid_input("too many shortlisted candidates")); } - criteria.insert( - NONE_OPTION.to_owned(), - Some(json!("No listed tool accomplishes the request.")), - ); - - let mut state = json!({ "request": intent }); - if !context.recent_turns.is_empty() - && let Some(object) = state.as_object_mut() - { - object.insert( - "recent_user_turns".to_owned(), - Value::Array( - context - .recent_turns - .iter() - .map(|turn| Value::String(turn.clone())) - .collect(), - ), - ); - } - - let questions = BTreeMap::from([ - ( - TOOL_QUESTION.to_owned(), - Question::Choice(Choice { - instructions: json!( - "Which tool accomplishes the user's `request`? Judge by what \ - each tool does, not by shared words. Pick `none` when no \ - listed tool does it." - ), - criteria, - }), - ), - ( - NEEDS_TOOL_QUESTION.to_owned(), - Question::Noul(Noul { - instructions: json!( - "Does fulfilling the user's `request` require calling a tool \ - — an action or a lookup outside the assistant's own knowledge?" - ), - criteria: Some(NoulCriteria { - r#true: json!( - "The request asks for an action or for information that \ - must be fetched." - ), - r#false: json!("The request can be answered by replying, with no tool."), - }), - }), - ), - ]); - - Ok(EvaluationRequest { - state, + let mut options: Vec<_> = shortlist + .iter() + .map(|c| JevOption { + key: c.key.clone(), + description: option_text(c), + }) + .collect(); + options.push(JevOption { + key: NONE_OPTION.into(), + description: "No listed tool accomplishes the request.".into(), + }); + Ok(JevRequest { + intent: intent.into(), + recent_turns: context.recent_turns.clone(), + options, model: self.config.model.clone(), - questions, }) } } - #[async_trait::async_trait] impl ToolRanker for JevRanker { fn kind(&self) -> &'static str { Self::KIND } - async fn rank( &self, intent: &str, @@ -286,14 +136,13 @@ impl ToolRanker for JevRanker { ) -> Result, RankError> { self.rank_detailed(intent, context, candidates, limit) .await - .map(|ranking| ranking.hits) + .map(|r| r.hits) } } - impl JevRanking { fn empty() -> Self { Self { - hits: Vec::new(), + hits: vec![], choice_confidence: 0.0, needs_tool: None, none_probability: 0.0, @@ -304,78 +153,62 @@ impl JevRanking { } } } - -/// `name: summary`, clipped, with the family named so "the Slack one" ranks. -fn option_text(candidate: &RankCandidate) -> Value { - let mut summary: String = candidate.summary.chars().take(MAX_SUMMARY_CHARS).collect(); - if summary.len() < candidate.summary.len() { - summary.push('…'); - } - match &candidate.family { - Some(family) => json!(format!("{summary} (from {family})")), - None => json!(summary), +fn validate_candidates(candidates: &[RankCandidate]) -> Result<(), RankError> { + let mut keys = BTreeSet::new(); + for c in candidates { + if c.key == NONE_OPTION { + return Err(RankError::invalid_input("candidate key `none` is reserved")); + } + if !keys.insert(c.key.as_str()) { + return Err(RankError::invalid_input("duplicate candidate key")); + } } + Ok(()) } - -fn map_failure(failure: EvaluationFailure) -> RankError { - match failure.error { - JevError::InvalidRequest { reason } | JevError::InvalidConfig { reason } => { - RankError::InvalidInput { reason } - } - JevError::Timeout => RankError::Timeout, - other => RankError::Backend { - // `Display` on every variant is credential-free by the client's - // contract; the transport source is dropped, not printed. - reason: format!("{other} after {} attempt(s)", failure.attempts), - }, +fn option_text(candidate: &RankCandidate) -> String { + let mut summary: String = candidate.summary.chars().take(MAX_SUMMARY_CHARS).collect(); + if candidate.summary.chars().count() > MAX_SUMMARY_CHARS { + summary.push('…'); } + candidate.family.as_ref().map_or(summary.clone(), |family| { + format!("{summary} (from {family})") + }) } - -/// Turns the provider's answers into hits, best first. -fn decode( - response: &tinyjevclient::EvaluationResponse, - shortlist: &[&RankCandidate], - min_probability: f64, -) -> Result { - let Some(Answer::Choice(choice)) = response.answers.get(TOOL_QUESTION) else { - return Err(RankError::Backend { - reason: "response has no choice answer for `tool`".to_owned(), - }); - }; - let needs_tool = match response.answers.get(NEEDS_TOOL_QUESTION) { - Some(Answer::Noul(noul)) => Some(noul.noul), - _ => None, - }; - let none_probability = choice +fn decode(decision: JevDecision, shortlist: &[&RankCandidate], floor: f64) -> JevRanking { + let none = decision .probabilities .get(NONE_OPTION) .copied() .unwrap_or(0.0); - let mut hits: Vec = shortlist + let best = shortlist .iter() - .filter_map(|candidate| { - let probability = *choice.probabilities.get(&candidate.key)?; - (probability >= min_probability).then(|| RankHit { - key: candidate.key.clone(), - score: probability, - confidence: Some(probability), + .filter_map(|c| decision.probabilities.get(&c.key).copied()) + .fold(0.0_f64, f64::max); + let abstained = none >= best || decision.needs_tool.is_some_and(|p| p < 0.5); + let mut hits: Vec<_> = if abstained { + vec![] + } else { + shortlist + .iter() + .filter_map(|c| { + let p = *decision.probabilities.get(&c.key)?; + (p >= floor).then(|| RankHit { + key: c.key.clone(), + score: p, + confidence: Some(p), + }) }) - }) - .collect(); - hits.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.key.cmp(&b.key)) - }); - Ok(JevRanking { + .collect() + }; + hits.sort_by(|a, b| b.score.total_cmp(&a.score).then_with(|| a.key.cmp(&b.key))); + JevRanking { hits, - choice_confidence: choice.confidence, - needs_tool, - none_probability, + choice_confidence: decision.choice_confidence, + needs_tool: decision.needs_tool, + none_probability: none, shortlisted: shortlist.len(), - input_tokens: None, + input_tokens: decision.input_tokens, latency: Duration::ZERO, - attempts: 0, - }) + attempts: decision.attempts, + } } diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index d8352f5..77ae285 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -1,370 +1,110 @@ -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::float_cmp, - clippy::needless_pass_by_value -)] - -use std::{sync::Arc, time::Duration}; - -use serde_json::{Value, json}; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::Mutex, -}; +//! Tests for provider-neutral Jev ranking. use super::*; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; -fn candidates() -> Vec { - vec![ - RankCandidate::new( - "SLACK_SEND_MESSAGE", - "SLACK_SEND_MESSAGE Send a message to a Slack channel or user. channel text", - ) - .with_family("slack"), - RankCandidate::new( - "GMAIL_SEND_EMAIL", - "GMAIL_SEND_EMAIL Send an email from the connected Gmail account. to subject body", - ) - .with_family("gmail"), - RankCandidate::new( - "stock_quote", - "stock_quote Fetch the latest price for a ticker symbol. symbol", - ), - ] +#[derive(Debug)] +struct FakeEvaluator { + decision: JevDecision, + seen: Mutex>, } - -fn response(status: u16, body: &str) -> String { - let reason = if status == 200 { "OK" } else { "Error" }; - format!( - "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ) -} - -fn answer(probabilities: Value, confidence: f64, needs_tool: Option) -> String { - let choice = probabilities - .as_object() - .unwrap() - .iter() - .max_by(|a, b| a.1.as_f64().partial_cmp(&b.1.as_f64()).unwrap()) - .map(|(k, _)| k.clone()) - .unwrap(); - let mut answers = json!({ - "tool": { - "type": "choice", - "choice": choice, - "probabilities": probabilities, - "confidence": confidence +#[async_trait::async_trait] +impl JevEvaluator for FakeEvaluator { + async fn evaluate(&self, request: &JevRequest) -> Result { + if let Ok(mut seen) = self.seen.lock() { + seen.push(request.clone()); } - }); - if let Some(p) = needs_tool { - answers["needs_tool"] = json!({"type": "noul", "noul": p}); + Ok(self.decision.clone()) } - json!({ - "model": "typesafe/jev-1.13", - "answers": answers, - "usage": {"input_tokens": 321, "output_tokens": 4} - }) - .to_string() -} - -/// One-shot loopback server: answers each connection with the next canned -/// response and records the request bodies it saw. -async fn server(responses: Vec) -> (String, Arc>>) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let seen = Arc::new(Mutex::new(Vec::new())); - let recorder = Arc::clone(&seen); - tokio::spawn(async move { - for canned in responses { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut buffer = vec![0_u8; 65_536]; - let mut raw = Vec::new(); - loop { - let read = socket.read(&mut buffer).await.unwrap(); - raw.extend_from_slice(&buffer[..read]); - let text = String::from_utf8_lossy(&raw); - if let Some((head, body)) = text.split_once("\r\n\r\n") { - let length: usize = head - .lines() - .find_map(|line| line.strip_prefix("Content-Length: ")) - .and_then(|v| v.trim().parse().ok()) - .unwrap_or(0); - if body.len() >= length { - recorder - .lock() - .await - .push(serde_json::from_str(body).unwrap()); - break; - } - } - if read == 0 { - break; - } - } - socket.write_all(canned.as_bytes()).await.unwrap(); - socket.shutdown().await.unwrap(); - } - }); - (format!("http://{address}"), seen) } -fn ranker(base_url: String, config: JevRankerConfig) -> JevRanker { - let mut client_config = ClientConfig::openrouter("test-key"); - client_config.base_url = base_url; - client_config.retry = RetryPolicy { - max_retries: 0, - ..RetryPolicy::default() - }; - JevRanker::from_config(client_config, config).unwrap() +fn candidates() -> Vec { + vec![ + RankCandidate::new("slack", "Send a Slack message").with_family("chat"), + RankCandidate::new("gmail", "Send an email"), + ] } -#[tokio::test] -async fn small_catalogue_goes_straight_to_jev_with_a_none_option() { - let (url, seen) = server(vec![response( - 200, - &answer( - json!({"SLACK_SEND_MESSAGE": 0.83, "GMAIL_SEND_EMAIL": 0.12, "stock_quote": 0.01, "none": 0.04}), - 0.79, - Some(0.97), - ), - )]) - .await; - let ranker = ranker(url, JevRankerConfig::new()); - - let ranking = ranker - .rank_detailed( - "ping alex on slack that I'm ten minutes late", - &RankContext::empty(), - &candidates(), - 3, - ) - .await - .unwrap(); - - assert_eq!( - ranking - .hits - .iter() - .map(|h| h.key.as_str()) - .collect::>(), - vec!["SLACK_SEND_MESSAGE", "GMAIL_SEND_EMAIL"], - "stock_quote sits below min_probability and none is never a hit" - ); - assert_eq!(ranking.hits[0].confidence, Some(0.83)); - assert_eq!(ranking.choice_confidence, 0.79); - assert_eq!(ranking.needs_tool, Some(0.97)); - assert_eq!(ranking.none_probability, 0.04); - assert_eq!(ranking.shortlisted, 3); - assert_eq!(ranking.input_tokens, Some(321)); - assert_eq!(ranking.attempts, 1); - - let request = &seen.lock().await[0]; - assert_eq!(request["model"], "jev-latest"); - assert_eq!( - request["state"]["request"], - "ping alex on slack that I'm ten minutes late" - ); - let criteria = request["questions"]["tool"]["criteria"] - .as_object() - .unwrap(); - assert_eq!(criteria.len(), 4, "three candidates plus `none`"); - assert!( - criteria["SLACK_SEND_MESSAGE"] - .as_str() - .unwrap() - .ends_with("(from slack)") - ); - assert_eq!(request["questions"]["needs_tool"]["type"], "noul"); +fn ranker( + probabilities: [(&str, f64); 3], + needs_tool: Option, +) -> (JevRanker, Arc) { + let evaluator = Arc::new(FakeEvaluator { + decision: JevDecision { + probabilities: probabilities + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect::>(), + choice_confidence: 0.8, + needs_tool, + input_tokens: Some(10), + attempts: 1, + }, + seen: Mutex::new(vec![]), + }); + ( + JevRanker::new(evaluator.clone(), JevRankerConfig::new()), + evaluator, + ) } #[tokio::test] -async fn large_catalogue_is_retrieved_first_then_decided() { - let (url, seen) = server(vec![response( - 200, - &answer( - json!({"t_send_7": 0.9, "t_send_3": 0.06, "none": 0.04}), - 0.85, - Some(0.9), - ), - )]) - .await; - let mut catalogue: Vec = (0..40) - .map(|i| { - RankCandidate::new( - format!("t_read_{i}"), - format!("t_read_{i} Read record {i}."), - ) - }) - .collect(); - catalogue.push(RankCandidate::new( - "t_send_7", - "t_send_7 Send a message to a person.", - )); - catalogue.push(RankCandidate::new( - "t_send_3", - "t_send_3 Send a message to a channel.", - )); - let ranker = ranker(url, JevRankerConfig::new().with_retrieval_k(5)); - - let ranking = ranker - .rank_detailed("send a message", &RankContext::empty(), &catalogue, 3) +async fn ranks_candidates_and_builds_none_option() { + let (ranker, evaluator) = ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.9)); + let result = ranker + .rank_detailed("message Alex", &RankContext::empty(), &candidates(), 2) .await; - let ranking = ranking.unwrap(); - - assert_eq!(ranking.hits[0].key, "t_send_7"); - assert_eq!(ranking.needs_tool, Some(0.9)); - let request = &seen.lock().await[0]; - let criteria = request["questions"]["tool"]["criteria"] - .as_object() - .unwrap(); - assert!( - criteria.len() <= 6, - "shortlist of at most 5 plus `none`, got {}", - criteria.len() - ); - assert!(criteria.contains_key("t_send_7")); - assert!(criteria.contains_key("t_send_3")); - assert_eq!(ranking.shortlisted, criteria.len() - 1); -} - -#[tokio::test] -async fn retriever_miss_on_a_small_catalogue_still_lets_jev_decide() { - let (url, seen) = server(vec![response( - 200, - &answer( - json!({"SLACK_SEND_MESSAGE": 0.7, "GMAIL_SEND_EMAIL": 0.2, "stock_quote": 0.05, "none": 0.05}), - 0.6, - Some(0.8), - ), - )]) - .await; - // retrieval_k of 1 forces retrieval; "ping" matches nothing lexically. - let ranker = ranker(url, JevRankerConfig::new().with_retrieval_k(1)); - - let hits = ranker - .rank("ping alex", &RankContext::empty(), &candidates(), 3) - .await - .unwrap(); - - assert_eq!(hits[0].key, "SLACK_SEND_MESSAGE"); - let request = &seen.lock().await[0]; - let criteria = request["questions"]["tool"]["criteria"] - .as_object() - .unwrap(); assert_eq!( - criteria.len(), - 4, - "the whole catalogue was shown after the retriever missed" + result + .as_ref() + .ok() + .and_then(|ranking| ranking.hits.first()) + .map(|hit| hit.key.as_str()), + Some("slack") ); + let options = evaluator + .seen + .lock() + .ok() + .and_then(|seen| seen.first().map(|request| request.options.len())); + assert_eq!(options, Some(3)); } #[tokio::test] -async fn empty_inputs_never_reach_the_network() { - let ranker = ranker("http://127.0.0.1:9".to_owned(), JevRankerConfig::new()); - assert!( - ranker - .rank("anything", &RankContext::empty(), &[], 3) - .await - .unwrap() - .is_empty() - ); - assert!( - ranker - .rank("anything", &RankContext::empty(), &candidates(), 0) - .await - .unwrap() - .is_empty() - ); - let err = ranker - .rank(" ", &RankContext::empty(), &candidates(), 3) +async fn suppresses_hits_when_none_wins_or_no_tool_is_needed() { + let (none_ranker, _) = ranker([("slack", 0.2), ("gmail", 0.1), ("none", 0.7)], Some(0.9)); + let none_hits = none_ranker + .rank("question", &RankContext::empty(), &candidates(), 2) .await - .unwrap_err(); - assert!(matches!(err, RankError::InvalidInput { .. }), "{err}"); -} - -#[tokio::test] -async fn reserved_and_duplicate_keys_are_rejected_before_sending() { - let ranker = ranker("http://127.0.0.1:9".to_owned(), JevRankerConfig::new()); - let reserved = vec![RankCandidate::new("none", "none nothing")]; - let err = ranker - .rank("x", &RankContext::empty(), &reserved, 3) - .await - .unwrap_err(); - assert_eq!( - err.to_string(), - "invalid ranking input: candidate key `none` is reserved" - ); - let duplicate = vec![ - RankCandidate::new("a", "a one"), - RankCandidate::new("a", "a two"), - ]; - let err = ranker - .rank("x", &RankContext::empty(), &duplicate, 3) + .unwrap_or_default(); + assert!(none_hits.is_empty()); + let (no_tool_ranker, _) = ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.2)); + let no_tool_hits = no_tool_ranker + .rank("question", &RankContext::empty(), &candidates(), 2) .await - .unwrap_err(); - assert_eq!( - err.to_string(), - "invalid ranking input: duplicate candidate key" - ); -} - -#[tokio::test] -async fn provider_failures_become_backend_errors_without_the_key() { - let (url, _) = server(vec![response(401, r#"{"error":"nope"}"#)]).await; - let ranker = ranker(url, JevRankerConfig::new()); - let err = ranker - .rank("send a message", &RankContext::empty(), &candidates(), 3) - .await - .unwrap_err(); - let text = err.to_string(); - assert!(matches!(err, RankError::Backend { .. }), "{text}"); - assert!(text.contains("authentication failed"), "{text}"); - assert!( - !text.contains("test-key"), - "the key must never surface: {text}" - ); -} - -#[tokio::test] -async fn the_deadline_is_enforced() { - // A listener that accepts and never answers. - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { - let (_socket, _) = listener.accept().await.unwrap(); - tokio::time::sleep(Duration::from_secs(30)).await; - }); - let ranker = ranker( - url, - JevRankerConfig::new().with_timeout(Duration::from_millis(200)), - ); - let err = ranker - .rank("send a message", &RankContext::empty(), &candidates(), 3) - .await - .unwrap_err(); - assert!(matches!(err, RankError::Timeout), "{err}"); + .unwrap_or_default(); + assert!(no_tool_hits.is_empty()); } #[test] -fn option_text_clips_long_summaries_and_names_the_family() { - let long = "x".repeat(400); - let candidate = RankCandidate::new("k", long).with_family("fam"); - let text = option_text(&candidate); - let text = text.as_str().unwrap(); - assert!(text.starts_with(&"x".repeat(MAX_SUMMARY_CHARS))); - assert!(text.ends_with("… (from fam)")); +fn configuration_reserves_none_slot_and_rejects_nan() { + let config = JevRankerConfig::new() + .with_retrieval_k(usize::MAX) + .with_min_probability(f64::NAN); + assert_eq!(config.retrieval_k, JevRankerConfig::MAX_CANDIDATES); + assert_eq!(config.min_probability, 0.05); } #[test] -fn config_debug_never_prints_a_client_and_clamps_knobs() { - let config = JevRankerConfig::new() - .with_retrieval_k(9_999) - .with_min_probability(7.0); - assert_eq!(config.retrieval_k, JevRankerConfig::MAX_OPTIONS); - assert_eq!(config.min_probability, 1.0); - assert!(format!("{config:?}").contains("bm25")); +fn option_text_clips_by_characters() { + let candidate = RankCandidate::new("key", "é".repeat(300)).with_family("family"); + let text = option_text(&candidate); + assert_eq!( + text.chars().filter(|character| *character == 'é').count(), + MAX_SUMMARY_CHARS + ); + assert!(text.ends_with("… (from family)")); } diff --git a/crates/tinytools-jev/src/types.rs b/crates/tinytools-jev/src/types.rs index 09cbd4f..bd5b3d3 100644 --- a/crates/tinytools-jev/src/types.rs +++ b/crates/tinytools-jev/src/types.rs @@ -1,127 +1,139 @@ -//! Configuration and the detailed answer a Jev ranking produces. - -use std::{fmt, sync::Arc, time::Duration}; +//! Provider-neutral request, response, and configuration types. +use std::{collections::BTreeMap, fmt, sync::Arc}; use tinytools::{Bm25Ranker, RankHit, ToolRanker}; +/// One candidate presented to an evaluator. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JevOption { + /// Opaque candidate key. + pub key: String, + /// Concise description the evaluator judges. + pub description: String, +} + +/// A provider-neutral tool-selection request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JevRequest { + /// The user's request. + pub intent: String, + /// Recent user turns, oldest first. + pub recent_turns: Vec, + /// Candidate tools, including `none`. + pub options: Vec, + /// Model identifier configured by the host. + pub model: String, +} + +/// The decision returned by a host-provided evaluator. +#[derive(Clone, Debug, PartialEq)] +pub struct JevDecision { + /// Probability for each option key. + pub probabilities: BTreeMap, + /// Confidence in the overall choice. + pub choice_confidence: f64, + /// Probability that the request needs a tool. + pub needs_tool: Option, + /// Input tokens billed, when reported. + pub input_tokens: Option, + /// Attempts made by the host client. + pub attempts: u32, +} + +/// Evaluates a request without coupling this workspace to a transport. +#[async_trait::async_trait] +pub trait JevEvaluator: Send + Sync + fmt::Debug { + /// Evaluates one tool-selection request. + /// + /// # Errors + /// Returns a ranking error for invalid requests, timeouts, or provider failures. + async fn evaluate(&self, request: &JevRequest) -> Result; +} + /// How [`JevRanker`][crate::JevRanker] retrieves and decides. #[derive(Clone)] pub struct JevRankerConfig { - /// Ranks the full catalogue down to a shortlist before Jev sees it. - /// [`Bm25Ranker`] by default; a host with an embedding index passes that. - pub retriever: Arc, - /// How many candidates the retriever hands to Jev. Twenty is the - /// documented sweet spot: small enough that one Choice question decides - /// in ~150 ms, large enough that a lexical retriever's recall is not the - /// bottleneck. Never above [`Self::MAX_OPTIONS`]. - pub retrieval_k: usize, - /// Hits whose probability falls below this are dropped, so a caller - /// never sees the long tail of a distribution as if it were a match. - pub min_probability: f64, - /// Deadline for the decision call, on top of the client's own per-attempt - /// timeout and retries. A tool search sits in a model's turn; a slow - /// answer is worse than a fallback. - pub timeout: Duration, - /// System One model id. `jev-latest` unless a host pins one. - pub model: String, + pub(crate) retriever: Arc, + pub(crate) retrieval_k: usize, + pub(crate) min_probability: f64, + pub(crate) model: String, } impl JevRankerConfig { - /// The most options one Jev Choice question accepts. + /// Maximum options accepted, including `none`. pub const MAX_OPTIONS: usize = 255; - - /// Defaults: BM25 retrieval to 20, `min_probability` 0.05, 3 s deadline, - /// `jev-latest`. + /// Maximum real candidates, reserving one slot for `none`. + pub const MAX_CANDIDATES: usize = Self::MAX_OPTIONS - 1; + /// Returns the default configuration. #[must_use] pub fn new() -> Self { Self { retriever: Arc::new(Bm25Ranker), retrieval_k: 20, min_probability: 0.05, - timeout: Duration::from_secs(3), - model: "jev-latest".to_owned(), + model: "jev-latest".into(), } } - /// Replaces the retriever. #[must_use] - pub fn with_retriever(mut self, retriever: Arc) -> Self { - self.retriever = retriever; - self - } - - /// Sets the shortlist size, clamped to `1..=MAX_OPTIONS`. - #[must_use] - pub fn with_retrieval_k(mut self, k: usize) -> Self { - self.retrieval_k = k.clamp(1, Self::MAX_OPTIONS); + pub fn with_retriever(mut self, value: Arc) -> Self { + self.retriever = value; self } - - /// Sets the probability floor, clamped to `0.0..=1.0`. + /// Sets the shortlist size, clamped to the valid candidate range. #[must_use] - pub fn with_min_probability(mut self, p: f64) -> Self { - self.min_probability = p.clamp(0.0, 1.0); + pub fn with_retrieval_k(mut self, value: usize) -> Self { + self.retrieval_k = value.clamp(1, Self::MAX_CANDIDATES); self } - - /// Sets the decision deadline. + /// Sets a finite probability floor, clamped to `0.0..=1.0`. #[must_use] - pub fn with_timeout(mut self, timeout: Duration) -> Self { - self.timeout = timeout; + pub fn with_min_probability(mut self, value: f64) -> Self { + if value.is_finite() { + self.min_probability = value.clamp(0.0, 1.0); + } self } - /// Sets the model id. #[must_use] - pub fn with_model(mut self, model: impl Into) -> Self { - self.model = model.into(); + pub fn with_model(mut self, value: impl Into) -> Self { + self.model = value.into(); self } } - impl Default for JevRankerConfig { fn default() -> Self { Self::new() } } - impl fmt::Debug for JevRankerConfig { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("JevRankerConfig") .field("retriever", &self.retriever.kind()) .field("retrieval_k", &self.retrieval_k) .field("min_probability", &self.min_probability) - .field("timeout", &self.timeout) .field("model", &self.model) .finish() } } -/// Everything one Jev ranking learned, beyond the hits the trait returns. -/// -/// A caller that only wants the hits uses [`tinytools::ToolRanker::rank`]; -/// one that gates on "does this need a tool at all", or that reports cost, -/// calls [`JevRanker::rank_detailed`][crate::JevRanker::rank_detailed]. +/// Everything one ranking learned, beyond its hits. #[derive(Clone, Debug, PartialEq)] pub struct JevRanking { - /// Ranked hits, best first; each `confidence` is Jev's probability for - /// that option. + /// Ranked hits. pub hits: Vec, - /// Jev's confidence in the Choice as a whole, `0.0..=1.0`. Low when the - /// distribution is flat — the signal to prefer asking over acting. + /// Confidence in the choice. pub choice_confidence: f64, - /// Probability that the request needs a tool at all, from the `Noul` - /// asked alongside. `None` when Jev did not answer it. + /// Probability that the request needs a tool. pub needs_tool: Option, - /// Probability Jev put on "none of these", the option every request - /// carries so an off-catalogue intent is not forced onto a tool. + /// Probability assigned to `none`. pub none_probability: f64, - /// How many candidates the retriever handed to Jev. + /// Candidate count shown. pub shortlisted: usize, - /// Input tokens billed, when the provider reports them. + /// Input tokens billed, when reported. pub input_tokens: Option, - /// Wall time of the decision call, including the client's retries. - pub latency: Duration, - /// Attempts the client made. + /// Evaluator wall time. + pub latency: std::time::Duration, + /// Host-client attempt count. pub attempts: u32, } diff --git a/crates/tinytools/src/lib.rs b/crates/tinytools/src/lib.rs index f9fb058..7564d34 100644 --- a/crates/tinytools/src/lib.rs +++ b/crates/tinytools/src/lib.rs @@ -127,7 +127,9 @@ pub use permission::PermissionLevel; pub use policy::{ ToolAccess, ToolDisplay, ToolPolicy, ToolReplay, ToolRuntime, ToolSideEffects, WorkspaceAccess, }; -pub use rank::{Bm25Ranker, RankCandidate, RankContext, RankError, RankHit, ToolRanker}; +pub use rank::{ + Bm25Index, Bm25Ranker, RankCandidate, RankContext, RankError, RankHit, ToolRanker, tokenize, +}; pub use result::{FileData, ImageData, ToolContent, ToolControl, ToolErrorKind, ToolResult}; pub use spec::ToolSpec; pub use tool::{Tool, ToolExposure}; diff --git a/crates/tinytools/src/rank/test.rs b/crates/tinytools/src/rank/test.rs index 3c81149..668c0d0 100644 --- a/crates/tinytools/src/rank/test.rs +++ b/crates/tinytools/src/rank/test.rs @@ -1,5 +1,3 @@ -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - use super::*; fn candidates() -> Vec { @@ -100,8 +98,11 @@ async fn ranker_is_object_safe_behind_an_arc() { let hits = ranker .rank("calendar invite", &RankContext::empty(), &candidates(), 3) .await - .unwrap(); - assert_eq!(hits[0].key, "calendar_invite"); + .unwrap_or_default(); + assert_eq!( + hits.first().map(|hit| hit.key.as_str()), + Some("calendar_invite") + ); } #[test] diff --git a/crates/tinytools/src/rank/types.rs b/crates/tinytools/src/rank/types.rs index 44382d2..352ad27 100644 --- a/crates/tinytools/src/rank/types.rs +++ b/crates/tinytools/src/rank/types.rs @@ -115,6 +115,24 @@ pub enum RankError { Timeout, } +impl RankError { + /// Creates a backend failure with a log-safe description. + #[must_use] + pub fn backend(reason: impl Into) -> Self { + Self::Backend { + reason: reason.into(), + } + } + + /// Creates an invalid-input failure. + #[must_use] + pub fn invalid_input(reason: impl Into) -> Self { + Self::InvalidInput { + reason: reason.into(), + } + } +} + impl fmt::Display for RankError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { From 0b43a13d2269cf5e47cd7902ed08f356f2a61842 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 04:59:55 +0300 Subject: [PATCH 4/7] refactor(jev): take decision by reference in decode The `decode` function now borrows the `JevDecision` instead of taking ownership, avoiding an unnecessary clone at the call site. This aligns the function's signature with its read-only usage and improves efficiency. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-jev/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinytools-jev/src/lib.rs b/crates/tinytools-jev/src/lib.rs index cff3a9b..18bb197 100644 --- a/crates/tinytools-jev/src/lib.rs +++ b/crates/tinytools-jev/src/lib.rs @@ -61,7 +61,7 @@ impl JevRanker { let request = self.build_request(intent, context, &shortlist)?; let started = std::time::Instant::now(); let decision = self.evaluator.evaluate(&request).await?; - let mut ranking = decode(decision, &shortlist, self.config.min_probability); + let mut ranking = decode(&decision, &shortlist, self.config.min_probability); ranking.hits.truncate(limit); ranking.latency = started.elapsed(); Ok(ranking) @@ -174,7 +174,7 @@ fn option_text(candidate: &RankCandidate) -> String { format!("{summary} (from {family})") }) } -fn decode(decision: JevDecision, shortlist: &[&RankCandidate], floor: f64) -> JevRanking { +fn decode(decision: &JevDecision, shortlist: &[&RankCandidate], floor: f64) -> JevRanking { let none = decision .probabilities .get(NONE_OPTION) From 557b9c60755cd6afca3dc6ba3aa2dcaf94e4ff3b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:00:09 +0300 Subject: [PATCH 5/7] fix(test): compare min_probability with epsilon The test previously asserted exact equality for a floating-point value, which can be flaky due to rounding. It now checks that the value is within a small epsilon of the expected 0.05, making the test more robust. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-jev/src/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index 77ae285..2f0e739 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -95,7 +95,7 @@ fn configuration_reserves_none_slot_and_rejects_nan() { .with_retrieval_k(usize::MAX) .with_min_probability(f64::NAN); assert_eq!(config.retrieval_k, JevRankerConfig::MAX_CANDIDATES); - assert_eq!(config.min_probability, 0.05); + assert!((config.min_probability - 0.05).abs() < f64::EPSILON); } #[test] From 6695eb03515202816e62eeb336f3645d5275924f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:03:29 +0300 Subject: [PATCH 6/7] test(rank): add coverage for input validation and error paths Add tests for the Jev ranker covering input validation, short-circuiting on empty candidate lists, retrieval behavior with large catalogues, and error forwarding from the evaluator. Also extend the rank error display test to cover backend and invalid input variants. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-jev/src/test.rs | 99 +++++++++++++++++++++++++++++++ crates/tinytools/src/rank/test.rs | 8 +++ 2 files changed, 107 insertions(+) diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index 2f0e739..ec0ae74 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -108,3 +108,102 @@ fn option_text_clips_by_characters() { ); assert!(text.ends_with("… (from family)")); } + +#[tokio::test] +async fn validates_inputs_and_short_circuits_empty_work() { + let (ranker, _) = ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.9)); + let empty = ranker + .rank("request", &RankContext::empty(), &[], 2) + .await + .unwrap_or_default(); + assert!(empty.is_empty()); + let zero = ranker + .rank("request", &RankContext::empty(), &candidates(), 0) + .await + .unwrap_or_default(); + assert!(zero.is_empty()); + assert!(matches!( + ranker + .rank(" ", &RankContext::empty(), &candidates(), 2) + .await, + Err(RankError::InvalidInput { .. }) + )); + let reserved = vec![RankCandidate::new("none", "reserved")]; + assert!(matches!( + ranker + .rank("request", &RankContext::empty(), &reserved, 2) + .await, + Err(RankError::InvalidInput { .. }) + )); + let duplicate = vec![ + RankCandidate::new("x", "one"), + RankCandidate::new("x", "two"), + ]; + assert!(matches!( + ranker + .rank("request", &RankContext::empty(), &duplicate, 2) + .await, + Err(RankError::InvalidInput { .. }) + )); +} + +#[tokio::test] +async fn retrieves_large_catalogues_and_handles_a_retrieval_miss() { + let (_initial_ranker, evaluator) = + ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.9)); + let configured_ranker = JevRanker::new(evaluator, JevRankerConfig::new().with_retrieval_k(1)); + let result = configured_ranker + .rank("Slack message", &RankContext::empty(), &candidates(), 2) + .await + .unwrap_or_default(); + assert_eq!(result.first().map(|hit| hit.key.as_str()), Some("slack")); + + let (miss_ranker, evaluator) = + ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.9)); + let miss_ranker = JevRanker::new( + evaluator.clone(), + JevRankerConfig::new().with_retrieval_k(1), + ); + let _ = miss_ranker + .rank("unrelated", &RankContext::empty(), &candidates(), 2) + .await; + let shown = evaluator + .seen + .lock() + .ok() + .and_then(|seen| seen.first().map(|request| request.options.len())); + assert_eq!(shown, Some(3)); +} + +#[derive(Debug)] +struct FailingEvaluator; +#[async_trait::async_trait] +impl JevEvaluator for FailingEvaluator { + async fn evaluate(&self, _request: &JevRequest) -> Result { + Err(RankError::backend("provider unavailable")) + } +} + +#[tokio::test] +async fn forwards_evaluator_errors() { + let ranker = JevRanker::new(Arc::new(FailingEvaluator), JevRankerConfig::default()); + let result = ranker + .rank("message", &RankContext::empty(), &candidates(), 2) + .await; + assert!(matches!(result, Err(RankError::Backend { .. }))); + assert_eq!(ranker.kind(), JevRanker::KIND); + assert!(format!("{:?}", ranker.config()).contains("bm25")); +} + +#[test] +fn configuration_builders_are_observable() { + let retriever: Arc = Arc::new(tinytools::Bm25Ranker); + let config = JevRankerConfig::new() + .with_retriever(retriever) + .with_retrieval_k(0) + .with_min_probability(2.0) + .with_model("jev-pinned"); + assert_eq!(config.retrieval_k, 1); + assert!((config.min_probability - 1.0).abs() < f64::EPSILON); + assert_eq!(config.model, "jev-pinned"); +} diff --git a/crates/tinytools/src/rank/test.rs b/crates/tinytools/src/rank/test.rs index 668c0d0..25ad1cf 100644 --- a/crates/tinytools/src/rank/test.rs +++ b/crates/tinytools/src/rank/test.rs @@ -112,4 +112,12 @@ fn rank_error_displays_without_credentials() { }; assert_eq!(err.to_string(), "ranker backend failed: status 401"); assert_eq!(RankError::Timeout.to_string(), "ranker timed out"); + assert_eq!( + RankError::backend("offline").to_string(), + "ranker backend failed: offline" + ); + assert_eq!( + RankError::invalid_input("bad key").to_string(), + "invalid ranking input: bad key" + ); } From 03649ca20074974b8793487e313ec2afbb9fb945 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:03:59 +0300 Subject: [PATCH 7/7] fix(test): rename unused variable in retrieval miss test The variable `miss_ranker` was assigned but never used, so it has been renamed to `_unused_ranker` to suppress the unused variable warning and clarify its purpose in the test. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-jev/src/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinytools-jev/src/test.rs b/crates/tinytools-jev/src/test.rs index ec0ae74..5ba6355 100644 --- a/crates/tinytools-jev/src/test.rs +++ b/crates/tinytools-jev/src/test.rs @@ -158,7 +158,7 @@ async fn retrieves_large_catalogues_and_handles_a_retrieval_miss() { .unwrap_or_default(); assert_eq!(result.first().map(|hit| hit.key.as_str()), Some("slack")); - let (miss_ranker, evaluator) = + let (_unused_ranker, evaluator) = ranker([("slack", 0.8), ("gmail", 0.1), ("none", 0.1)], Some(0.9)); let miss_ranker = JevRanker::new( evaluator.clone(),