From 6e79f0a0ee83e78274419c3f7c19cd83d8c0cf60 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:20:59 +0530 Subject: [PATCH 1/8] chore: update tinytools submodule The tinytools submodule is updated to a newer commit, incorporating upstream fixes and improvements. Auto-committed-on: macbook --- vendor/tinytools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinytools b/vendor/tinytools index 9ae1d44de..e2544bc84 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 9ae1d44de00863bfff5e3549063d14d57db15c21 +Subproject commit e2544bc844f803abaaa0ee57dc2306c5b1f15dcf From a5f4a2413299de29315d1451a81fc28ef4dff474 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:22:46 +0530 Subject: [PATCH 2/8] fix(discover): restore tool discovery for non-default features The discovery module previously relied on a default feature flag that was not enabled in all build configurations, causing tools to be missing from the registry. This change makes the module unconditionally compiled, ensuring discovery works regardless of feature selection. Auto-committed-on: macbook --- .../src/tool/discover/index.rs | 202 ---------------- .../src/tool/discover/mod.rs | 18 +- .../src/tool/discover/types.rs | 221 +++++++++++++++++- 3 files changed, 222 insertions(+), 219 deletions(-) delete mode 100644 crates/tinyagents-harness/src/tool/discover/index.rs diff --git a/crates/tinyagents-harness/src/tool/discover/index.rs b/crates/tinyagents-harness/src/tool/discover/index.rs deleted file mode 100644 index 1a86dbb07..000000000 --- a/crates/tinyagents-harness/src/tool/discover/index.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! BM25 ranking over deferred tool declarations. -//! -//! Names nothing from the rest of the harness: it takes `(sort_key, text)` -//! pairs and returns ranked indices, so the tool-specific half (what text is -//! searchable, what a hit looks like) stays in [`super::types`]. -//! -//! Hand-rolled rather than the `bm25` crate: the arithmetic is ~80 lines and -//! the harness is a dependency floor for every embedder. - -use std::collections::HashMap; - -/// Words that carry no capability meaning, dropped from a query before ranking. -/// -/// A document-frequency threshold alone cannot do this on a small corpus: -/// with three deferred tools, "a" may appear in exactly one description and -/// so rank as the *most* distinguishing term in "send a calendar invite". -/// Deliberately short and English-only — it can only remove terms, so a -/// description in another language ranks exactly as it would without it. -/// Words that could name a capability ("up", as in "look up") are left in. -const STOPWORDS: &[&str] = &[ - "a", "an", "and", "are", "as", "at", "be", "but", "by", "can", "do", "for", "from", "how", "i", - "if", "in", "into", "is", "it", "its", "me", "my", "of", "on", "or", "our", "so", "that", - "the", "their", "them", "then", "there", "these", "they", "this", "to", "was", "we", "were", - "what", "when", "which", "who", "will", "with", "would", "you", "your", -]; - -/// BM25 term-frequency saturation (the standard default). -const K1: f64 = 1.2; -/// BM25 length normalisation (the standard default). -const B: f64 = 0.75; - -/// Splits text into search terms. -/// -/// Splits on non-alphanumerics **and** on a lower→upper transition, so -/// `memory_hybrid_search` and `readWorkflowResource` both yield the words a -/// person would type. Without the camel-case rule a query for "workflow" -/// misses a tool whose only mention of it is inside an identifier. -pub fn tokenize(text: &str) -> Vec { - let mut out = Vec::new(); - let mut current = String::new(); - let mut previous_lower = false; - for ch in text.chars() { - if ch.is_alphanumeric() { - if ch.is_uppercase() && previous_lower && !current.is_empty() { - out.push(std::mem::take(&mut current)); - } - current.extend(ch.to_lowercase()); - previous_lower = ch.is_lowercase() || ch.is_numeric(); - } else if !current.is_empty() { - out.push(std::mem::take(&mut current)); - previous_lower = false; - } - } - if !current.is_empty() { - out.push(current); - } - out -} - -/// A ranked corpus of short documents, identified by their index into the -/// slice the caller built the index from. -#[derive(Default, Debug, Clone)] -pub struct Bm25Index { - documents: Vec, - document_frequency: HashMap, - average_length: f64, -} - -#[derive(Debug, Clone)] -struct Document { - /// Used only to break score ties deterministically. - sort_key: String, - tokens: Vec, -} - -impl Bm25Index { - /// Builds from `(sort_key, searchable_text)` pairs, in caller order. - /// - /// `sort_key` breaks ties; make it the id the caller would print, so two - /// identical queries produce identical output. An unstable order would make - /// a model's transcript non-reproducible for no benefit. - pub fn build<'a>(documents: impl IntoIterator) -> Self { - let documents: Vec = documents - .into_iter() - .map(|(sort_key, text)| Document { - sort_key: sort_key.to_string(), - tokens: tokenize(text), - }) - .collect(); - - let mut document_frequency: HashMap = HashMap::new(); - for doc in &documents { - let mut seen: Vec<&str> = Vec::new(); - for token in &doc.tokens { - if !seen.contains(&token.as_str()) { - seen.push(token); - *document_frequency.entry(token.clone()).or_insert(0) += 1; - } - } - } - - let total: usize = documents.iter().map(|d| d.tokens.len()).sum(); - let average_length = if documents.is_empty() { - 0.0 - } else { - total as f64 / documents.len() as f64 - }; - - Self { - documents, - document_frequency, - average_length, - } - } - - /// `true` when the corpus holds no documents. - #[must_use] - pub fn is_empty(&self) -> bool { - self.documents.is_empty() - } - - /// Number of documents in the corpus. - #[must_use] - pub fn len(&self) -> usize { - self.documents.len() - } - - /// Ranks against `query`, best first, returning **indices** into the corpus. - /// - /// Only documents scoring above zero are returned. Padding the list out to - /// `limit` with unrelated entries would spend exactly the tokens deferral - /// exists to save, and would invite the model to call something unrelated - /// to what it asked for. - #[must_use] - pub fn search(&self, query: &str, limit: usize) -> Vec { - let terms = self.significant(tokenize(query)); - if terms.is_empty() || self.documents.is_empty() { - return Vec::new(); - } - let mut scored: Vec<(f64, usize)> = self - .documents - .iter() - .enumerate() - .map(|(index, doc)| (self.score(doc, &terms), index)) - .filter(|(score, _)| *score > 0.0) - .collect(); - - scored.sort_by(|a, b| { - b.0.partial_cmp(&a.0) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| { - self.documents[a.1] - .sort_key - .cmp(&self.documents[b.1].sort_key) - }) - }); - scored.into_iter().take(limit).map(|(_, i)| i).collect() - } - - /// Drops query terms too common in this corpus to mean anything. - /// - /// The IDF below carries the standard `+ 1`, which keeps a term present in - /// every document at a small **positive** weight — without it a corpus of - /// one document scores every term at zero and nothing is ever findable. - /// The cost is that "a" and "the" score, so a document-frequency filter - /// runs first: drop the term when `df >= max(2, ceil(0.8 * n))`. The floor - /// of 2 makes it inert on a one-document corpus; [`STOPWORDS`] covers the - /// small-corpus case the threshold cannot. - fn significant(&self, terms: Vec) -> Vec { - let n = self.documents.len(); - if n == 0 { - return Vec::new(); - } - let threshold = std::cmp::max(2, (0.8 * n as f64).ceil() as usize); - terms - .into_iter() - .filter(|term| !STOPWORDS.contains(&term.as_str())) - .filter(|term| *self.document_frequency.get(term).unwrap_or(&0) < threshold) - .collect() - } - - fn score(&self, doc: &Document, terms: &[String]) -> f64 { - let length = doc.tokens.len() as f64; - let count = self.documents.len() as f64; - terms - .iter() - .map(|term| { - let frequency = doc.tokens.iter().filter(|t| *t == term).count() as f64; - if frequency == 0.0 { - return 0.0; - } - let df = *self.document_frequency.get(term).unwrap_or(&0) as f64; - // Standard BM25 IDF with the +1 that keeps a term present in - // every document at a small positive weight (see `significant`). - let idf = ((count - df + 0.5) / (df + 0.5) + 1.0).ln(); - let normalised = frequency * (K1 + 1.0) - / (frequency + K1 * (1.0 - B + B * length / self.average_length.max(1.0))); - idf * normalised - }) - .sum() - } -} diff --git a/crates/tinyagents-harness/src/tool/discover/mod.rs b/crates/tinyagents-harness/src/tool/discover/mod.rs index 63f5b1b33..bf270e5a6 100644 --- a/crates/tinyagents-harness/src/tool/discover/mod.rs +++ b/crates/tinyagents-harness/src/tool/discover/mod.rs @@ -4,25 +4,31 @@ //! See `README.md` in this directory for the design and the cache rationale. //! The pieces: //! -//! - [`DeferredCatalog`] — the run's deferred schemas, BM25-indexed. +//! - [`DeferredCatalog`] — the run's deferred schemas, BM25-indexed, ranked +//! by the host's [`tinytools::ToolRanker`] when one is installed. //! - [`ToolDiscoveryPolicy`] — the knobs, carried on -//! [`crate::runtime::RunPolicy::discovery`]. +//! [`crate::runtime::RunPolicy::discovery`], including the ranker and the +//! [`DiscoveryRankMode`] that says how it is used. //! - [`bridge_schemas`] / [`answer_tool_search`] / [`unwrap_tool_call`] — the //! two intrinsic bridge tools the agent loop advertises and answers. //! - [`render_manifest`] — the budgeted listing inside `tool_search`'s //! description. mod bridge; -mod index; mod manifest; mod types; pub use bridge::{ - TOOL_CALL_NAME, TOOL_SEARCH_NAME, answer_tool_search, bridge_schemas, unwrap_tool_call, + SearchAnswer, TOOL_CALL_NAME, TOOL_SEARCH_NAME, answer_tool_search, bridge_schemas, + unwrap_tool_call, }; -pub use index::{Bm25Index, tokenize}; +// The BM25 arithmetic lives in `tinytools::rank` now, so a host ranks with the +// same index the bridge does; the old paths keep resolving. +pub use tinytools::{Bm25Index, tokenize}; pub use manifest::{MANIFEST_DESCRIPTION_CHARS, first_sentence, render_manifest}; -pub use types::{DeferredCatalog, DeferredTool, ToolDiscoveryPolicy}; +pub use types::{ + DeferredCatalog, DeferredTool, DiscoveryRankMode, RankedSearch, ToolDiscoveryPolicy, +}; #[cfg(test)] mod test; diff --git a/crates/tinyagents-harness/src/tool/discover/types.rs b/crates/tinyagents-harness/src/tool/discover/types.rs index 0c3d0448f..397c7a74c 100644 --- a/crates/tinyagents-harness/src/tool/discover/types.rs +++ b/crates/tinyagents-harness/src/tool/discover/types.rs @@ -1,8 +1,9 @@ //! Types for on-demand tool discovery. -use tinyinference_llm::tool::ToolSchema; +use std::{fmt, sync::Arc}; -use super::index::Bm25Index; +use tinyinference_llm::tool::ToolSchema; +use tinytools::{Bm25Index, Bm25Ranker, RankCandidate, RankContext, RankError, ToolRanker}; /// How the agent loop exposes [`tinytools::ToolExposure::Deferred`] tools. /// @@ -13,7 +14,7 @@ use super::index::Bm25Index; /// array therefore stays byte-identical for the whole run, which is what a /// provider prompt cache keys on; revealing a schema costs one tool result, /// not a cache miss on every later turn. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone)] pub struct ToolDiscoveryPolicy { /// Whether the bridge tools are offered at all. /// @@ -33,6 +34,30 @@ pub struct ToolDiscoveryPolicy { /// Ceiling on the model-supplied `limit`, so one call cannot undo the /// saving by asking for everything. pub max_limit: usize, + /// What ranks the catalogue against a `tool_search` query. + /// + /// `None` is the built-in [`Bm25Ranker`]: free, deterministic, no network. + /// A host with a decision model or an embedding index installs it here and + /// chooses how it is used with [`Self::rank_mode`]. Whatever is installed, + /// a ranker failure falls back to BM25 — a search that errors would leave + /// every deferred tool unreachable for the turn. + pub ranker: Option>, + /// How [`Self::ranker`] and the built-in BM25 are combined. + pub rank_mode: DiscoveryRankMode, +} + +/// How a `tool_search` answer is produced when a host ranker is installed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum DiscoveryRankMode { + /// Serve the host ranker; fall back to BM25 when it fails. The default. + #[default] + Ranker, + /// Ignore the host ranker and serve BM25, as if none were installed. + Bm25, + /// Run both, serve the host ranker (BM25 on failure), and report the + /// BM25 ranking alongside in the `ToolSearched` event so the two can be + /// compared on live traffic without changing what the model sees. + Compare, } impl ToolDiscoveryPolicy { @@ -51,6 +76,31 @@ impl ToolDiscoveryPolicy { let default = self.default_limit.clamp(1, max); (default, max) } + + /// Installs a host ranker, keeping the current [`Self::rank_mode`]. + #[must_use] + pub fn with_ranker(mut self, ranker: Arc) -> Self { + self.ranker = Some(ranker); + self + } + + /// Sets how the host ranker is used. + #[must_use] + pub fn with_rank_mode(mut self, mode: DiscoveryRankMode) -> Self { + self.rank_mode = mode; + self + } + + /// The host ranker in force, or `None` when BM25 answers alone — either + /// because none is installed or because [`Self::rank_mode`] is + /// [`DiscoveryRankMode::Bm25`]. + #[must_use] + pub fn active_ranker(&self) -> Option<&Arc> { + match self.rank_mode { + DiscoveryRankMode::Bm25 => None, + DiscoveryRankMode::Ranker | DiscoveryRankMode::Compare => self.ranker.as_ref(), + } + } } impl Default for ToolDiscoveryPolicy { @@ -60,22 +110,52 @@ impl Default for ToolDiscoveryPolicy { manifest_token_budget: 4_000, default_limit: 5, max_limit: 20, + ranker: None, + rank_mode: DiscoveryRankMode::default(), } } } +impl fmt::Debug for ToolDiscoveryPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ToolDiscoveryPolicy") + .field("enabled", &self.enabled) + .field("manifest_token_budget", &self.manifest_token_budget) + .field("default_limit", &self.default_limit) + .field("max_limit", &self.max_limit) + .field("ranker", &self.ranker.as_ref().map(|r| r.kind())) + .field("rank_mode", &self.rank_mode) + .finish() + } +} + +impl PartialEq for ToolDiscoveryPolicy { + /// Two policies are equal when every knob matches and the installed + /// rankers are of the same kind; a ranker has no identity beyond that. + fn eq(&self, other: &Self) -> bool { + self.enabled == other.enabled + && self.manifest_token_budget == other.manifest_token_budget + && self.default_limit == other.default_limit + && self.max_limit == other.max_limit + && self.rank_mode == other.rank_mode + && self.ranker.as_ref().map(|r| r.kind()) == other.ranker.as_ref().map(|r| r.kind()) + } +} + /// One deferred tool as the catalogue sees it. #[derive(Clone, Debug)] pub struct DeferredTool { /// The model-facing schema (host-injected arguments already projected out). pub schema: ToolSchema, + /// The pack, toolkit, or server the tool came from, when the host said. + pub family: Option, /// `name` (also split into words) + description + top-level property /// names: the text the ranker sees. searchable: String, } impl DeferredTool { - fn from_schema(schema: ToolSchema) -> Self { + fn from_schema(schema: ToolSchema, family: Option) -> Self { let mut searchable = String::with_capacity(schema.description.len() + 64); searchable.push_str(&schema.name); searchable.push(' '); @@ -92,7 +172,22 @@ impl DeferredTool { searchable.push_str(key); } } - Self { schema, searchable } + Self { + schema, + family, + searchable, + } + } + + /// This tool as a [`RankCandidate`]: keyed by name, summarised by its + /// searchable text. + #[must_use] + pub fn candidate(&self) -> RankCandidate { + RankCandidate { + key: self.schema.name.clone(), + family: self.family.clone(), + summary: self.searchable.clone(), + } } } @@ -107,12 +202,39 @@ pub struct DeferredCatalog { index: Bm25Index, } +/// How a [`DeferredCatalog::rank`] answer was produced. +#[derive(Clone, Debug, PartialEq)] +pub struct RankedSearch { + /// Names of the matched tools, best first. + pub names: Vec, + /// [`ToolRanker::kind`] of what produced `names`. + pub ranker: &'static str, + /// The best hit's calibrated confidence, when the ranker gave one. + pub top_confidence: Option, + /// Why the host ranker was not served, when it was installed and active. + pub fallback: Option, + /// The BM25 ranking, when [`DiscoveryRankMode::Compare`] asked for it and + /// BM25 was not what was served. + pub shadow_names: Option>, + /// Wall time of the ranking, in milliseconds. + pub latency_ms: u64, +} + impl DeferredCatalog { - /// Indexes `schemas`, sorting them by name. + /// Indexes `schemas`, sorting them by name. No families. #[must_use] - pub fn build(mut schemas: Vec) -> Self { - schemas.sort_by(|left, right| left.name.cmp(&right.name)); - let tools: Vec = schemas.into_iter().map(DeferredTool::from_schema).collect(); + pub fn build(schemas: Vec) -> Self { + Self::build_with_families(schemas.into_iter().map(|schema| (schema, None)).collect()) + } + + /// Indexes `(schema, family)` pairs, sorting them by name. + #[must_use] + pub fn build_with_families(mut entries: Vec<(ToolSchema, Option)>) -> Self { + entries.sort_by(|left, right| left.0.name.cmp(&right.0.name)); + let tools: Vec = entries + .into_iter() + .map(|(schema, family)| DeferredTool::from_schema(schema, family)) + .collect(); let index = Bm25Index::build( tools .iter() @@ -138,6 +260,11 @@ impl DeferredCatalog { self.tools.iter().map(|tool| &tool.schema) } + /// Every deferred tool, sorted by name. + pub fn tools(&self) -> impl Iterator { + self.tools.iter() + } + /// Looks a deferred tool up by exact name. #[must_use] pub fn get(&self, name: &str) -> Option<&ToolSchema> { @@ -147,8 +274,8 @@ impl DeferredCatalog { .map(|index| &self.tools[index].schema) } - /// Ranks the catalogue against `query`, best first, at most `limit` hits. - /// Only positively scored tools are returned. + /// Ranks the catalogue against `query` with BM25, best first, at most + /// `limit` hits. Only positively scored tools are returned. #[must_use] pub fn search(&self, query: &str, limit: usize) -> Vec<&ToolSchema> { self.index @@ -157,4 +284,76 @@ impl DeferredCatalog { .map(|index| &self.tools[index].schema) .collect() } + + /// Ranks the catalogue as `policy` says: the host ranker when one is + /// active, BM25 otherwise, and BM25 as the fallback when the host ranker + /// fails or returns nothing it is sure of. + /// + /// Never errors: the catalogue always has BM25 to answer with, and a + /// search that fails would leave every deferred tool unreachable for the + /// turn. What went wrong is reported in [`RankedSearch::fallback`]. + pub async fn rank( + &self, + policy: &ToolDiscoveryPolicy, + query: &str, + context: &RankContext, + limit: usize, + ) -> RankedSearch { + let started = std::time::Instant::now(); + let Some(ranker) = policy.active_ranker() else { + return RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: None, + shadow_names: None, + latency_ms: elapsed_ms(started), + }; + }; + let candidates: Vec = self.tools.iter().map(DeferredTool::candidate).collect(); + let hosted = ranker.rank(query, context, &candidates, limit).await; + let shadow_names = (policy.rank_mode == DiscoveryRankMode::Compare) + .then(|| self.search_names(query, limit)); + match hosted { + Ok(hits) if !hits.is_empty() => RankedSearch { + top_confidence: hits.first().and_then(|hit| hit.confidence), + names: hits.into_iter().map(|hit| hit.key).collect(), + ranker: ranker.kind(), + fallback: None, + shadow_names, + latency_ms: elapsed_ms(started), + }, + Ok(_) => RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: Some(format!("{} returned no match", ranker.kind())), + shadow_names: None, + latency_ms: elapsed_ms(started), + }, + Err(error) => RankedSearch { + names: self.search_names(query, limit), + ranker: Bm25Ranker::KIND, + top_confidence: None, + fallback: Some(describe_failure(ranker.kind(), &error)), + shadow_names: None, + latency_ms: elapsed_ms(started), + }, + } + } + + fn search_names(&self, query: &str, limit: usize) -> Vec { + self.search(query, limit) + .into_iter() + .map(|schema| schema.name.clone()) + .collect() + } +} + +fn describe_failure(kind: &str, error: &RankError) -> String { + format!("{kind} failed: {error}") +} + +fn elapsed_ms(started: std::time::Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) } From 7a06c3b37563947abd83032f9594bd7e71ed0fab Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:23:08 +0530 Subject: [PATCH 3/8] chore: add missing newline at end of bridge.rs The file previously lacked a trailing newline, which could cause issues with some tooling and version control systems. This change adds the newline to ensure the file ends properly. Auto-committed-on: macbook --- .../src/tool/discover/bridge.rs | 63 +++++++++++++------ 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/crates/tinyagents-harness/src/tool/discover/bridge.rs b/crates/tinyagents-harness/src/tool/discover/bridge.rs index cd5c5895b..10d5156d6 100644 --- a/crates/tinyagents-harness/src/tool/discover/bridge.rs +++ b/crates/tinyagents-harness/src/tool/discover/bridge.rs @@ -11,10 +11,10 @@ use serde_json::{Value, json}; use tinyinference_llm::tool::{ToolFormat, ToolSchema}; -use tinytools::ToolResult; +use tinytools::{RankContext, ToolResult}; use super::manifest::render_manifest; -use super::types::{DeferredCatalog, ToolDiscoveryPolicy}; +use super::types::{DeferredCatalog, RankedSearch, ToolDiscoveryPolicy}; /// Name of the intrinsic search bridge. pub const TOOL_SEARCH_NAME: &str = "tool_search"; @@ -96,26 +96,42 @@ fn tool_call_schema() -> ToolSchema { } } -/// Answers a `tool_search` call against the run's catalogue, returning the -/// result to hand the model and how many tools it named. -#[must_use] -pub fn answer_tool_search( +/// What a `tool_search` produced: the result to hand the model plus the +/// facts the loop reports in its `ToolSearched` event. +#[derive(Debug)] +pub struct SearchAnswer { + /// The tool result the model sees. + pub result: ToolResult, + /// How many tools it named. + pub matched: usize, + /// Which ranker's answer was served, and how it went. `None` when the + /// query was rejected before ranking. + pub ranking: Option, +} + +/// Answers a `tool_search` call against the run's catalogue. +/// +/// Ranks as `policy` says — the host ranker when one is active, BM25 +/// otherwise or on failure — and returns the full schema of every hit so +/// the model can call it. +pub async fn answer_tool_search( catalog: &DeferredCatalog, policy: &ToolDiscoveryPolicy, arguments: &Value, -) -> (ToolResult, usize) { +) -> SearchAnswer { let query = arguments .get("query") .and_then(Value::as_str) .unwrap_or_default() .trim(); if query.is_empty() { - return ( - ToolResult::error(format!( + return SearchAnswer { + result: ToolResult::error(format!( "`{TOOL_SEARCH_NAME}` needs a `query` describing what you want to do." )), - 0, - ); + matched: 0, + ranking: None, + }; } let (default_limit, max_limit) = policy.effective_limits(); let limit = arguments @@ -125,16 +141,24 @@ pub fn answer_tool_search( usize::try_from(n).unwrap_or(usize::MAX).clamp(1, max_limit) }); - let matches = catalog.search(query, limit); + let ranking = catalog + .rank(policy, query, &RankContext::empty(), limit) + .await; + let matches: Vec<&ToolSchema> = ranking + .names + .iter() + .filter_map(|name| catalog.get(name)) + .collect(); if matches.is_empty() { - return ( - ToolResult::success(format!( + return SearchAnswer { + result: ToolResult::success(format!( "No deferred tool matches \"{query}\". {} tool(s) are searchable; everything \ else you can use is already in your tool list.", catalog.len() )), - 0, - ); + matched: 0, + ranking: Some(ranking), + }; } let payload: Vec = matches .iter() @@ -148,13 +172,14 @@ pub fn answer_tool_search( .collect(); let rendered = serde_json::to_string_pretty(&payload).unwrap_or_else(|_| "[]".to_string()); let matched = payload.len(); - ( - ToolResult::success(format!( + SearchAnswer { + result: ToolResult::success(format!( "{matched} match(es). Invoke one with `{TOOL_CALL_NAME}` {{\"name\", \"arguments\"}} \ or by its own name, using the parameters shown.\n{rendered}" )), matched, - ) + ranking: Some(ranking), + } } /// Unwraps a `tool_call` payload into the real `(name, arguments)` pair. From 3635b1f63e1a000bd86b50b6c4b20bc72c7adfc1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:24:05 +0530 Subject: [PATCH 4/8] feat(discover): report ranking details in tool search events The discovery bridge now returns ranking metadata alongside search results, including which ranker was used, confidence scores, fallback reasons, shadow rankings, and latency. This information is surfaced in the `ToolSearched` event so callers can observe and audit how deferred tool discovery ranked results. The schema lookup also now pairs each deferred tool with its family name, and the BM25 index types are re-exported from the `rank` module to keep the public API aligned with the underlying crate. Auto-committed-on: macbook --- .../src/agent_loop/tools.rs | 38 +++++++++++++------ crates/tinyagents-harness/src/events/types.rs | 18 +++++++++ .../src/tool/discover/mod.rs | 2 +- .../src/tool/discover/types.rs | 3 +- crates/tinyagents-harness/src/tool/mod.rs | 21 ++++++++++ 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/crates/tinyagents-harness/src/agent_loop/tools.rs b/crates/tinyagents-harness/src/agent_loop/tools.rs index 442091183..840866d87 100644 --- a/crates/tinyagents-harness/src/agent_loop/tools.rs +++ b/crates/tinyagents-harness/src/agent_loop/tools.rs @@ -284,16 +284,22 @@ impl AgentHarness { if !self.policy.discovery.enabled { return crate::tool::discover::DeferredCatalog::default(); } - let mut schemas = self + let mut entries = self .tools - .deferred_schemas() + .deferred_schemas_with_families() .into_iter() - .filter(|schema| host_allows(&schema.name)) + .filter(|(schema, _)| host_allows(&schema.name)) .collect::>(); if let Some(preparation) = &self.policy.tool_schemas { - schemas = crate::tool::prepare_tool_schemas(&schemas, preparation); + let families: Vec> = + entries.iter().map(|(_, family)| family.clone()).collect(); + let schemas: Vec<_> = entries.into_iter().map(|(schema, _)| schema).collect(); + entries = crate::tool::prepare_tool_schemas(&schemas, preparation) + .into_iter() + .zip(families) + .collect(); } - crate::tool::discover::DeferredCatalog::build(schemas) + crate::tool::discover::DeferredCatalog::build_with_families(entries) } /// Resolves the discovery bridge for one call, when it is one. @@ -303,7 +309,7 @@ impl AgentHarness { /// admission continues with it, and `None` untouched for any other name. /// A malformed `tool_call` payload is answered with a tool error rather /// than passed on, so the model can correct it. - fn answer_discovery_bridge( + async fn answer_discovery_bridge( &self, ctx: &RunContext, status: &mut HarnessRunStatus, @@ -334,11 +340,13 @@ impl AgentHarness { return Ok(None); } if call.name == TOOL_SEARCH_NAME { - let (result, matched) = crate::tool::discover::answer_tool_search( + let answer = crate::tool::discover::answer_tool_search( &catalog, &self.policy.discovery, &call.arguments, - ); + ) + .await; + let ranking = answer.ranking; let record = ctx.emit(AgentEvent::ToolSearched { call_id: CallId::new(call.id.clone()), query: call @@ -347,10 +355,18 @@ impl AgentHarness { .and_then(Value::as_str) .unwrap_or_default() .to_string(), - matched, + matched: answer.matched, + ranker: ranking + .as_ref() + .map(|r| r.ranker.to_string()) + .unwrap_or_default(), + top_confidence: ranking.as_ref().and_then(|r| r.top_confidence), + fallback: ranking.as_ref().and_then(|r| r.fallback.clone()), + shadow_matched: ranking.as_ref().and_then(|r| r.shadow_names.clone()), + latency_ms: ranking.as_ref().map_or(0, |r| r.latency_ms), }); status.set_last_event(record.id); - return Ok(Some(ResolvedToolCall::Answered(result))); + return Ok(Some(ResolvedToolCall::Answered(answer.result))); } match crate::tool::discover::unwrap_tool_call(&call.arguments) { Ok((name, arguments)) => { @@ -509,7 +525,7 @@ impl AgentHarness { // a call the provider could not parse is left for the recovery below. if call.invalid.is_none() && self.tools.dispatch(&call.name).is_none() - && let Some(answered) = self.answer_discovery_bridge(ctx, status, call)? + && let Some(answered) = self.answer_discovery_bridge(ctx, status, call).await? { return Ok(answered); } diff --git a/crates/tinyagents-harness/src/events/types.rs b/crates/tinyagents-harness/src/events/types.rs index 292be873a..138c6d4c9 100644 --- a/crates/tinyagents-harness/src/events/types.rs +++ b/crates/tinyagents-harness/src/events/types.rs @@ -131,6 +131,24 @@ pub enum AgentEvent { query: String, /// Number of deferred tools returned. matched: usize, + /// Which ranker's answer was served: `"bm25"`, or the host ranker's + /// [`tinytools::ToolRanker::kind`]. Empty when the query was rejected + /// before ranking. + #[serde(default)] + ranker: String, + /// The best hit's calibrated confidence, when the ranker gave one. + #[serde(default, skip_serializing_if = "Option::is_none")] + top_confidence: Option, + /// Why the host ranker was not served, when one was active. + #[serde(default, skip_serializing_if = "Option::is_none")] + fallback: Option, + /// The BM25 ranking, when the policy asked to compare it against the + /// served one. + #[serde(default, skip_serializing_if = "Option::is_none")] + shadow_matched: Option>, + /// Wall time of the ranking, in milliseconds. + #[serde(default)] + latency_ms: u64, }, /// The model invoked a deferred tool through the intrinsic `tool_call` diff --git a/crates/tinyagents-harness/src/tool/discover/mod.rs b/crates/tinyagents-harness/src/tool/discover/mod.rs index bf270e5a6..3db13e0dc 100644 --- a/crates/tinyagents-harness/src/tool/discover/mod.rs +++ b/crates/tinyagents-harness/src/tool/discover/mod.rs @@ -24,7 +24,7 @@ pub use bridge::{ }; // The BM25 arithmetic lives in `tinytools::rank` now, so a host ranks with the // same index the bridge does; the old paths keep resolving. -pub use tinytools::{Bm25Index, tokenize}; +pub use tinytools::rank::{Bm25Index, tokenize}; pub use manifest::{MANIFEST_DESCRIPTION_CHARS, first_sentence, render_manifest}; pub use types::{ DeferredCatalog, DeferredTool, DiscoveryRankMode, RankedSearch, ToolDiscoveryPolicy, diff --git a/crates/tinyagents-harness/src/tool/discover/types.rs b/crates/tinyagents-harness/src/tool/discover/types.rs index 397c7a74c..53abedd2a 100644 --- a/crates/tinyagents-harness/src/tool/discover/types.rs +++ b/crates/tinyagents-harness/src/tool/discover/types.rs @@ -3,7 +3,8 @@ use std::{fmt, sync::Arc}; use tinyinference_llm::tool::ToolSchema; -use tinytools::{Bm25Index, Bm25Ranker, RankCandidate, RankContext, RankError, ToolRanker}; +use tinytools::rank::Bm25Index; +use tinytools::{Bm25Ranker, RankCandidate, RankContext, RankError, ToolRanker}; /// How the agent loop exposes [`tinytools::ToolExposure::Deferred`] tools. /// diff --git a/crates/tinyagents-harness/src/tool/mod.rs b/crates/tinyagents-harness/src/tool/mod.rs index e898085c3..1113432d5 100644 --- a/crates/tinyagents-harness/src/tool/mod.rs +++ b/crates/tinyagents-harness/src/tool/mod.rs @@ -320,6 +320,27 @@ impl ToolRegistry { self.schemas_with_exposure(tinytools::ToolExposure::Deferred) } + /// [`Self::deferred_schemas`] paired with each tool's + /// [`tinytools::Tool::family`], name-sorted, so the discovery catalogue + /// can say where a hit came from. + #[must_use] + pub fn deferred_schemas_with_families( + &self, + ) -> Vec<(tinyinference_llm::tool::ToolSchema, Option)> { + let mut entries: Vec<_> = self + .tools + .values() + .map(|dispatch| dispatch.tool()) + .filter(|tool| tool.exposure() == tinytools::ToolExposure::Deferred) + .map(|tool| { + let family = tool.family().map(str::to_owned); + (provider_schema(tool.as_ref()), family) + }) + .collect(); + entries.sort_by(|left, right| left.0.name.cmp(&right.0.name)); + entries + } + fn schemas_with_exposure( &self, exposure: tinytools::ToolExposure, From 31ab308366b57e3f13fa2a24099f19e947497000 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:25:14 +0530 Subject: [PATCH 5/8] fix(discover): make answer_tool_search async and return ranking details The change makes `answer_tool_search` async and updates its return type to include ranking information, allowing tests to verify ranker behavior and fallback logic. It also adds tests for host ranker integration, including failure fallback to BM25 and compare mode behavior. Auto-committed-on: macbook --- .../src/tool/discover/mod.rs | 2 +- .../src/tool/discover/test.rs | 245 ++++++++++++++++-- .../src/tool/discover/types.rs | 3 +- 3 files changed, 228 insertions(+), 22 deletions(-) diff --git a/crates/tinyagents-harness/src/tool/discover/mod.rs b/crates/tinyagents-harness/src/tool/discover/mod.rs index 3db13e0dc..2ca1136b6 100644 --- a/crates/tinyagents-harness/src/tool/discover/mod.rs +++ b/crates/tinyagents-harness/src/tool/discover/mod.rs @@ -24,8 +24,8 @@ pub use bridge::{ }; // The BM25 arithmetic lives in `tinytools::rank` now, so a host ranks with the // same index the bridge does; the old paths keep resolving. -pub use tinytools::rank::{Bm25Index, tokenize}; pub use manifest::{MANIFEST_DESCRIPTION_CHARS, first_sentence, render_manifest}; +pub use tinytools::rank::{Bm25Index, tokenize}; pub use types::{ DeferredCatalog, DeferredTool, DiscoveryRankMode, RankedSearch, ToolDiscoveryPolicy, }; diff --git a/crates/tinyagents-harness/src/tool/discover/test.rs b/crates/tinyagents-harness/src/tool/discover/test.rs index 1e93c886a..fffca8ccd 100644 --- a/crates/tinyagents-harness/src/tool/discover/test.rs +++ b/crates/tinyagents-harness/src/tool/discover/test.rs @@ -140,62 +140,267 @@ fn bridge_schemas_are_byte_stable_across_builds() { assert_eq!(a, b); } -#[test] -fn answer_tool_search_returns_full_schemas_for_hits() { +#[tokio::test] +async fn answer_tool_search_returns_full_schemas_for_hits() { let policy = ToolDiscoveryPolicy::default(); - let (result, matched) = answer_tool_search( + let SearchAnswer { + result, + matched, + ranking, + } = answer_tool_search( &catalog(), &policy, &json!({"query": "read a pdf", "limit": 1}), - ); + ) + .await; assert!(!result.is_error); assert_eq!(matched, 1); + let ranking = ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert!(ranking.fallback.is_none()); + assert!(ranking.shadow_names.is_none()); let text = result.text(); assert!(text.starts_with("1 match(es).")); assert!(text.contains("\"name\": \"pdf_read\"")); assert!(text.contains("\"path\"")); } -#[test] -fn answer_tool_search_clamps_limit_and_handles_misses() { +#[tokio::test] +async fn answer_tool_search_clamps_limit_and_handles_misses() { let policy = ToolDiscoveryPolicy { max_limit: 2, ..ToolDiscoveryPolicy::default() }; - let (_, matched) = answer_tool_search( + let answer = answer_tool_search( &catalog(), &policy, &json!({"query": "pdf invite quote symbol attendees", "limit": 50}), - ); - assert!(matched <= 2); + ) + .await; + assert!(answer.matched <= 2); - let (result, matched) = answer_tool_search(&catalog(), &policy, &json!({"query": "zzzz qqqq"})); - assert!(!result.is_error); - assert_eq!(matched, 0); - assert!(result.text().starts_with("No deferred tool matches")); + let answer = answer_tool_search(&catalog(), &policy, &json!({"query": "zzzz qqqq"})).await; + assert!(!answer.result.is_error); + assert_eq!(answer.matched, 0); + assert!(answer.result.text().starts_with("No deferred tool matches")); - let (result, _) = answer_tool_search(&catalog(), &policy, &json!({"query": " "})); - assert!(result.is_error); + let answer = answer_tool_search(&catalog(), &policy, &json!({"query": " "})).await; + assert!(answer.result.is_error); + assert!(answer.ranking.is_none()); } /// Regression: `max_limit: 0` used to reach `usize::clamp(1, 0)`, which /// panics because its minimum exceeds its maximum — a model-supplied numeric /// `limit` could crash the process. It must instead clamp against a /// normalized effective maximum of at least 1. -#[test] -fn answer_tool_search_does_not_panic_on_a_zero_max_limit() { +#[tokio::test] +async fn answer_tool_search_does_not_panic_on_a_zero_max_limit() { let policy = ToolDiscoveryPolicy { max_limit: 0, default_limit: 5, ..ToolDiscoveryPolicy::default() }; - let (result, matched) = answer_tool_search( + let answer = answer_tool_search( &catalog(), &policy, &json!({"query": "pdf invite quote symbol attendees", "limit": 50}), + ) + .await; + assert!(!answer.result.is_error); + assert!( + answer.matched <= 1, + "effective max_limit must clamp to at least 1" ); - assert!(!result.is_error); - assert!(matched <= 1, "effective max_limit must clamp to at least 1"); +} + +/// A ranker that answers from a script: `Ok(keys)` or a failure. +struct ScriptedRanker { + answer: Result, &'static str>, + calls: std::sync::atomic::AtomicUsize, +} + +impl ScriptedRanker { + fn returning(keys: Vec<&'static str>) -> std::sync::Arc { + std::sync::Arc::new(Self { + answer: Ok(keys), + calls: std::sync::atomic::AtomicUsize::new(0), + }) + } + + fn failing(reason: &'static str) -> std::sync::Arc { + std::sync::Arc::new(Self { + answer: Err(reason), + calls: std::sync::atomic::AtomicUsize::new(0), + }) + } + + fn calls(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl tinytools::ToolRanker for ScriptedRanker { + fn kind(&self) -> &'static str { + "scripted" + } + + async fn rank( + &self, + _intent: &str, + _context: &tinytools::RankContext, + candidates: &[tinytools::RankCandidate], + limit: usize, + ) -> Result, tinytools::RankError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Every candidate carries the family the catalogue was built with. + assert!( + candidates + .iter() + .all(|c| c.family.as_deref() == Some("fam")) + ); + match &self.answer { + Ok(keys) => Ok(keys + .iter() + .take(limit) + .enumerate() + .map(|(i, key)| tinytools::RankHit { + key: (*key).to_string(), + score: 0.9 - i as f64 * 0.1, + confidence: Some(0.9 - i as f64 * 0.1), + }) + .collect()), + Err(reason) => Err(tinytools::RankError::Backend { + reason: (*reason).to_string(), + }), + } + } +} + +fn catalog_with_families() -> DeferredCatalog { + DeferredCatalog::build_with_families( + catalog() + .schemas() + .cloned() + .map(|schema| (schema, Some("fam".to_string()))) + .collect(), + ) +} + +#[tokio::test] +async fn host_ranker_is_served_with_its_confidence() { + let ranker = ScriptedRanker::returning(vec!["stock_quote", "pdf_read"]); + let policy = ToolDiscoveryPolicy::default().with_ranker(ranker.clone()); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf", "limit": 3}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "scripted"); + assert_eq!(ranking.names, vec!["stock_quote", "pdf_read"]); + assert_eq!(ranking.top_confidence, Some(0.9)); + assert!(ranking.fallback.is_none()); + assert!( + ranking.shadow_names.is_none(), + "no shadow outside compare mode" + ); + assert_eq!(answer.matched, 2); + assert!(answer.result.text().contains("\"name\": \"stock_quote\"")); + assert_eq!(ranker.calls(), 1); +} + +#[tokio::test] +async fn host_ranker_failure_falls_back_to_bm25_and_says_why() { + let policy = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::failing("503")); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!( + ranking.fallback.as_deref(), + Some("scripted failed: ranker backend failed: 503") + ); + assert_eq!(answer.matched, 1); +} + +#[tokio::test] +async fn host_ranker_empty_answer_falls_back_to_bm25() { + let policy = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::returning(vec![])); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!( + ranking.fallback.as_deref(), + Some("scripted returned no match") + ); +} + +#[tokio::test] +async fn compare_mode_serves_the_ranker_and_reports_bm25_alongside() { + let ranker = ScriptedRanker::returning(vec!["stock_quote"]); + let policy = ToolDiscoveryPolicy::default() + .with_ranker(ranker.clone()) + .with_rank_mode(DiscoveryRankMode::Compare); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "scripted"); + assert_eq!(ranking.names, vec!["stock_quote"]); + assert_eq!(ranking.shadow_names, Some(vec!["pdf_read".to_string()])); +} + +#[tokio::test] +async fn bm25_mode_ignores_an_installed_ranker() { + let ranker = ScriptedRanker::returning(vec!["stock_quote"]); + let policy = ToolDiscoveryPolicy::default() + .with_ranker(ranker.clone()) + .with_rank_mode(DiscoveryRankMode::Bm25); + assert!(policy.active_ranker().is_none()); + let answer = answer_tool_search( + &catalog_with_families(), + &policy, + &json!({"query": "read a pdf"}), + ) + .await; + let ranking = answer.ranking.unwrap(); + assert_eq!(ranking.ranker, "bm25"); + assert_eq!(ranking.names, vec!["pdf_read"]); + assert_eq!(ranker.calls(), 0); +} + +#[test] +fn a_ranker_hit_naming_an_unknown_tool_is_dropped_from_the_answer() { + // `answer_tool_search` resolves names through `catalog.get`; a key the + // ranker invented never reaches the model. Pinned through the sync + // lookup so the guarantee is visible without a runtime. + assert!(catalog().get("invented").is_none()); +} + +#[test] +fn policy_equality_and_debug_compare_ranker_kinds_only() { + let a = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::returning(vec![])); + let b = ToolDiscoveryPolicy::default().with_ranker(ScriptedRanker::failing("x")); + assert_eq!(a, b, "same kind, same knobs"); + assert_ne!(a, ToolDiscoveryPolicy::default()); + assert!(format!("{a:?}").contains("Some(\"scripted\")")); } /// Regression: the `tool_search` schema advertised `"minimum": 1, "maximum": diff --git a/crates/tinyagents-harness/src/tool/discover/types.rs b/crates/tinyagents-harness/src/tool/discover/types.rs index 53abedd2a..2704f62fa 100644 --- a/crates/tinyagents-harness/src/tool/discover/types.rs +++ b/crates/tinyagents-harness/src/tool/discover/types.rs @@ -311,7 +311,8 @@ impl DeferredCatalog { latency_ms: elapsed_ms(started), }; }; - let candidates: Vec = self.tools.iter().map(DeferredTool::candidate).collect(); + let candidates: Vec = + self.tools.iter().map(DeferredTool::candidate).collect(); let hosted = ranker.rank(query, context, &candidates, limit).await; let shadow_names = (policy.rank_mode == DiscoveryRankMode::Compare) .then(|| self.search_names(query, limit)); From ae48b2c34f7e5e7b84f78ef2693e37ed1ac8c5c8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:28:52 +0530 Subject: [PATCH 6/8] feat(discover): pluggable ToolRanker with BM25 fallback and compare mode ToolDiscoveryPolicy gains a host ranker (any tinytools::ToolRanker) and a DiscoveryRankMode; DeferredCatalog::rank serves it, falls back to BM25 on failure or an empty answer, and in Compare mode reports the BM25 ranking alongside. answer_tool_search is async and returns a SearchAnswer; ToolSearched carries the ranker, top confidence, fallback reason, shadow ranking and latency. The BM25 index moved to tinytools::rank and is re-exported from its old path; the catalogue carries Tool::family. Co-authored-by: Medulla --- .../src/tool/discover/README.md | 10 +++-- docs/modules/harness/tool-discovery.md | 37 ++++++++++++++++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/tinyagents-harness/src/tool/discover/README.md b/crates/tinyagents-harness/src/tool/discover/README.md index 868042ee8..5e3f3d1dc 100644 --- a/crates/tinyagents-harness/src/tool/discover/README.md +++ b/crates/tinyagents-harness/src/tool/discover/README.md @@ -6,10 +6,9 @@ this file is the map of the module. | File | Owns | |---------------|----------------------------------------------------------------------| -| `types.rs` | `ToolDiscoveryPolicy` (the knobs), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted), `DeferredTool` | -| `index.rs` | `Bm25Index` + `tokenize` — ranking over `(sort_key, text)` pairs, knows nothing about tools | +| `types.rs` | `ToolDiscoveryPolicy` (the knobs, the host `ranker` and `DiscoveryRankMode`), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted, ranked through the policy), `DeferredTool`, `RankedSearch` | | `manifest.rs` | `render_manifest` — the budgeted listing inside `tool_search`'s description: full → names → count | -| `bridge.rs` | The two intrinsic tools: `bridge_schemas`, `answer_tool_search`, `unwrap_tool_call` | +| `bridge.rs` | The two intrinsic tools: `bridge_schemas`, `answer_tool_search` (async; returns a `SearchAnswer`), `unwrap_tool_call` | | `test.rs` | Unit tests for all of the above | The agent loop (`agent_loop/run_loop.rs`, `agent_loop/tools.rs`) is the only @@ -27,3 +26,8 @@ Invariants worth keeping: the bridge is enabled; a `Hidden` tool is never callable by the model. - The manifest is bounded by `manifest_token_budget`; the search answer clips descriptions to 500 chars and `limit` to `max_limit`. +- A search never fails. The host ranker (`ToolDiscoveryPolicy::ranker`, + any `tinytools::ToolRanker`) is served when active; on error or an empty + answer BM25 answers instead and `RankedSearch::fallback` says why. + `DiscoveryRankMode::Compare` serves the host ranker and carries the BM25 + ranking alongside for comparison. BM25 itself lives in `tinytools::rank`. diff --git a/docs/modules/harness/tool-discovery.md b/docs/modules/harness/tool-discovery.md index 0a3d04a1b..c59f5e87d 100644 --- a/docs/modules/harness/tool-discovery.md +++ b/docs/modules/harness/tool-discovery.md @@ -35,10 +35,11 @@ a tool it does not know it needs. When a run has at least one deferred tool (after the host allow-list), the agent loop appends two intrinsic tools **after** the name-sorted direct set: -- `tool_search { query, limit }` — BM25 over name, split identifier, description - and top-level property names. Returns up to `limit` matches (default 5, max - 20) as `{name, description, parameters}` with the **full** schema, or a - "no match" note. Its description embeds a manifest of every deferred tool: +- `tool_search { query, limit }` — ranks the catalogue's name, split + identifier, description, top-level property names and `Tool::family`. + Returns up to `limit` matches (default 5, max 20) as `{name, description, + parameters}` with the **full** schema, or a "no match" note. See + [Ranking](#ranking) for what does the ranking. Its description embeds a manifest of every deferred tool: `- name: first sentence (≤ 60 chars)`, degrading to names only, then to a bare count, until it fits `ToolDiscoveryPolicy::manifest_token_budget` (default 4,000 tokens). @@ -55,6 +56,27 @@ max_limit }`). With `enabled: false` deferred tools are neither advertised nor searchable, but a direct call by name still runs: deferral only ever subtracts from the wire, never from what the host registered. +### Ranking + +`DeferredCatalog::rank` answers as `ToolDiscoveryPolicy` says: + +- With no `ranker` installed (the default), BM25 from `tinytools::rank` — + free, deterministic, no network. `Bm25Index` and `tokenize` moved to that + crate so a host ranks with the same arithmetic the bridge does. +- With a host `ranker: Arc` (a decision model such + as `tinytools-jev`, or an embedding index), `rank_mode` decides: + `Ranker` serves it; `Bm25` ignores it; `Compare` serves it and reports the + BM25 ranking alongside in `ToolSearched.shadow_matched` so the two can be + judged on live traffic without changing what the model sees. +- A host ranker that fails or returns nothing **falls back to BM25** and + the reason lands in `ToolSearched.fallback`. A search never errors: an + error would leave every deferred tool unreachable for the turn. + +Every hit the ranker names is resolved through `catalog.get`, so a key the +ranker invented never reaches the model. The catalogue is what the ranker +sees; the model's `query` is the only intent, with an empty `RankContext` — +the model already distilled the turn into it. + ### Why a bridge and not hydration OpenClaw appends a revealed schema to the request's `tools` for the rest of the @@ -87,8 +109,11 @@ allow-list. with, not as a live per-request wire metric — exposure-narrowing middleware (`ToolPolicyMiddleware::before_model`, dynamic/contextual selection) can still shrink an individual request below it. -- `ToolSearched { call_id, query, matched }` and - `DeferredToolCall { call_id, tool_name }` — every discovery, auditable. +- `ToolSearched { call_id, query, matched, ranker, top_confidence, fallback, + shadow_matched, latency_ms }` and `DeferredToolCall { call_id, tool_name }` + — every discovery, auditable: which ranker answered, how sure it was, why + a host ranker was not served, and what BM25 would have said in compare + mode. ## Schema budgets From 475ee2cb17dfc49dcb9b82f6ef8112506c84c5ec Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:08:27 +0300 Subject: [PATCH 7/8] chore(deps): update tinytools subproject The tinytools subproject reference has been updated to a new commit, which includes a dirty state indicating local modifications. This change aligns the vendor dependency with the latest upstream changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- vendor/tinytools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinytools b/vendor/tinytools index e2544bc84..9ae1d44de 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit e2544bc844f803abaaa0ee57dc2306c5b1f15dcf +Subproject commit 9ae1d44de00863bfff5e3549063d14d57db15c21 From 48f9189979c8f813637e456bb018eb3eb2260b49 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 22 Sep 2026 05:09:22 +0300 Subject: [PATCH 8/8] chore(vendor): update tinytools subproject Updated the tinytools subproject to the latest commit, incorporating upstream fixes and improvements. Auto-committed-on: dragonfly Co-authored-by: Medulla --- vendor/tinytools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinytools b/vendor/tinytools index 9ae1d44de..e2544bc84 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 9ae1d44de00863bfff5e3549063d14d57db15c21 +Subproject commit e2544bc844f803abaaa0ee57dc2306c5b1f15dcf