feat(discover): pluggable ToolRanker with BM25 fallback and compare mode

crates/tinyagents-graph/Cargo.toml

@@ -19,7 +19,7 @@ tinyagents-harness = { path = "../tinyagents-harness", version = "2.1.2", defaul
     "langfuse",
 ] }
 tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" }
-tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" }
+tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" }
 tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs"] }
 tracing = { workspace = true }
 

crates/tinyagents-harness/Cargo.toml

@@ -28,10 +28,10 @@ sha2 = "0.11"
 thiserror = "2"
 tracing = { workspace = true }
 tinyagents-definition = { path = "../tinyagents-definition", version = "2.1.2" }
-tinytools-agent = { path = "../../vendor/tinytools/crates/tinytools-agent", version = "0.3.0", default-features = false }
+tinytools-agent = { path = "../../vendor/tinytools/crates/tinytools-agent", version = "0.4.1", default-features = false }
 tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-llm", version = "0.3.0" }
 tinyinference-embeddings = { path = "../../vendor/tinyinference/crates/tinyinference-embeddings", version = "0.3.0" }
-tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.3.0" }
+tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.4.1" }
 tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs", "io-util", "process"] }
 tempfile = { workspace = true }
 wait-timeout = { version = "0.2", optional = true }

crates/tinyagents-harness/src/agent_loop/tools.rs

@@ -284,16 +284,22 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
         if !self.policy.discovery.enabled {
             return crate::tool::discover::DeferredCatalog::default();
         }
-        let mut schemas = self
+        let mut entries = self
             .tools
-            .deferred_schemas()
+            .deferred_schemas_with_families()
             .into_iter()
-            .filter(|schema| host_allows(&schema.name))
+            .filter(|(schema, _)| host_allows(&schema.name))
             .collect::<Vec<_>>();
         if let Some(preparation) = &self.policy.tool_schemas {
-            schemas = crate::tool::prepare_tool_schemas(&schemas, preparation);
+            let families: Vec<Option<String>> =
+                entries.iter().map(|(_, family)| family.clone()).collect();
+            let schemas: Vec<_> = entries.into_iter().map(|(schema, _)| schema).collect();
+            entries = crate::tool::prepare_tool_schemas(&schemas, preparation)
+                .into_iter()
+                .zip(families)
+                .collect();
         }
-        crate::tool::discover::DeferredCatalog::build(schemas)
+        crate::tool::discover::DeferredCatalog::build_with_families(entries)
     }
 
     /// Resolves the discovery bridge for one call, when it is one.
@@ -303,7 +309,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
     /// admission continues with it, and `None` untouched for any other name.
     /// A malformed `tool_call` payload is answered with a tool error rather
     /// than passed on, so the model can correct it.
-    fn answer_discovery_bridge(
+    async fn answer_discovery_bridge(
         &self,
         ctx: &RunContext<Ctx>,
         status: &mut HarnessRunStatus,
@@ -334,11 +340,13 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             return Ok(None);
         }
         if call.name == TOOL_SEARCH_NAME {
-            let (result, matched) = crate::tool::discover::answer_tool_search(
+            let answer = crate::tool::discover::answer_tool_search(
                 &catalog,
                 &self.policy.discovery,
                 &call.arguments,
-            );
+            )
+            .await;
+            let ranking = answer.ranking;
             // `query` is model-supplied tool-call content, same privacy
             // class as a normal tool call's arguments, so it honors the same
             // `RunPolicy::capture.tool_io` gate (default `false`, payload
@@ -355,10 +363,18 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                 } else {
                     String::new()
                 },
-                matched,
+                matched: answer.matched,
+                ranker: ranking
+                    .as_ref()
+                    .map(|r| r.ranker.to_string())
+                    .unwrap_or_default(),
+                top_confidence: ranking.as_ref().and_then(|r| r.top_confidence),
+                fallback: ranking.as_ref().and_then(|r| r.fallback.clone()),
+                shadow_matched: ranking.as_ref().and_then(|r| r.shadow_names.clone()),
+                latency_ms: ranking.as_ref().map_or(0, |r| r.latency_ms),
             });
             status.set_last_event(record.id);
-            return Ok(Some(ResolvedToolCall::Answered(result)));
+            return Ok(Some(ResolvedToolCall::Answered(answer.result)));
         }
         match crate::tool::discover::unwrap_tool_call(&call.arguments) {
             Ok((name, arguments)) => {
@@ -517,7 +533,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
         // a call the provider could not parse is left for the recovery below.
         if call.invalid.is_none()
             && self.tools.dispatch(&call.name).is_none()
-            && let Some(answered) = self.answer_discovery_bridge(ctx, status, call)?
+            && let Some(answered) = self.answer_discovery_bridge(ctx, status, call).await?
         {
             return Ok(answered);
         }

crates/tinyagents-harness/src/events/types.rs

@@ -139,6 +139,24 @@ pub enum AgentEvent {
         query: String,
         /// Number of deferred tools returned.
         matched: usize,
+        /// Which ranker's answer was served: `"bm25"`, or the host ranker's
+        /// [`tinytools::ToolRanker::kind`]. Empty when the query was rejected
+        /// before ranking.
+        #[serde(default)]
+        ranker: String,
+        /// The best hit's calibrated confidence, when the ranker gave one.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        top_confidence: Option<f64>,
+        /// Why the host ranker was not served, when one was active.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        fallback: Option<String>,
+        /// The BM25 ranking, when the policy asked to compare it against the
+        /// served one.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        shadow_matched: Option<Vec<String>>,
+        /// Wall time of the ranking, in milliseconds.
+        #[serde(default)]
+        latency_ms: u64,
     },
 
     /// The model invoked a deferred tool through the intrinsic `tool_call`

crates/tinyagents-harness/src/tool/discover/README.md

@@ -6,10 +6,9 @@ this file is the map of the module.
 
 | File          | Owns                                                                 |
 |---------------|----------------------------------------------------------------------|
-| `types.rs`    | `ToolDiscoveryPolicy` (the knobs), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted), `DeferredTool` |
-| `index.rs`    | `Bm25Index` + `tokenize` — ranking over `(sort_key, text)` pairs, knows nothing about tools |
+| `types.rs`    | `ToolDiscoveryPolicy` (the knobs, the host `ranker` and `DiscoveryRankMode`), `DeferredCatalog` (a run's deferred schemas, BM25-indexed, name-sorted, ranked through the policy), `DeferredTool`, `RankedSearch` |
 | `manifest.rs` | `render_manifest` — the budgeted listing inside `tool_search`'s description: full → names → count |
-| `bridge.rs`   | The two intrinsic tools: `bridge_schemas`, `answer_tool_search`, `unwrap_tool_call` |
+| `bridge.rs`   | The two intrinsic tools: `bridge_schemas`, `answer_tool_search` (async; returns a `SearchAnswer`), `unwrap_tool_call` |
 | `test.rs`     | Unit tests for all of the above                                      |
 
 The agent loop (`agent_loop/run_loop.rs`, `agent_loop/tools.rs`) is the only
@@ -27,3 +26,8 @@ Invariants worth keeping:
   the bridge is enabled; a `Hidden` tool is never callable by the model.
 - The manifest is bounded by `manifest_token_budget`; the search answer clips
   descriptions to 500 chars and `limit` to `max_limit`.
+- A search never fails. The host ranker (`ToolDiscoveryPolicy::ranker`,
+  any `tinytools::ToolRanker`) is served when active; on error or an empty
+  answer BM25 answers instead and `RankedSearch::fallback` says why.
+  `DiscoveryRankMode::Compare` serves the host ranker and carries the BM25
+  ranking alongside for comparison. BM25 itself lives in `tinytools::rank`.

crates/tinyagents-harness/src/tool/discover/bridge.rs

@@ -11,10 +11,10 @@
 
 use serde_json::{Value, json};
 use tinyinference_llm::tool::{ToolFormat, ToolSchema};
-use tinytools::ToolResult;
+use tinytools::{RankContext, ToolResult};
 
 use super::manifest::render_manifest;
-use super::types::{DeferredCatalog, ToolDiscoveryPolicy};
+use super::types::{DeferredCatalog, RankedSearch, ToolDiscoveryPolicy};
 
 /// Name of the intrinsic search bridge.
 pub const TOOL_SEARCH_NAME: &str = "tool_search";
@@ -96,26 +96,42 @@ fn tool_call_schema() -> ToolSchema {
     }
 }
 
-/// Answers a `tool_search` call against the run's catalogue, returning the
-/// result to hand the model and how many tools it named.
-#[must_use]
-pub fn answer_tool_search(
+/// What a `tool_search` produced: the result to hand the model plus the
+/// facts the loop reports in its `ToolSearched` event.
+#[derive(Debug)]
+pub struct SearchAnswer {
+    /// The tool result the model sees.
+    pub result: ToolResult,
+    /// How many tools it named.
+    pub matched: usize,
+    /// Which ranker's answer was served, and how it went. `None` when the
+    /// query was rejected before ranking.
+    pub ranking: Option<RankedSearch>,
+}
+
+/// Answers a `tool_search` call against the run's catalogue.
+///
+/// Ranks as `policy` says — the host ranker when one is active, BM25
+/// otherwise or on failure — and returns the full schema of every hit so
+/// the model can call it.
+pub async fn answer_tool_search(
     catalog: &DeferredCatalog,
     policy: &ToolDiscoveryPolicy,
     arguments: &Value,
-) -> (ToolResult, usize) {
+) -> SearchAnswer {
     let query = arguments
         .get("query")
         .and_then(Value::as_str)
         .unwrap_or_default()
         .trim();
     if query.is_empty() {
-        return (
-            ToolResult::error(format!(
+        return SearchAnswer {
+            result: ToolResult::error(format!(
                 "`{TOOL_SEARCH_NAME}` needs a `query` describing what you want to do."
             )),
-            0,
-        );
+            matched: 0,
+            ranking: None,
+        };
     }
     let (default_limit, max_limit) = policy.effective_limits();
     let limit = arguments
@@ -125,16 +141,24 @@ pub fn answer_tool_search(
             usize::try_from(n).unwrap_or(usize::MAX).clamp(1, max_limit)
         });
 
-    let matches = catalog.search(query, limit);
+    let ranking = catalog
+        .rank(policy, query, &RankContext::empty(), limit)
+        .await;
+    let matches: Vec<&ToolSchema> = ranking
+        .names
+        .iter()
+        .filter_map(|name| catalog.get(name))
+        .collect();
     if matches.is_empty() {
-        return (
-            ToolResult::success(format!(
+        return SearchAnswer {
+            result: ToolResult::success(format!(
                 "No deferred tool matches \"{query}\". {} tool(s) are searchable; everything \
                  else you can use is already in your tool list.",
                 catalog.len()
             )),
-            0,
-        );
+            matched: 0,
+            ranking: Some(ranking),
+        };
     }
     let payload: Vec<Value> = matches
         .iter()
@@ -148,13 +172,14 @@ pub fn answer_tool_search(
         .collect();
     let rendered = serde_json::to_string_pretty(&payload).unwrap_or_else(|_| "[]".to_string());
     let matched = payload.len();
-    (
-        ToolResult::success(format!(
+    SearchAnswer {
+        result: ToolResult::success(format!(
             "{matched} match(es). Invoke one with `{TOOL_CALL_NAME}` {{\"name\", \"arguments\"}} \
              or by its own name, using the parameters shown.\n{rendered}"
         )),
         matched,
-    )
+        ranking: Some(ranking),
+    }
 }
 
 /// Unwraps a `tool_call` payload into the real `(name, arguments)` pair.