diff --git a/Cargo.lock b/Cargo.lock index b432a6d9..9662c355 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -67,6 +76,16 @@ dependencies = [ "rustversion", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -348,6 +367,24 @@ dependencies = [ "libc", ] +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "diff" version = "0.1.13" @@ -593,6 +630,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.2" @@ -851,6 +894,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -940,6 +989,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1260,6 +1319,35 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "reqwest" version = "0.12.28" @@ -1629,6 +1717,22 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "switchyard-llm-client" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "reqwest", + "serde_json", + "switchyard-protocol", + "switchyard-translation", + "thiserror", + "tokio", + "wiremock", +] + [[package]] name = "switchyard-protocol" version = "0.1.0" @@ -2199,6 +2303,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index ec2ee2c7..551d52cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" members = [ "crates/libsy", "crates/libsy-examples", + "crates/libsy-llm-client", "crates/switchyard-components", "crates/switchyard-components-v2", "crates/switchyard-components-v2-macros", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml new file mode 100644 index 00000000..67c33893 --- /dev/null +++ b/crates/libsy-llm-client/Cargo.toml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-llm-client" +version = "0.1.0" +description = "HTTP LLM client speaking Switchyard's neutral IR directly" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +switchyard-protocol = { path = "../protocol", version = "0.1.0" } +switchyard-translation = { path = "../switchyard-translation", version = "0.1.0" } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } +async-trait = "0.1" +futures-util = "0.3" +serde_json = "1" +thiserror = "2" + +[dev-dependencies] +futures = "0.3" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +wiremock = "0.6" diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md new file mode 100644 index 00000000..e769123f --- /dev/null +++ b/crates/libsy-llm-client/README.md @@ -0,0 +1,193 @@ + + +# libsy-llm-client + +An HTTP client that speaks Switchyard's neutral IR directly. You hand it a +[`switchyard_protocol::Request`] and a model name; it looks up the configured backend, +encodes the request to that backend's wire format, adds auth and forwards your +headers, makes the call with a shared `reqwest::Client`, and decodes the reply +back into a [`switchyard_protocol::Response`] — buffered or streamed. + +It depends only on `switchyard-protocol` and `switchyard-translation`; no server, no +provider SDK. + +## Concepts + +- **Model configs.** A client is built from [`ModelConfig`] values. Each model has a + default [`Backend`] and can have additional backends for other wire formats. + [`TranslatingLlmClient::call_rewrite_model`] uses the request's metadata wire format + when set, otherwise the model's default backend. +- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or + `Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`, + static `extra_headers`). The variant fixes the URL path and auth scheme + (Bearer vs `x-api-key` + `anthropic-version`). +- **Model rewrite.** The resolved model name is both the map key and the model id + sent upstream — it overwrites whatever `model` the request arrived with. +- **Streaming is chosen by the request.** If the encoded body has `stream: true` + (i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise + `LlmResponse::Agg`. + +## Add the dependency + +Within this workspace: + +```toml +[dependencies] +switchyard-llm-client = { path = "../libsy-llm-client" } +switchyard-protocol = { path = "../libsy-protocol" } +switchyard-translation = { path = "../switchyard-translation" } # for WireFormat +``` + +## Quickstart + +### Build a client + +```rust +use std::collections::BTreeMap; +use switchyard_llm_client::{ + Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, +}; + +fn build_client() -> switchyard_llm_client::Result { + let openai = HttpBackendConfig { + base_url: "https://api.openai.com/v1".to_string(), + api_key: std::env::var("OPENAI_API_KEY").ok(), + extra_headers: BTreeMap::new(), + }; + + let models = [ModelConfig::new( + "gpt-4o-mini", + Backend::OpenAiChat(openai), + None, + )]; + + TranslatingLlmClient::new(&models) +} +``` + +### Buffered call + +```rust +use switchyard_llm_client::{LlmClientError, TranslatingLlmClient}; +use switchyard_protocol::{completion_text, text_request, Context, LlmResponse, Request}; + +async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result { + let request = Request { + llm_request: text_request(None, "Say hello in five words."), + raw_request: None, + metadata: None, + }; + + // model_name wins over request.llm_request.model; it is also sent upstream. + let response = client + .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini")) + .await?; + + match response.llm_response { + LlmResponse::Agg(agg) => Ok(completion_text(&agg)), + LlmResponse::Stream(_) => Err(LlmClientError::Stream( + "expected a buffered response".to_string(), + )), + } +} +``` + +### Streaming call + +Set `stream` on the IR request and drive the returned chunk stream: + +```rust +use futures_util::StreamExt; +use switchyard_llm_client::{LlmClientError, TranslatingLlmClient}; +use switchyard_protocol::{text_request, Context, LlmResponse, LlmResponseChunk, Request}; + +async fn stream(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<()> { + let mut llm_request = text_request(None, "Count to five."); + llm_request.stream = true; + let request = Request { llm_request, raw_request: None, metadata: None }; + + let response = client + .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini")) + .await?; + + if let LlmResponse::Stream(mut chunks) = response.llm_response { + while let Some(item) = chunks.next().await { + match item { + Ok(LlmResponseChunk::TextDelta { text, .. }) => print!("{text}"), + Ok(_) => {} // usage, tool-call deltas, message start/stop + Err(error) => return Err(LlmClientError::Stream(error.to_string())), + } + } + } + Ok(()) +} +``` + +## Cross-format translation + +The request/response are translated through the neutral IR, so the inbound shape +you build and the backend's wire format are independent. Pointing a +`WireFormat::AnthropicMessages` backend at an Anthropic endpoint while building +requests with the same helpers works the same way. Set `request.metadata.wire_format` +to select a non-default backend before calling `call_rewrite_model`. Register several +formats under one model to serve it over more than one upstream API: + +```rust +use switchyard_llm_client::{ + Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, +}; + +fn build_multi_format_client( + openai_chat: HttpBackendConfig, + openai_responses: HttpBackendConfig, + anthropic: HttpBackendConfig, +) -> switchyard_llm_client::Result { + let models = [ModelConfig::new( + "my-model", + Backend::OpenAiChat(openai_chat), + Some(vec![ + Backend::OpenAiResponses(openai_responses), + Backend::Anthropic(anthropic), + ]), + )]; + + TranslatingLlmClient::new(&models) +} +``` + +## Headers & auth + +- The `Backend` variant sets auth: OpenAI formats send `Authorization: Bearer `; + Anthropic sends `x-api-key: ` plus `anthropic-version`. +- `request.metadata.http_headers` are forwarded upstream, **except** reserved ones: + `host`, `content-length`, `connection`, and the backend-owned + `authorization` / `x-api-key` / `anthropic-version` / `content-type`. So a + caller's placeholder credential never overrides the backend's real key. +- Per-backend static headers go in `HttpBackendConfig::extra_headers`. + +## Errors + +`call_rewrite_model` returns [`LlmClientError`]: + +| Variant | When | +|---------|------| +| `MissingModel` | no `model_name` arg and no `request.llm_request.model` | +| `UnknownModel(model)` | model has no backends configured | +| `UnknownModelFormat { model, format }` | model has no backend for that format | +| `Translation(msg)` | encode/decode failed in the translation engine | +| `Transport(msg)` | connect/timeout/transport failure | +| `ContextWindowExceeded { model, message }` | upstream 400 detected as a context overflow (checked before `UpstreamHttp`, so callers can evict-and-retry) | +| `UpstreamHttp { status, body }` | any other non-2xx upstream response | +| `Stream(msg)` | mid-stream read / malformed frame | + +[`switchyard_protocol::Request`]: ../libsy-protocol +[`switchyard_protocol::Response`]: ../libsy-protocol +[`libsy-proxy`]: ../libsy-proxy +[`Backend`]: src/backend.rs +[`HttpBackendConfig`]: src/backend.rs +[`ModelConfig`]: src/client.rs +[`TranslatingLlmClient::call_rewrite_model`]: src/client.rs +[`LlmClientError`]: src/error.rs diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs new file mode 100644 index 00000000..c907f290 --- /dev/null +++ b/crates/libsy-llm-client/src/backend.rs @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Per-provider backend configuration: wire format, upstream URL, and auth. + +use std::{collections::BTreeMap, fmt}; + +use reqwest::RequestBuilder; +use switchyard_protocol::WireFormat; + +use crate::error::is_overflow_body; + +const ANTHROPIC_VERSION: &str = "2023-06-01"; + +// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new +// provider-wrap is a one-line entry here, not a fork of the parsing logic. +const OPENAI_OVERFLOW_PHRASES: &[&str] = &[ + "maximum context length", + "context length exceeded", + "context window", + "context length is only", + "please reduce the length of the input", +]; + +// Anthropic has no structured `error.code`, so detection is phrase-based only. +const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[ + "prompt is too long", + "maximum number of tokens", + "context window", + "context length", +]; + +/// Shared HTTP configuration for one upstream backend. +#[derive(Clone)] +pub struct HttpBackendConfig { + /// Base URL of the provider API (e.g. `https://api.openai.com/v1`). + pub base_url: String, + /// API key for the provider, loaded by the caller. `None` sends no auth. + pub api_key: Option, + /// Static headers added to every outbound call to this backend. + pub extra_headers: BTreeMap, +} + +impl fmt::Debug for HttpBackendConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HttpBackendConfig") + .field("base_url", &self.base_url) + .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]")) + .field("extra_headers", &self.extra_headers) + .finish() + } +} + +/// A configured upstream backend, one variant per built-in wire format. +/// +/// The variant fixes the wire format, URL path, and auth scheme together so no +/// invalid combination can be constructed. +#[derive(Clone, Debug)] +pub enum Backend { + /// OpenAI-compatible Chat Completions API. + OpenAiChat(HttpBackendConfig), + /// OpenAI Responses API. + OpenAiResponses(HttpBackendConfig), + /// Anthropic Messages API. + Anthropic(HttpBackendConfig), +} + +impl Backend { + /// The wire format the request IR is encoded to for this backend. + pub fn wire_format(&self) -> WireFormat { + match self { + Backend::OpenAiChat(_) => WireFormat::OpenAiChat, + Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses, + Backend::Anthropic(_) => WireFormat::AnthropicMessages, + } + } + + // Shared HTTP config, regardless of variant. + fn config(&self) -> &HttpBackendConfig { + match self { + Backend::OpenAiChat(config) + | Backend::OpenAiResponses(config) + | Backend::Anthropic(config) => config, + } + } + + /// The fully resolved upstream URL for this backend's endpoint. + /// + /// Tolerates base URLs that already include the provider path (or a bare + /// `/v1`), matching the join rules of the existing native backends. + pub fn url(&self) -> String { + let base_url = self.config().base_url.trim_end_matches('/'); + match self { + Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"), + Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"), + Backend::Anthropic(_) => anthropic_url(base_url), + } + } + + /// Applies this backend's auth and version headers to a request builder. + /// + /// OpenAI variants use `Authorization: Bearer `; Anthropic uses + /// `x-api-key: ` plus the required `anthropic-version` header. + pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder { + let api_key = self.config().api_key.as_deref(); + match self { + Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => { + if let Some(api_key) = api_key { + builder = builder.bearer_auth(api_key); + } + } + Backend::Anthropic(_) => { + builder = builder.header("anthropic-version", ANTHROPIC_VERSION); + if let Some(api_key) = api_key { + builder = builder.header("x-api-key", api_key); + } + } + } + builder + } + + /// Static per-backend headers to forward on every call. + pub fn extra_headers(&self) -> &BTreeMap { + &self.config().extra_headers + } + + /// Whether an upstream 400 `body` looks like a context-window overflow for + /// this backend's provider. + pub(crate) fn is_context_overflow(&self, body: &str) -> bool { + match self { + Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body( + body, + |value| { + value + .get("error") + .and_then(|err| err.get("code")) + .and_then(serde_json::Value::as_str) + == Some("context_length_exceeded") + }, + OPENAI_OVERFLOW_PHRASES, + ), + Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES), + } + } +} + +// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL. +fn openai_url(base_url: &str, suffix: &str) -> String { + let base_root = base_url + .strip_suffix("/chat/completions") + .or_else(|| base_url.strip_suffix("/responses")) + .unwrap_or(base_url); + format!("{base_root}{suffix}") +} + +// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL. +fn anthropic_url(base_url: &str) -> String { + if base_url.ends_with("/v1/messages") { + base_url.to_string() + } else if base_url.ends_with("/v1") { + format!("{base_url}/messages") + } else { + format!("{base_url}/v1/messages") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(base_url: &str) -> HttpBackendConfig { + HttpBackendConfig { + base_url: base_url.to_string(), + api_key: Some("secret".to_string()), + extra_headers: BTreeMap::new(), + } + } + + #[test] + fn openai_chat_url_joins_bare_v1() { + let backend = Backend::OpenAiChat(config("https://api.openai.com/v1")); + assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions"); + } + + #[test] + fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() { + assert_eq!( + Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(), + "https://api.openai.com/v1/chat/completions" + ); + assert_eq!( + Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(), + "https://api.openai.com/v1/chat/completions" + ); + } + + #[test] + fn openai_responses_url_uses_responses_path() { + assert_eq!( + Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(), + "https://api.openai.com/v1/responses" + ); + } + + #[test] + fn anthropic_url_join_cases() { + assert_eq!( + Backend::Anthropic(config("https://api.anthropic.com")).url(), + "https://api.anthropic.com/v1/messages" + ); + assert_eq!( + Backend::Anthropic(config("https://api.anthropic.com/v1")).url(), + "https://api.anthropic.com/v1/messages" + ); + assert_eq!( + Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(), + "https://api.anthropic.com/v1/messages" + ); + } + + #[test] + fn wire_format_matches_variant() { + assert_eq!( + Backend::OpenAiChat(config("x")).wire_format(), + WireFormat::OpenAiChat + ); + assert_eq!( + Backend::OpenAiResponses(config("x")).wire_format(), + WireFormat::OpenAiResponses + ); + assert_eq!( + Backend::Anthropic(config("x")).wire_format(), + WireFormat::AnthropicMessages + ); + } + + #[test] + fn openai_detects_canonical_and_wrapped_overflow() { + let backend = Backend::OpenAiChat(config("x")); + assert!(backend + .is_context_overflow(r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#)); + // NVIDIA/LiteLLM message wrap with no structured code. + assert!(backend.is_context_overflow( + r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"# + )); + assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#)); + } + + #[test] + fn anthropic_detects_prompt_too_long() { + let backend = Backend::Anthropic(config("x")); + assert!(backend + .is_context_overflow(r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#)); + assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#)); + } +} diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs new file mode 100644 index 00000000..a745ddef --- /dev/null +++ b/crates/libsy-llm-client/src/client.rs @@ -0,0 +1,828 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral +//! request, call the configured backend over HTTP, decode the neutral response. + +use std::collections::{BTreeMap, HashMap}; + +use async_trait::async_trait; +use futures_util::StreamExt; +use reqwest::RequestBuilder; +use serde_json::Value; +use switchyard_protocol::{ + Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, +}; +use switchyard_translation::{ + decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response, + encode_request, encode_stream, WireFormat, +}; + +use crate::backend::Backend; +use crate::error::{LlmClientError, Result}; +use crate::raw::RawResponse; + +// TODO: Why is this here? What does it do? +// Headers this client owns or that are hop-by-hop; never forwarded from the +// caller's metadata. Auth/version/content-type are set by the backend or the +// JSON body, so a forwarded copy would either be ignored or conflict. Compared +// case-insensitively. Aligns with `_SENSITIVE_HEADERS` in the Python +// `switchyard/lib/request_metadata.py` forwarding logic. +const RESERVED_HEADERS: &[&str] = &[ + "host", + "content-length", + "connection", + "authorization", + "proxy-authorization", + "proxy-authenticate", + "cookie", + "set-cookie", + "x-api-key", + "anthropic-version", + "content-type", +]; + +/// How one model is served: the `default_backend` used when the request does not +/// pin a wire format, plus any `other_backends` reachable over additional formats. +#[derive(Clone, Debug)] +pub struct ModelConfig { + model_name: String, + default_backend: Backend, + other_backends: Option>, +} + +impl ModelConfig { + /// A model named `model_name` served by `default_backend`, optionally reachable + /// over additional wire formats via `other_backends`. + pub fn new( + model_name: impl Into, + default_backend: Backend, + other_backends: Option>, + ) -> Self { + Self { + model_name: model_name.into(), + default_backend, + other_backends, + } + } +} + +/// A client that dispatches neutral-IR requests to per-model HTTP backends. +/// +/// Construct it with a list of [`ModelConfig`]s — one per model, each naming a +/// default [`Backend`] and any additional per-format backends. Each call resolves +/// the model and wire format, encodes the request to that backend's wire format, +/// applies auth and forwarded headers, sends the HTTP request with a shared +/// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered +/// or streamed). +pub struct TranslatingLlmClient { + model_to_config: HashMap, + client: reqwest::Client, +} + +impl TranslatingLlmClient { + /// Builds a client over the given [`ModelConfig`]s, with a fresh shared HTTP + /// client and the built-in translation codecs. + pub fn new(model_configs: &[ModelConfig]) -> Result { + let client = reqwest::Client::builder().build().map_err(|error| { + LlmClientError::Transport(format!("failed to build HTTP client: {error}")) + })?; + let model_to_config = model_configs + .iter() + .map(|config| (config.model_name.clone(), config.clone())) + .collect(); + + Ok(Self { + model_to_config, + client, + }) + } + + /// The backend serving `model` over `format` — the default backend when its + /// format matches, otherwise a matching entry in `other_backends`; `None` when + /// the model is unknown or has no backend for `format`. + pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> { + self.model_to_config.get(model).and_then(|config| { + if config.default_backend.wire_format() == format { + Some(&config.default_backend) + } else { + config + .other_backends + .as_ref() + .and_then(|backends| backends.iter().find(|b| b.wire_format() == format)) + } + }) + } + + /// Calls the backend for `model_name` (or the request's own model), over the + /// wire format the request pins in its metadata (else the model's default + /// backend), and returns the neutral response. + /// + /// Resolution: `model_name` wins over `request.llm_request.model`; the + /// resolved name is both the outer map key and the model id written into the + /// request before translation. Errors with [`LlmClientError::MissingModel`] + /// when neither is set, [`LlmClientError::UnknownModel`] when the model has + /// no backends, and [`LlmClientError::UnknownModelFormat`] when the model has + /// no backend for `format`. + pub async fn call_rewrite_model( + &self, + _ctx: Context, + request: Request, + model_name: Option<&str>, + ) -> Result { + // Own the request's parts so the model can be set without a `mut` param + // and without cloning the messages. `raw_request` is unused here. + let Request { + mut llm_request, + metadata, + .. + } = request; + + let model = model_name + .map(str::to_string) + .or_else(|| llm_request.model.clone()) + .ok_or(LlmClientError::MissingModel)?; + + let orig_format = metadata.as_ref().and_then(|m| m.wire_format); + let wire_format = orig_format.unwrap_or( + self.model_to_config + .get(&model) + .map(|config| config.default_backend.wire_format()) + .ok_or(LlmClientError::UnknownModel(model.clone()))?, + ); + let backend = + self.backend_for(&model, wire_format) + .ok_or(LlmClientError::UnknownModelFormat { + model: model.clone(), + format: wire_format, + })?; + + // The resolved name is the upstream model id (per the crate contract). + llm_request.model = Some(model.clone()); + + let mut body = encode_request(&llm_request, wire_format) + .map_err(|error| LlmClientError::Translation(error.to_string()))?; + // `encode_request` round-trips a preserved same-format body verbatim, + // which keeps the caller's original `model`; force the resolved model so + // the upstream always sees the target id. + set_json_model(&mut body, &model); + let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false); + + let builder = self.client.post(backend.url()).json(&body); + let builder = forward_metadata_headers(builder, metadata.as_ref()); + let builder = apply_extra_headers(builder, backend); + let builder = backend.apply_auth(builder); + + let http_response = builder + .send() + .await + .map_err(|error| LlmClientError::Transport(error.to_string()))?; + let status = http_response.status(); + if !status.is_success() { + let body = http_response + .text() + .await + .unwrap_or_else(|error| format!("")); + if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) { + return Err(LlmClientError::ContextWindowExceeded { + model, + message: body, + }); + } + return Err(LlmClientError::UpstreamHttp { + status: status.as_u16(), + body, + }); + } + + let llm_response = if streaming { + // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is + // transport-agnostic and lives in `switchyard-translation`. + let bytes = http_response.bytes_stream().map(|chunk| { + chunk.map(|bytes| bytes.to_vec()).map_err(|error| { + Box::new(LlmClientError::Stream(format!( + "stream read failed: {error}" + ))) as Box + }) + }); + let chunks = decode_stream(bytes, wire_format) + .map_err(|error| LlmClientError::Stream(error.to_string()))?; + LlmResponse::Stream(chunks) + } else { + let body = http_response.json::().await.map_err(|error| { + LlmClientError::Transport(format!("invalid upstream JSON: {error}")) + })?; + let agg = decode_aggregated_response(&body, wire_format) + .map_err(|error| LlmClientError::Translation(error.to_string()))?; + LlmResponse::Agg(agg) + }; + + Ok(Response { + llm_response, + metadata, + }) + } + + /// The whole decode → call → encode path a wire endpoint needs, in one call. + /// + /// Decodes `raw_http_request` from `wire_format` to the neutral IR, serves it via + /// [`call_rewrite_model`](Self::call_rewrite_model) — the *upstream* wire format is + /// resolved there from the model's backend, independently of `wire_format` — then + /// encodes the neutral response back into `wire_format`. The result is a buffered + /// [`RawResponse::Buffered`] JSON body or a streamed [`RawResponse::Stream`] of + /// wire events (the caller frames the stream as SSE). The response's `model` is + /// restamped with the model the request asked for, never the upstream id. + /// + /// `http_headers` are carried through as the request's + /// [`Metadata::http_headers`] and forwarded to the upstream (minus the reserved + /// set); pass `None` to forward nothing. + pub async fn call_rewrite_model_raw( + &self, + ctx: Context, + raw_http_request: Value, + http_headers: Option>, + model: Option<&str>, + wire_format: WireFormat, + ) -> Result { + let llm_request = decode_request(wire_format, &raw_http_request) + .map_err(|error| LlmClientError::Translation(error.to_string()))?; + // The model the client asked for; restamped onto the response so it never + // leaks the upstream id. + let requested_model = llm_request.model.clone(); + + let request = Request { + llm_request, + raw_request: None, + metadata: Some(Metadata { + session_id: None, + agent_id: None, + task_id: None, + correlation_id: None, + extra_metadata: None, + http_headers, + wire_format: None, + }), + }; + let response = self.call_rewrite_model(ctx, request, model).await?; + + match response.llm_response { + LlmResponse::Agg(agg) => { + let body = + encode_aggregated_response(&agg, wire_format, requested_model.as_deref()) + .map_err(|error| LlmClientError::Translation(error.to_string()))?; + Ok(RawResponse::Buffered(body)) + } + LlmResponse::Stream(chunks) => { + let events = encode_stream(chunks, wire_format, requested_model) + .map_err(|error| LlmClientError::Stream(error.to_string()))?; + Ok(RawResponse::Stream(events)) + } + } + } +} + +#[async_trait] +impl RoutedLlmClient for TranslatingLlmClient { + async fn call( + &self, + ctx: Context, + request: Request, + decision: std::sync::Arc, + ) -> std::result::Result> { + let model_name = Some(decision.selected_model()); + self.call_rewrite_model(ctx, request, model_name) + .await + .map_err(|error| Box::new(error) as Box) + } +} + +// Forwards caller-supplied metadata headers, skipping the reserved set. +fn forward_metadata_headers( + mut builder: RequestBuilder, + metadata: Option<&Metadata>, +) -> RequestBuilder { + let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else { + return builder; + }; + for (name, value) in headers { + if is_reserved_header(name) { + continue; + } + builder = builder.header(name, value); + } + builder +} + +// Adds the backend's static per-call headers. +fn apply_extra_headers(mut builder: RequestBuilder, backend: &Backend) -> RequestBuilder { + for (name, value) in backend.extra_headers() { + builder = builder.header(name, value); + } + builder +} + +// Overwrites the outbound body's `model` field with the resolved model id. +fn set_json_model(body: &mut Value, model: &str) { + if let Value::Object(object) = body { + object.insert("model".to_string(), Value::String(model.to_string())); + } +} + +// Case-insensitive membership test against RESERVED_HEADERS. +fn is_reserved_header(name: &str) -> bool { + RESERVED_HEADERS + .iter() + .any(|reserved| name.eq_ignore_ascii_case(reserved)) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::error::Error; + + use serde_json::json; + use switchyard_protocol::{completion_text, text_request, LlmRequest}; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + use super::*; + use crate::backend::HttpBackendConfig; + + fn config(base_url: &str) -> HttpBackendConfig { + HttpBackendConfig { + base_url: base_url.to_string(), + api_key: Some("secret".to_string()), + extra_headers: BTreeMap::new(), + } + } + + // A one-model config list: "gpt" served over OpenAI Chat at base_url. + fn chat_map(base_url: &str) -> Vec { + vec![ModelConfig::new( + "gpt", + Backend::OpenAiChat(config(base_url)), + None, + )] + } + + fn request_for(model: Option<&str>, stream: bool) -> Request { + let mut llm_request = text_request(model.map(str::to_string), "hi"); + llm_request.stream = stream; + Request { + llm_request, + raw_request: None, + metadata: None, + } + } + + // A request that pins `format` in its metadata, so the client resolves that + // wire format instead of the model's default backend. + fn request_with_wire_format(model: &str, format: WireFormat) -> Request { + let mut request = request_for(Some(model), false); + request.metadata = Some(Metadata { + session_id: None, + agent_id: None, + task_id: None, + correlation_id: None, + extra_metadata: None, + http_headers: None, + wire_format: Some(format), + }); + request + } + + #[tokio::test] + async fn missing_model_errors( + ) -> std::result::Result<(), Box> { + let client = TranslatingLlmClient::new(&[])?; + let Err(error) = client + .call_rewrite_model(Context::default(), request_for(None, false), None) + .await + else { + panic!("expected an error"); + }; + assert!(matches!(error, LlmClientError::MissingModel)); + Ok(()) + } + + #[tokio::test] + async fn unknown_model_errors( + ) -> std::result::Result<(), Box> { + let client = TranslatingLlmClient::new(&[])?; + let Err(error) = client + .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None) + .await + else { + panic!("expected an error"); + }; + assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "gpt")); + Ok(()) + } + + #[tokio::test] + async fn unknown_model_format_errors( + ) -> std::result::Result<(), Box> { + // "gpt" exists but only over OpenAI Chat; the request pins Anthropic. + let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?; + let Err(error) = client + .call_rewrite_model( + Context::default(), + request_with_wire_format("gpt", WireFormat::AnthropicMessages), + None, + ) + .await + else { + panic!("expected an error"); + }; + assert!(matches!( + error, + LlmClientError::UnknownModelFormat { model, format } + if model == "gpt" && format == WireFormat::AnthropicMessages + )); + Ok(()) + } + + #[test] + fn backend_for_resolves_configured_format( + ) -> std::result::Result<(), Box> { + let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?; + // "gpt" is served over OpenAI Chat only; other formats and models miss. + assert!(client.backend_for("gpt", WireFormat::OpenAiChat).is_some()); + assert!(client + .backend_for("gpt", WireFormat::AnthropicMessages) + .is_none()); + assert!(client + .backend_for("missing", WireFormat::OpenAiChat) + .is_none()); + Ok(()) + } + + #[tokio::test] + async fn model_name_arg_wins_over_request_model( + ) -> std::result::Result<(), Box> { + let client = TranslatingLlmClient::new(&[])?; + // Arg "b" is looked up (and reported), not the request's "a". + let Err(error) = client + .call_rewrite_model(Context::default(), request_for(Some("a"), false), Some("b")) + .await + else { + panic!("expected an error"); + }; + assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "b")); + Ok(()) + } + + #[tokio::test] + async fn buffered_openai_chat_round_trips( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "chatcmpl-1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hi there"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + let response = client + .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None) + .await?; + let agg = response.llm_response.into_agg().await?; + assert_eq!(completion_text(&agg), "Hi there"); + + Ok(()) + } + + #[tokio::test] + async fn rewrites_model_to_resolved_upstream_id( + ) -> std::result::Result<(), Box> { + // Inbound body says "switchyard"; the upstream must receive "gpt". + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", "model": "gpt", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + // Inbound model differs from the map key / resolved model. + client + .call_rewrite_model( + Context::default(), + request_for(Some("switchyard"), false), + Some("gpt"), + ) + .await?; + // The body_partial_json matcher asserts the upstream saw model "gpt". + Ok(()) + } + + #[tokio::test] + async fn streaming_openai_chat_aggregates( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\ + data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\ + data: [DONE]\n\n"; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + let response = client + .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None) + .await?; + assert!(matches!(response.llm_response, LlmResponse::Stream(_))); + let agg = response.llm_response.into_agg().await?; + assert_eq!(completion_text(&agg), "Hello world"); + Ok(()) + } + + #[tokio::test] + async fn upstream_500_is_upstream_http( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(500).set_body_string("boom")) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + let Err(error) = client + .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None) + .await + else { + panic!("expected an error"); + }; + assert!(matches!( + error, + LlmClientError::UpstreamHttp { status: 500, .. } + )); + Ok(()) + } + + #[tokio::test] + async fn context_overflow_400_is_mapped( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": {"code": "context_length_exceeded", "message": "too big"} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + let Err(error) = client + .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None) + .await + else { + panic!("expected an error"); + }; + assert!(matches!( + error, + LlmClientError::ContextWindowExceeded { model, .. } if model == "gpt" + )); + Ok(()) + } + + #[tokio::test] + async fn forwards_metadata_headers_except_reserved( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(wiremock::matchers::header("x-request-id", "abc")) + // A forwarded Authorization must NOT override the backend's bearer key. + .and(wiremock::matchers::header("authorization", "Bearer secret")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", "model": "gpt", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {} + }))) + .mount(&server) + .await; + + let mut headers = BTreeMap::new(); + headers.insert("x-request-id".to_string(), "abc".to_string()); + headers.insert("authorization".to_string(), "Bearer client-key".to_string()); + let request = Request { + llm_request: LlmRequest { + model: Some("gpt".to_string()), + ..LlmRequest::default() + }, + raw_request: None, + metadata: Some(Metadata { + session_id: None, + agent_id: None, + task_id: None, + correlation_id: None, + extra_metadata: None, + http_headers: Some(headers), + wire_format: None, + }), + }; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + // Matchers assert forwarded x-request-id survives and reserved + // authorization is the backend's, not the client's. + client + .call_rewrite_model(Context::default(), request, None) + .await?; + Ok(()) + } + + // Minimal `Decision` for driving the client through the `RoutedLlmClient` trait. + struct FixedDecision(&'static str); + + impl Decision for FixedDecision { + fn selected_model(&self) -> &str { + self.0 + } + fn reasoning(&self) -> Option<&str> { + None + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + // Exercises the `RoutedLlmClient` impl: `call` resolves the upstream model from the + // decision (the request carries none) and round-trips a buffered response. + #[tokio::test] + async fn routed_llm_client_serves_the_decision_model( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "chatcmpl-1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "routed hi"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + let decision: std::sync::Arc = std::sync::Arc::new(FixedDecision("gpt")); + // Called through the trait; the request has no model, so "gpt" comes from the decision. + let response = client + .call(Context::default(), request_for(None, false), decision) + .await?; + let agg = response.llm_response.into_agg().await?; + assert_eq!(completion_text(&agg), "routed hi"); + Ok(()) + } + + // Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI + // Chat JSON, with the client-facing `model` restamped over the upstream id. + #[tokio::test] + async fn call_rewrite_model_raw_round_trips_buffered_json( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "chatcmpl-1", + "model": "gpt", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hi there"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "messages": [{"role": "user", "content": "hi"}] + }); + let RawResponse::Buffered(body) = client + .call_rewrite_model_raw( + Context::default(), + raw, + None, + Some("gpt"), + WireFormat::OpenAiChat, + ) + .await? + else { + panic!("expected a buffered response"); + }; + + assert_eq!(body["choices"][0]["message"]["content"], "Hi there"); + // The client sees the model it asked for, not the upstream "gpt". + assert_eq!(body["model"], "client-facing"); + Ok(()) + } + + // Raw path, streaming: an inbound `stream: true` request yields an unframed stream + // of OpenAI Chat chunk objects whose deltas reassemble the completion. + #[tokio::test] + async fn call_rewrite_model_raw_streams_wire_events( + ) -> std::result::Result<(), Box> { + use futures::TryStreamExt; + + let server = MockServer::start().await; + let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\ + data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\ + data: [DONE]\n\n"; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "messages": [{"role": "user", "content": "hi"}], + "stream": true + }); + let RawResponse::Stream(stream) = client + .call_rewrite_model_raw( + Context::default(), + raw, + None, + Some("gpt"), + WireFormat::OpenAiChat, + ) + .await? + else { + panic!("expected a streamed response"); + }; + + let events: Vec = stream.try_collect().await?; + assert!(!events.is_empty(), "expected at least one wire event"); + let content: String = events + .iter() + .filter_map(|event| event["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(content, "Hello world"); + Ok(()) + } + + // Raw path forwards caller headers (minus the reserved set) to the upstream. + #[tokio::test] + async fn call_rewrite_model_raw_forwards_headers( + ) -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(wiremock::matchers::header("x-request-id", "abc")) + // A forwarded authorization must NOT override the backend's bearer key. + .and(wiremock::matchers::header("authorization", "Bearer secret")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "1", "model": "gpt", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {} + }))) + .mount(&server) + .await; + + let mut headers = BTreeMap::new(); + headers.insert("x-request-id".to_string(), "abc".to_string()); + headers.insert("authorization".to_string(), "Bearer client-key".to_string()); + + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]}); + // Matchers assert the forwarded x-request-id survives and reserved + // authorization is the backend's, not the client's. + client + .call_rewrite_model_raw( + Context::default(), + raw, + Some(headers), + Some("gpt"), + WireFormat::OpenAiChat, + ) + .await?; + Ok(()) + } +} diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rs new file mode 100644 index 00000000..d02c0523 --- /dev/null +++ b/crates/libsy-llm-client/src/error.rs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Error type for the LLM client, plus shared context-window-overflow detection. +//! +//! The overflow detection is ported from +//! `switchyard-components/src/backends/context_overflow.rs`; that helper is +//! crate-private there and this crate cannot depend on `switchyard-components`, +//! so the small, self-contained logic is vendored here. + +use serde_json::Value; +use switchyard_translation::WireFormat; +use thiserror::Error; + +/// Result alias for LLM client operations. +pub type Result = std::result::Result; + +/// Failures surfaced while resolving, translating, sending, or decoding a call. +#[derive(Debug, Error)] +pub enum LlmClientError { + /// The resolved model name has no entry in the client's backend map. + #[error("no backend configured for model {0:?}")] + UnknownModel(String), + + /// The model exists but has no backend configured for the requested format. + #[error("model {model:?} has no backend for format {format}")] + UnknownModelFormat { + /// Model that was resolved. + model: String, + /// Wire format requested for the call. + format: WireFormat, + }, + + /// Neither an explicit `model_name` nor a model on the request was given. + #[error("no model given: pass model_name or set request.llm_request.model")] + MissingModel, + + /// Request encoding or response decoding failed in the translation engine. + #[error("translation failed: {0}")] + Translation(String), + + /// The HTTP request could not be sent (connect/timeout/transport failure). + #[error("upstream transport error: {0}")] + Transport(String), + + /// An upstream 400 whose body is detected as a context-window overflow. + /// + /// Kept distinct from [`LlmClientError::UpstreamHttp`] and checked first so a + /// caller can implement evict-and-retry by matching this variant instead of + /// sniffing error strings. + #[error("context window exceeded for model {model}: {message}")] + ContextWindowExceeded { + /// Model that overflowed, as sent upstream. + model: String, + /// Raw upstream error body. + message: String, + }, + + /// A non-2xx upstream response that is not a context-window overflow. + #[error("upstream returned HTTP {status}: {body}")] + UpstreamHttp { + /// Upstream HTTP status code. + status: u16, + /// Raw upstream error body. + body: String, + }, + + /// A streamed response failed mid-flight (read error or malformed frame). + #[error("stream error: {0}")] + Stream(String), +} + +/// Detects a context-overflow body using a provider-supplied structured check +/// and a substring phrase list. +/// +/// Parses the body once, runs the structured check (e.g. against `error.code`), +/// then falls back to matching phrases against `error.message` or — when the +/// body is not JSON — the raw body. Centralizing the shape means each new +/// provider-wrap of the canonical error is a one-line phrase entry, not a fork +/// of the parsing logic. +pub(crate) fn is_overflow_body(body: &str, structured_check: F, phrases: &[&str]) -> bool +where + F: Fn(&Value) -> bool, +{ + if let Ok(value) = serde_json::from_str::(body) { + if structured_check(&value) { + return true; + } + if let Some(message) = value + .get("error") + .and_then(|err| err.get("message")) + .and_then(Value::as_str) + { + if contains_any(message, phrases) { + return true; + } + } + } + // Some upstream proxies return plain-text bodies; fall through to a string + // match on the raw body. + contains_any(body, phrases) +} + +// Case-insensitive substring match of any phrase against the message. +fn contains_any(message: &str, phrases: &[&str]) -> bool { + let lower = message.to_ascii_lowercase(); + phrases.iter().any(|phrase| lower.contains(phrase)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PHRASES: &[&str] = &["context window", "too long"]; + + fn never(_value: &Value) -> bool { + false + } + + #[test] + fn structured_check_short_circuits() { + let body = r#"{"error":{"code":"context_length_exceeded","message":"unrelated"}}"#; + let matched = is_overflow_body( + body, + |value| { + value + .get("error") + .and_then(|err| err.get("code")) + .and_then(Value::as_str) + == Some("context_length_exceeded") + }, + &[], + ); + assert!(matched); + } + + #[test] + fn falls_back_to_message_phrase_match() { + let body = r#"{"error":{"message":"prompt too long"}}"#; + assert!(is_overflow_body(body, never, PHRASES)); + } + + #[test] + fn matches_plain_text_body() { + assert!(is_overflow_body( + "plain text mentioning context window", + never, + PHRASES + )); + } + + #[test] + fn non_match_returns_false() { + let body = r#"{"error":{"message":"rate limit exceeded"}}"#; + assert!(!is_overflow_body(body, never, PHRASES)); + } +} diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs new file mode 100644 index 00000000..22a1c472 --- /dev/null +++ b/crates/libsy-llm-client/src/lib.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP LLM client that speaks Switchyard's neutral IR directly. +//! +//! [`TranslatingLlmClient`] maps a model name (and the wire format resolved from +//! the request) to a [`Backend`], +//! encodes a [`switchyard_protocol::Request`] to that backend's wire format via +//! `switchyard-translation`, applies auth and forwards caller headers, makes the +//! HTTP call with a shared [`reqwest::Client`], and decodes the wire response +//! back to a [`switchyard_protocol::Response`] — supporting both buffered and +//! streamed responses. + +pub mod backend; +pub mod client; +pub mod error; +pub mod raw; + +pub use backend::{Backend, HttpBackendConfig}; +pub use client::{ModelConfig, TranslatingLlmClient}; +pub use error::{LlmClientError, Result}; +pub use raw::RawResponse; +pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/raw.rs b/crates/libsy-llm-client/src/raw.rs new file mode 100644 index 00000000..7391c1a0 --- /dev/null +++ b/crates/libsy-llm-client/src/raw.rs @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The raw wire result of a call. +//! +//! The encoders that produce it — buffered JSON and the streamed wire-event +//! encoder — live in `switchyard-translation` alongside the rest of the neutral-IR +//! codecs; this module only holds the result type they feed. + +use serde_json::Value; +use switchyard_translation::RawEventStream; + +/// The wire result of +/// [`call_rewrite_model_raw`](crate::TranslatingLlmClient::call_rewrite_model_raw): +/// a buffered JSON body, or a live stream of wire events — both already in the +/// requested wire format. +pub enum RawResponse { + /// A complete JSON response body. + Buffered(Value), + /// A live stream of wire-format event objects, ready for SSE framing. + Stream(RawEventStream), +}