From a4f0e7ceeafa7c74722ec71effa7c5ec22ea6d57 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Mon, 20 Jul 2026 14:28:05 -0700 Subject: [PATCH 1/4] refactor(server): run libsy random routing directly Signed-off-by: nachiketb --- Cargo.lock | 6 +- crates/switchyard-py/Cargo.toml | 1 - crates/switchyard-py/src/lib.rs | 2 - crates/switchyard-py/src/server_bindings.rs | 60 - crates/switchyard-server/Cargo.toml | 7 +- crates/switchyard-server/src/cli.rs | 207 +++- crates/switchyard-server/src/lib.rs | 618 +++++----- crates/switchyard-server/src/registry.rs | 119 -- crates/switchyard-server/src/response.rs | 141 +-- crates/switchyard-server/src/sse.rs | 78 +- .../tests/profile_migration.rs | 837 ------------- crates/switchyard-server/tests/server.rs | 1074 +++++------------ docs/cli_reference.md | 7 +- docs/internal/metrics_reference.md | 2 +- switchyard/cli/switchyard_cli.py | 58 +- .../random_routing_request_processor.py | 2 - switchyard/lib/profiles/loader.py | 13 +- switchyard_rust/core.py | 9 - switchyard_rust/server.py | 20 - tests/test_profile_migration.py | 72 -- tests/test_route_bundle.py | 67 - tests/test_serve_profile_config.py | 88 +- 22 files changed, 833 insertions(+), 2655 deletions(-) delete mode 100644 crates/switchyard-py/src/server_bindings.rs delete mode 100644 crates/switchyard-server/src/registry.rs delete mode 100644 crates/switchyard-server/tests/profile_migration.rs delete mode 100644 switchyard_rust/server.py diff --git a/Cargo.lock b/Cargo.lock index 0370a7e5..f5a4d4e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1747,7 +1747,6 @@ dependencies = [ "switchyard-components-v2", "switchyard-core", "switchyard-libsy", - "switchyard-server", "switchyard-translation", "tokio", "tracing", @@ -1764,11 +1763,10 @@ dependencies = [ "clap", "futures-util", "http-body-util", + "libsy", "rustls", - "serde", "serde_json", - "switchyard-components-v2", - "switchyard-core", + "switchyard-llm-client", "switchyard-translation", "tokio", "tower", diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml index 074625a6..560ea2f6 100644 --- a/crates/switchyard-py/Cargo.toml +++ b/crates/switchyard-py/Cargo.toml @@ -27,7 +27,6 @@ serde_json = "1" switchyard-core = { path = "../switchyard-core" } switchyard-components = { path = "../switchyard-components" } switchyard-components-v2 = { path = "../switchyard-components-v2" } -switchyard-server = { path = "../switchyard-server" } switchyard-translation = { path = "../switchyard-translation" } tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync"] } tracing = { version = "0.1", default-features = false, features = ["std"] } diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 6b3fe73d..5ef43c27 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -9,7 +9,6 @@ mod errors; mod libsy_bindings; mod profile_bindings; mod py_serde; -mod server_bindings; mod translation; #[pymodule] @@ -20,6 +19,5 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> { component_bindings::register(module)?; libsy_bindings::register(module)?; profile_bindings::register(module)?; - server_bindings::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs deleted file mode 100644 index a1b5cec8..00000000 --- a/crates/switchyard-py/src/server_bindings.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for the Rust components-v2 profile server. - -use std::net::{IpAddr, SocketAddr}; -use std::path::PathBuf; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use switchyard_server::{run_server, ServerRunOptions, DEFAULT_LISTEN_BACKLOG}; - -use crate::errors::py_core_error; - -/// Run the Rust components-v2 profile server from Python. -#[pyfunction] -#[pyo3(signature = ( - config_path, - host = "127.0.0.1", - port = 4000, - backlog = DEFAULT_LISTEN_BACKLOG, - dry_run = false, -))] -fn run_profile_server( - py: Python<'_>, - config_path: String, - host: &str, - port: u16, - backlog: u32, - dry_run: bool, -) -> PyResult<()> { - let ip: IpAddr = host.parse().map_err(|error| { - PyValueError::new_err(format!( - "host must be an IP address accepted by the Rust server, got {host:?}: {error}" - )) - })?; - let options = ServerRunOptions { - config: PathBuf::from(config_path), - addr: SocketAddr::new(ip, port), - backlog, - dry_run, - tls: None, - }; - - // `detach` runs synchronously with the GIL released, so startup errors still - // return to the Python caller instead of disappearing into a background task. - py.detach(move || { - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .map_err(|error| PyValueError::new_err(error.to_string()))?; - runtime.block_on(run_server(options)).map_err(py_core_error) - }) -} - -/// Registers Rust server bindings with the native Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_profile_server, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 23012b85..34e4c302 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "switchyard-server" version = "0.1.0" -description = "Rust HTTP server surface for Switchyard profiles" +description = "Rust HTTP server surface for libsy algorithms" authors.workspace = true edition.workspace = true license.workspace = true @@ -18,10 +18,9 @@ axum-server = { version = "0.8", features = ["tls-rustls"] } rustls = { version = "0.23" } clap = { version = "4", features = ["derive", "env"] } futures-util = "0.3" -serde = { version = "1", features = ["derive"] } +libsy = { path = "../libsy" } +switchyard-llm-client = { path = "../libsy-llm-client" } serde_json = "1" -switchyard-components-v2 = { path = "../switchyard-components-v2" } -switchyard-core = { path = "../switchyard-core" } switchyard-translation = { path = "../switchyard-translation" } tokio = { version = "1", features = ["macros", "net", "rt-multi-thread", "signal"] } tracing = { version = "0.1", default-features = false, features = ["std"] } diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index d71008ce..17760208 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -1,53 +1,96 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! CLI entrypoint for running the components-v2 Rust profile server. +//! CLI entrypoint for running the libsy server with random routing. +use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; +use std::sync::Arc; -use clap::Parser; -use switchyard_core::{Result, SwitchyardError}; -use switchyard_server::{run_server, ServerRunOptions, TLSOptions, DEFAULT_LISTEN_BACKLOG}; +use clap::{Parser, ValueEnum}; +use libsy::{Algorithm, LlmTarget, LlmTargetSet, RandomAlgo, RoutedLlmClient}; +use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; +use switchyard_server::{ + run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions, + DEFAULT_LISTEN_BACKLOG, +}; const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED); const DEFAULT_PORT: u16 = 4000; +const DEFAULT_ROUTE_MODEL: &str = "switchyard/random"; + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum UpstreamFormat { + #[value(name = "openai-chat")] + OpenAiChat, + #[value(name = "openai-responses")] + OpenAiResponses, + #[value(name = "anthropic")] + Anthropic, +} + +impl UpstreamFormat { + fn backend(self, config: HttpBackendConfig) -> Backend { + match self { + Self::OpenAiChat => Backend::OpenAiChat(config), + Self::OpenAiResponses => Backend::OpenAiResponses(config), + Self::Anthropic => Backend::Anthropic(config), + } + } +} /// Command-line arguments accepted by the Rust server binary. #[derive(Debug, Parser)] #[command( name = "switchyard-server", - about = "Run the Rust Switchyard server from a components-v2 profile config", + about = "Run uniform random routing with libsy", version )] pub(crate) struct ServerArgs { - /// Path to a components-v2 profile config file. - #[arg(short, long, env = "SWITCHYARD_PROFILE_CONFIG", value_name = "PATH")] - pub(crate) config: PathBuf, + /// Public model id clients send to this server. + #[arg(long, default_value = DEFAULT_ROUTE_MODEL)] + route_model: String, + + /// Upstream model id eligible for random selection. Repeat for each target. + #[arg(long = "target", required = true)] + targets: Vec, + + /// Base URL shared by the configured upstream targets. + #[arg(long, env = "SWITCHYARD_UPSTREAM_BASE_URL")] + base_url: String, + + /// Upstream API key. Omit when the backend needs no authentication. + #[arg(long, env = "SWITCHYARD_UPSTREAM_API_KEY")] + api_key: Option, + + /// Provider wire format used by the upstream backend. + #[arg(long, value_enum, default_value = "openai-chat")] + upstream_format: UpstreamFormat, /// Host address to bind. #[arg(long, default_value_t = DEFAULT_HOST)] - pub(crate) host: IpAddr, + host: IpAddr, /// Port to bind. #[arg(short, long, default_value_t = DEFAULT_PORT)] - pub(crate) port: u16, + port: u16, /// TCP listen backlog passed to the socket before Axum accepts traffic. #[arg(long, default_value_t = DEFAULT_LISTEN_BACKLOG)] - pub(crate) backlog: u32, + backlog: u32, - /// Validate and build the config without starting the HTTP listener. + /// Validate the algorithm and client configuration without binding a socket. #[arg(long)] - pub(crate) dry_run: bool, + dry_run: bool, - /// TLS certificate path, PEM format + /// TLS certificate path in PEM format. #[arg(long, requires = "tls_key")] - pub(crate) tls_cert: Option, + tls_cert: Option, - /// TLS certificate key path, PEM format + /// TLS private-key path in PEM format. #[arg(long, requires = "tls_cert")] - pub(crate) tls_key: Option, + tls_key: Option, } impl ServerArgs { @@ -56,30 +99,124 @@ impl ServerArgs { Self::parse() } - fn into_options(self) -> Result { - let mut tls_options = None; - if let (Some(cert), Some(key)) = (self.tls_cert, self.tls_key) { - if !cert.exists() || !key.exists() { - return Err(SwitchyardError::InvalidConfig(format!( - "Invalid path in --tls-cert {} or --tls-key {}. File does not exist.", - cert.display(), - key.display() - ))); + fn into_runtime(self) -> ServerResult<(ServerState, ServerRunOptions)> { + let targets = validated_targets(self.targets)?; + if self.base_url.trim().is_empty() { + return Err(ServerError::new("--base-url must not be empty")); + } + + let backend = self.upstream_format.backend(HttpBackendConfig { + base_url: self.base_url, + api_key: self.api_key, + extra_headers: BTreeMap::new(), + }); + let model_configs = targets + .iter() + .map(|model| ModelConfig::new(model, backend.clone(), None)) + .collect::>(); + let client: Arc = Arc::new( + TranslatingLlmClient::new(&model_configs) + .map_err(|error| ServerError::new(error.to_string()))?, + ); + let target_set = LlmTargetSet::new( + targets + .iter() + .map(|model| LlmTarget { + semantic_name: model.clone(), + llm_client: Some(Arc::clone(&client)), + }) + .collect(), + ); + let algorithm: Arc = Arc::new(RandomAlgo::new(target_set)); + let state = ServerState::new( + self.route_model, + format!("uniform random routing across {}", targets.join(", ")), + algorithm, + )?; + + let tls = match (self.tls_cert, self.tls_key) { + (Some(cert), Some(key)) => { + if !cert.exists() || !key.exists() { + return Err(ServerError::new(format!( + "invalid --tls-cert {} or --tls-key {}: file does not exist", + cert.display(), + key.display() + ))); + } + Some(TlsOptions { cert, key }) } - tls_options = Some(TLSOptions { cert, key }) + _ => None, }; - Ok(ServerRunOptions { - config: self.config, + let options = ServerRunOptions { addr: SocketAddr::new(self.host, self.port), backlog: self.backlog, dry_run: self.dry_run, - tls: tls_options, - }) + tls, + }; + Ok((state, options)) } } -/// Loads config, optionally validates it, then starts the Rust server. -pub(crate) async fn run(args: ServerArgs) -> Result<()> { - let opts = args.into_options()?; - run_server(opts).await +fn validated_targets(targets: Vec) -> ServerResult> { + if targets.iter().any(|target| target.trim().is_empty()) { + return Err(ServerError::new("--target values must not be empty")); + } + let unique = targets.iter().collect::>(); + if unique.len() != targets.len() { + return Err(ServerError::new("--target values must be unique")); + } + Ok(targets) +} + +/// Builds the random algorithm and starts the server. +pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> { + let (state, options) = args.into_runtime()?; + run_server(state, options).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repeated_targets_build_random_runtime() -> ServerResult<()> { + let args = ServerArgs::try_parse_from([ + "switchyard-server", + "--target", + "model/a", + "--target", + "model/b", + "--base-url", + "http://127.0.0.1:9/v1", + "--dry-run", + ]) + .map_err(|error| ServerError::new(error.to_string()))?; + + let (state, options) = args.into_runtime()?; + + assert_eq!(state.served_model().id, DEFAULT_ROUTE_MODEL); + assert!(state + .served_model() + .display_name + .contains("model/a, model/b")); + assert!(options.dry_run); + Ok(()) + } + + #[test] + fn duplicate_targets_are_rejected() -> ServerResult<()> { + let args = ServerArgs::try_parse_from([ + "switchyard-server", + "--target", + "model/a", + "--target", + "model/a", + "--base-url", + "http://127.0.0.1:9/v1", + ]) + .map_err(|error| ServerError::new(error.to_string()))?; + + assert!(args.into_runtime().is_err()); + Ok(()) + } } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 50595aaa..54527bc1 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1,19 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Rust HTTP server surface for components-v2 profile configs. -//! -//! The serving path is profile-native: config files load into -//! `ProfileConfigPlan`, plans build `Profile` runtimes, and HTTP requests -//! call `Profile::run()` directly. +//! Rust HTTP server for libsy algorithms. -mod registry; mod response; mod sse; use std::collections::BTreeMap; +use std::error::Error; +use std::fmt::{Display, Formatter}; use std::net::SocketAddr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -23,88 +20,131 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use axum_server::tls_rustls::RustlsConfig; +use libsy::{Algorithm, Context, Decision, Metadata, Request}; use serde_json::{json, Value}; +use switchyard_llm_client::LlmClientError; use tokio::net::{TcpListener, TcpSocket}; -use switchyard_components_v2::{ - parse_profile_config_path, profile_stats_accumulator, ProfileConfigPlan, ProfileInput, - ProfileResponse, RequestMetadata, RoutingMetadata, -}; -use switchyard_core::{ChatRequest, ChatRequestType, RequestId, Result, SwitchyardError}; -use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; +use switchyard_translation::{decode_request, WireFormat}; -pub use registry::{ProfileRegistry, ServedModel}; - -use crate::response::{translate_chain_response, TranslatedResponse}; +use crate::response::{translate_response, TranslatedResponse}; /// Default TCP listen backlog used by the Rust server. pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535; const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model"; -const HEADER_SELECTED_TIER: &str = "x-model-router-selected-tier"; -const HEADER_CONFIDENCE: &str = "x-model-router-confidence"; -const HEADER_ROUTER_VERSION: &str = "x-model-router-version"; -const HEADER_TOLERANCE: &str = "x-model-router-tolerance"; const HEADER_RATIONALE: &str = "x-model-router-rationale"; const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512; +type BoxError = Box; + +/// Error returned while configuring or running the server. +#[derive(Debug)] +pub struct ServerError { + message: String, +} + +impl ServerError { + /// Creates a server error with a user-facing message. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl Display for ServerError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl Error for ServerError {} + +/// Result returned by server setup and lifecycle operations. +pub type ServerResult = std::result::Result; + +/// Public model entry advertised by `/v1/models`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServedModel { + /// Public model or route ID accepted in inbound request bodies. + pub id: String, + /// Human-readable label shown by startup logs and `/v1/models`. + pub display_name: String, +} + /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { - registry: Arc, - translation: Arc, - translation_policy: TranslationPolicy, + algorithm: Arc, + served_model: Arc, } impl ServerState { - /// Creates server state for a profile registry. - pub fn new(registry: ProfileRegistry) -> Self { - Self { - registry: Arc::new(registry), - translation: Arc::new(TranslationEngine::default()), - translation_policy: TranslationPolicy::default(), + /// Creates server state for one public route backed by a libsy algorithm. + pub fn new( + model: impl Into, + display_name: impl Into, + algorithm: Arc, + ) -> ServerResult { + let model = model.into(); + if model.trim().is_empty() { + return Err(ServerError::new("route model must not be empty")); } + Ok(Self { + algorithm, + served_model: Arc::new(ServedModel { + id: model, + display_name: display_name.into(), + }), + }) } - /// Builds server state from a resolved profile config plan. - pub fn from_plan(plan: &ProfileConfigPlan) -> Result { - Ok(Self::new(ProfileRegistry::from_plan(plan)?)) - } - - /// Returns the profile registry used by this server. - pub fn registry(&self) -> &ProfileRegistry { - self.registry.as_ref() + /// Returns the public model served by this algorithm. + pub fn served_model(&self) -> &ServedModel { + self.served_model.as_ref() } - /// Dispatches one request to the profile selected by its `model` field. - async fn run_profile(&self, input: ProfileInput) -> Result { - let profile = self.registry.lookup(input.request.model())?; - profile.run(input).await + fn validate_model(&self, model: Option<&str>) -> std::result::Result<(), Box> { + let Some(model) = model.filter(|model| !model.trim().is_empty()) else { + return Err(Box::new(error_response( + StatusCode::BAD_REQUEST, + "request body must include a non-empty string `model`", + "invalid_request_error", + "invalid_request_error", + ))); + }; + if model != self.served_model.id { + return Err(Box::new(error_response( + StatusCode::NOT_FOUND, + format!("No route registered for model {model}"), + "model_not_found", + "model_not_found", + ))); + } + Ok(()) } } -/// Runtime options shared by the Rust binary and Python binding. +/// Runtime options shared by server entry points. #[derive(Clone, Debug)] pub struct ServerRunOptions { - /// Path to the components-v2 profile config file. - pub config: PathBuf, /// Socket address to bind. pub addr: SocketAddr, /// TCP listen backlog. pub backlog: u32, - /// Validate and print public model IDs without binding a socket. + /// Validate runtime construction without binding a socket. pub dry_run: bool, - /// If this is set server is HTTPS. - /// Caller must validate that the files exist. - pub tls: Option, + /// TLS certificate configuration, when HTTPS is enabled. + pub tls: Option, } +/// TLS certificate paths used by the server. #[derive(Clone, Debug)] -pub struct TLSOptions { - /// TLS certificate path, PEM format +pub struct TlsOptions { + /// TLS certificate path in PEM format. pub cert: PathBuf, - - /// TLS certificate key path, PEM format + /// TLS private-key path in PEM format. pub key: PathBuf, } @@ -114,19 +154,10 @@ impl ServerRunOptions { } } -/// Builds a server state by loading and resolving a profile config path. -fn state_from_config_path(path: impl AsRef) -> Result { - let document = parse_profile_config_path(path)?; - let plan = document.resolve()?; - ServerState::from_plan(&plan) -} - -/// Entry point to this module. -/// Loads config, optionally validates it, then starts the Rust server. -pub async fn run_server(options: ServerRunOptions) -> Result<()> { - let state = state_from_config_path(&options.config)?; +/// Validates the runtime and starts the HTTP server unless `dry_run` is set. +pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> { if options.dry_run { - println!("{}", dry_run_summary(&options.config, state.registry())); + println!("{}", dry_run_summary(&state)); return Ok(()); } @@ -136,7 +167,7 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> { addr: bound_addr, ..options }; - eprintln!("{}", startup_banner(&server_options, state.registry())); + eprintln!("{}", startup_banner(&server_options, &state)); let router = build_switchyard_router(state); if let Some(tls) = server_options.tls { serve_tls(listener, router, tls).await @@ -145,11 +176,9 @@ pub async fn run_server(options: ServerRunOptions) -> Result<()> { } } -async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Result<()> { - // aws_lc_rs is the default but other crates pull in `ring` also, - // so rustls doesn't know which one to use. Tell it. - if let Err(e) = rustls::crypto::aws_lc_rs::default_provider().install_default() { - eprintln!("TLS crypto provider already installed: {e:?}"); +async fn serve_tls(listener: TcpListener, router: Router, tls: TlsOptions) -> ServerResult<()> { + if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + tracing::debug!(?error, "TLS crypto provider was already installed"); } let config = RustlsConfig::from_pem_file(tls.cert, tls.key) @@ -160,46 +189,38 @@ async fn serve_tls(listener: TcpListener, router: Router, tls: TLSOptions) -> Re let shutdown_handle = handle.clone(); tokio::spawn(async move { shutdown_signal().await; - // Drain connections. Later should loop on connection_count() > 0. shutdown_handle.graceful_shutdown(Some(Duration::from_secs(2))); }); let std_listener = listener.into_std().map_err(server_io_error)?; axum_server::from_tcp_rustls(std_listener, config) .map_err(server_io_error)? - .handle(handle.clone()) + .handle(handle) .serve(router.into_make_service()) .await .map_err(server_io_error) } -/// Serves a Switchyard router on an already-bound TCP listener. -async fn serve(listener: TcpListener, router: Router) -> Result<()> { +async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> { axum::serve(listener, router) .with_graceful_shutdown(shutdown_signal()) .await .map_err(server_io_error) } -/// Builds an Axum router with the same primary endpoint paths as the Python app. -/// Public so that integration tests can see it. +/// Builds an Axum router for the supported LLM wire formats. pub fn build_switchyard_router(state: ServerState) -> Router { Router::new() .route("/v1/chat/completions", post(openai_chat_completions)) .route("/v1/messages", post(anthropic_messages)) .route("/v1/responses", post(openai_responses)) .route("/v1/models", get(models)) - // Keep the legacy routing stats aliases wired to the same handlers. - .route("/v1/stats", get(stats)) - .route("/v1/stats/reset", post(reset_stats)) - .route("/v1/routing/stats", get(stats)) - .route("/v1/routing/stats/reset", post(reset_stats)) .route("/health", get(health)) .fallback(not_found) .with_state(state) } -fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result { +fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult { let socket = if addr.is_ipv4() { TcpSocket::new_v4() } else { @@ -212,8 +233,8 @@ fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> Result { socket.listen(backlog).map_err(server_io_error) } -fn server_io_error(error: std::io::Error) -> SwitchyardError { - SwitchyardError::Other(error.to_string()) +fn server_io_error(error: std::io::Error) -> ServerError { + ServerError::new(error.to_string()) } async fn shutdown_signal() { @@ -231,17 +252,7 @@ async fn openai_chat_completions( headers: HeaderMap, body: std::result::Result, JsonRejection>, ) -> Response { - let body = match llm_json_body(body) { - Ok(body) => body, - Err(response) => return *response, - }; - handle_llm_request( - state, - ChatRequest::openai_chat(body), - WireFormat::OpenAiChat, - metadata_from_headers(&headers, ChatRequestType::OpenAiChat), - ) - .await + handle_endpoint(state, headers, body, WireFormat::OpenAiChat).await } async fn anthropic_messages( @@ -249,35 +260,28 @@ async fn anthropic_messages( headers: HeaderMap, body: std::result::Result, JsonRejection>, ) -> Response { - let body = match llm_json_body(body) { - Ok(body) => body, - Err(response) => return *response, - }; - handle_llm_request( - state, - ChatRequest::anthropic(body), - WireFormat::AnthropicMessages, - metadata_from_headers(&headers, ChatRequestType::Anthropic), - ) - .await + handle_endpoint(state, headers, body, WireFormat::AnthropicMessages).await } async fn openai_responses( State(state): State, headers: HeaderMap, body: std::result::Result, JsonRejection>, +) -> Response { + handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await +} + +async fn handle_endpoint( + state: ServerState, + headers: HeaderMap, + body: std::result::Result, JsonRejection>, + wire_format: WireFormat, ) -> Response { let body = match llm_json_body(body) { Ok(body) => body, Err(response) => return *response, }; - handle_llm_request( - state, - ChatRequest::openai_responses(body), - WireFormat::OpenAiResponses, - metadata_from_headers(&headers, ChatRequestType::OpenAiResponses), - ) - .await + handle_llm_request(state, headers, body, wire_format).await } fn llm_json_body( @@ -296,69 +300,92 @@ fn llm_json_body( async fn handle_llm_request( state: ServerState, - request: ChatRequest, - target_format: WireFormat, - metadata: RequestMetadata, + headers: HeaderMap, + body: Value, + wire_format: WireFormat, ) -> Response { - if let Err(error) = request.validate() { - return llm_error(error); + let llm_request = match decode_request(wire_format, &body) { + Ok(request) => request, + Err(error) => return invalid_body_error(error.to_string()), + }; + let requested_model = llm_request.model.clone(); + if let Err(response) = state.validate_model(requested_model.as_deref()) { + return *response; } - let profile_response = match state.run_profile(ProfileInput { request, metadata }).await { - Ok(response) => response, - Err(error) => return llm_error(error), + let request = Request { + llm_request, + raw_request: Some(body), + metadata: Some(metadata_from_headers(&headers)), + }; + let (trace, response) = match Arc::clone(&state.algorithm) + .run(Context::default(), request) + .await + { + Ok(result) => result, + Err(error) => return algorithm_error(error), }; - let (response, routing_metadata) = profile_response.into_parts(); - - let mut response = match translate_chain_response( - response, - target_format, - Arc::clone(&state.translation), - state.translation_policy.clone(), - ) { + + let mut response = match translate_response(response, wire_format, requested_model) { Ok(TranslatedResponse::Buffered(body)) => Json(body).into_response(), Ok(TranslatedResponse::Stream(stream)) => stream.into_response(), Err(error) => return server_error(error.to_string()), }; - attach_routing_metadata_headers(&mut response, routing_metadata.as_ref()); + if let Some(decision) = trace.last() { + attach_routing_headers(&mut response, decision.as_ref()); + } response } -fn attach_routing_metadata_headers(response: &mut Response, metadata: Option<&RoutingMetadata>) { - let Some(metadata) = metadata else { - return; - }; - for (name, value) in routing_metadata_headers(metadata) { - let name = HeaderName::from_static(name); - let Ok(value) = HeaderValue::from_str(&value) else { - continue; - }; - response.headers_mut().insert(name, value); +fn metadata_from_headers(headers: &HeaderMap) -> Metadata { + Metadata { + session_id: None, + agent_id: None, + task_id: None, + correlation_id: header_text(headers, "x-request-id"), + extra_metadata: None, + http_headers: Some(normalized_headers(headers)), + // Inbound format must not constrain the independently configured upstream. + wire_format: None, } } -fn routing_metadata_headers(metadata: &RoutingMetadata) -> Vec<(&'static str, String)> { - [ - (HEADER_SELECTED_MODEL, text_header(&metadata.selected_model)), - (HEADER_SELECTED_TIER, text_header(&metadata.selected_tier)), - (HEADER_CONFIDENCE, number_header(metadata.confidence)), - (HEADER_ROUTER_VERSION, text_header(&metadata.router_version)), - (HEADER_TOLERANCE, number_header(metadata.tolerance)), - (HEADER_RATIONALE, text_header(&metadata.rationale)), - ] - .into_iter() - .filter_map(|(name, value)| value.map(|value| (name, value))) - .collect() +fn header_text(headers: &HeaderMap, name: &str) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_string) } -fn text_header(value: &Option) -> Option { - value.as_deref().and_then(sanitize_routing_header_value) +fn normalized_headers(headers: &HeaderMap) -> BTreeMap { + headers + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|value| (name.as_str().to_ascii_lowercase(), value.to_string())) + }) + .collect() +} + +fn attach_routing_headers(response: &mut Response, decision: &dyn Decision) { + insert_routing_header(response, HEADER_SELECTED_MODEL, decision.selected_model()); + if let Some(reasoning) = decision.reasoning() { + insert_routing_header(response, HEADER_RATIONALE, reasoning); + } } -fn number_header(value: Option) -> Option { - value - .filter(|value| value.is_finite()) - .map(|value| value.to_string()) +fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) { + let Some(value) = sanitize_routing_header_value(value) else { + return; + }; + let Ok(value) = HeaderValue::from_str(&value) else { + return; + }; + response + .headers_mut() + .insert(HeaderName::from_static(name), value); } fn sanitize_routing_header_value(value: &str) -> Option { @@ -366,112 +393,83 @@ fn sanitize_routing_header_value(value: &str) -> Option { (!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect()) } -fn metadata_from_headers(headers: &HeaderMap, inbound_format: ChatRequestType) -> RequestMetadata { - RequestMetadata { - request_id: request_id_from_headers(headers), - inbound_format: Some(inbound_format), - headers: normalized_headers(headers), - } -} - -fn request_id_from_headers(headers: &HeaderMap) -> Option { - headers - .get("x-request-id") - .and_then(|value| value.to_str().ok()) - .and_then(|value| RequestId::new(value.to_string()).ok()) -} - -fn normalized_headers(headers: &HeaderMap) -> BTreeMap> { - let mut normalized = BTreeMap::>::new(); - for (name, value) in headers { - let Ok(value) = value.to_str() else { - continue; - }; - normalized - .entry(name.as_str().to_ascii_lowercase()) - .or_default() - .push(value.to_string()); - } - normalized -} - -fn llm_error(error: SwitchyardError) -> Response { +fn algorithm_error(error: BoxError) -> Response { + let Some(error) = error.downcast_ref::() else { + return server_error(error.to_string()); + }; match error { - SwitchyardError::ModelNotFound { model } => ( - StatusCode::NOT_FOUND, - Json(json!({ - "error": { - "message": format!("No route registered for model {}", model.as_str()), - "type": "model_not_found", - "code": "model_not_found", - } - })), - ) - .into_response(), - SwitchyardError::InvalidConfig(message) | SwitchyardError::InvalidRequest(message) => ( + LlmClientError::UnknownModel(model) => error_response( + StatusCode::BAD_GATEWAY, + format!("No upstream backend configured for model {model}"), + "upstream_error", + "upstream_model_not_found", + ), + LlmClientError::UnknownModelFormat { model, format } => error_response( + StatusCode::BAD_GATEWAY, + format!("No upstream backend configured for model {model} and format {format}"), + "upstream_error", + "upstream_format_not_found", + ), + LlmClientError::MissingModel => error_response( StatusCode::BAD_REQUEST, - Json(json!({ - "error": { - "message": message, - "type": "invalid_request_error", - "code": "invalid_request_error", - } - })), - ) - .into_response(), - SwitchyardError::InvalidId(error) => ( + error.to_string(), + "invalid_request_error", + "invalid_request_error", + ), + LlmClientError::ContextWindowExceeded { message, .. } => error_response( StatusCode::BAD_REQUEST, - Json(json!({ - "error": { - "message": error.to_string(), - "type": "invalid_request_error", - "code": "invalid_request_error", - } - })), - ) - .into_response(), - SwitchyardError::UpstreamHttp { - provider, - status_code, + message, + "invalid_request_error", + "context_length_exceeded", + ), + LlmClientError::UpstreamHttp { status, body } => error_response( + StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY), body, - } => ( - StatusCode::from_u16(status_code).unwrap_or(StatusCode::BAD_GATEWAY), - Json(json!({ - "error": { - "message": body, - "type": "upstream_error", - "code": "upstream_error", - "provider": provider, - } - })), - ) - .into_response(), - error => server_error(error.to_string()), + "upstream_error", + "upstream_error", + ), + LlmClientError::Translation(message) + | LlmClientError::Transport(message) + | LlmClientError::Stream(message) => error_response( + StatusCode::BAD_GATEWAY, + message, + "upstream_error", + "upstream_error", + ), } } -fn server_error(message: String) -> Response { - ( +fn server_error(message: impl Into) -> Response { + error_response( StatusCode::INTERNAL_SERVER_ERROR, - Json(json!({ - "error": { - "message": message, - "type": "server_error", - "code": "server_error", - } - })), + message, + "server_error", + "server_error", ) - .into_response() } fn invalid_body_error(message: impl Into) -> Response { - ( + error_response( StatusCode::BAD_REQUEST, + message, + "invalid_request_error", + "invalid_body", + ) +} + +fn error_response( + status: StatusCode, + message: impl Into, + error_type: &'static str, + code: &'static str, +) -> Response { + ( + status, Json(json!({ "error": { "message": message.into(), - "type": "invalid_request_error", - "code": "invalid_body", + "type": error_type, + "code": code, } })), ) @@ -479,22 +477,7 @@ fn invalid_body_error(message: impl Into) -> Response { } async fn models(State(state): State) -> Json { - let entries = state.registry().served_models(); - Json(model_list_payload(&entries)) -} - -async fn stats() -> Response { - match profile_stats_accumulator().snapshot() { - Ok(snapshot) => Json(json!(snapshot)).into_response(), - Err(error) => server_error(error.to_string()), - } -} - -async fn reset_stats() -> Response { - match profile_stats_accumulator().reset() { - Ok(()) => Json(json!({"status": "reset"})).into_response(), - Err(error) => server_error(error.to_string()), - } + Json(model_list_payload(state.served_model())) } async fn health() -> Json { @@ -502,38 +485,29 @@ async fn health() -> Json { } async fn not_found() -> Response { - ( + error_response( StatusCode::NOT_FOUND, - Json(json!({ - "detail": "Not Found", - })), + "Not Found", + "not_found", + "endpoint_not_found", ) - .into_response() } -fn model_list_payload(entries: &[ServedModel]) -> Value { - let data = entries.iter().map(model_entry_json).collect::>(); - let model_ids = entries - .iter() - .map(|entry| entry.id.as_str().to_string()) - .collect::>(); - let first_id = model_ids.first().cloned(); - let last_id = model_ids.last().cloned(); - +fn model_list_payload(entry: &ServedModel) -> Value { json!({ "object": "list", - "data": data, - "first_id": first_id, - "last_id": last_id, + "data": [model_entry_json(entry)], + "first_id": entry.id, + "last_id": entry.id, "has_more": false, - "default_model": first_id, - "model_pool": model_ids, + "default_model": entry.id, + "model_pool": [entry.id], }) } fn model_entry_json(entry: &ServedModel) -> Value { json!({ - "id": entry.id.as_str(), + "id": entry.id, "object": "model", "type": "model", "created": 0, @@ -552,17 +526,17 @@ fn model_entry_json(entry: &ServedModel) -> Value { }) } -fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String { - let entries = registry.served_models(); +fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String { let scheme = if options.is_tls() { "https" } else { "http" }; let listen_url = url_for_addr(scheme, options.addr); let local_url = local_url_for_addr(scheme, options.addr); let mut output = String::new(); - push_line(&mut output, "Switchyard Rust profile server"); + push_line(&mut output, "Switchyard libsy server"); + push_line(&mut output, format!(" model: {}", state.served_model.id)); push_line( &mut output, - format!(" config: {}", options.config.display()), + format!(" algorithm: {}", state.served_model.display_name), ); push_line(&mut output, format!(" listening: {listen_url}")); if local_url != listen_url { @@ -575,50 +549,22 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Str push_line(&mut output, " POST /v1/chat/completions"); push_line(&mut output, " POST /v1/messages"); push_line(&mut output, " POST /v1/responses"); - push_line(&mut output, " GET /v1/routing/stats"); - push_line(&mut output, " POST /v1/routing/stats/reset"); - push_line(&mut output, ""); - push_line(&mut output, " available models:"); - for entry in &entries { - if entry.display_name == entry.id.as_str() { - push_line(&mut output, format!(" - {}", entry.id.as_str())); - } else { - push_line( - &mut output, - format!(" - {} ({})", entry.id.as_str(), entry.display_name), - ); - } - } push_line(&mut output, ""); push_line(&mut output, " try:"); push_line(&mut output, format!(" curl -s {local_url}/health")); - push_line(&mut output, format!(" curl -s {local_url}/v1/models")); - if let Some(entry) = entries.first() { - let chat_payload = json!({ - "model": entry.id.as_str(), - "messages": [{"role": "user", "content": "Say OK"}], - "max_tokens": 256, // Space for thinking tokens - }); - push_line( - &mut output, - format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'") - ); - push_line( - &mut output, - format!(" curl -s {local_url}/v1/messages -H 'content-type: application/json' -H 'anthropic-version: 2023-06-01' -d '{chat_payload}'") - ); - - let responses_payload = json!({ - "model": entry.id.as_str(), - "max_output_tokens": 256, - "input": "Say OK", - }); - push_line(&mut output, format!(" curl -s {local_url}/v1/responses -H 'content-type: application/json' -d '{responses_payload}'")); - } + let chat_payload = json!({ + "model": state.served_model.id, + "messages": [{"role": "user", "content": "Say OK"}], + "max_tokens": 256, + }); + push_line( + &mut output, + format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'"), + ); if options.is_tls() { push_line( &mut output, - " For self-signed certificates use `curl --insecure ..`", + " For self-signed certificates use `curl --insecure`.", ); } push_line(&mut output, ""); @@ -626,21 +572,11 @@ fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> Str output } -fn dry_run_summary(path: &Path, registry: &ProfileRegistry) -> String { - let entries = registry.served_models(); - let mut output = String::new(); - push_line( - &mut output, - format!( - "config OK: {}, public_models={}", - path.display(), - entries.len() - ), - ); - for entry in &entries { - push_line(&mut output, format!(" - {}", entry.id.as_str())); - } - output +fn dry_run_summary(state: &ServerState) -> String { + format!( + "server OK: model={}, algorithm={}\n", + state.served_model.id, state.served_model.display_name + ) } fn push_line(output: &mut String, line: impl AsRef) { diff --git a/crates/switchyard-server/src/registry.rs b/crates/switchyard-server/src/registry.rs deleted file mode 100644 index 20b829c0..00000000 --- a/crates/switchyard-server/src/registry.rs +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Profile registry used by the components-v2 Rust server. - -use std::sync::Arc; - -use serde::{Deserialize, Serialize}; -use switchyard_components_v2::{ - PassthroughProfileConfig, Profile, ProfileConfig, ProfileConfigPlan, -}; -use switchyard_core::{ModelId, Result, SwitchyardError}; - -/// Public model entry advertised by `/v1/models`. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct ServedModel { - /// Public model or route ID accepted in inbound request bodies. - pub id: ModelId, - /// Human-readable label shown by CLI startup logs and `/v1/models`. - pub display_name: String, -} - -#[derive(Clone)] -struct RegistryEntry { - model: ServedModel, - profile: Arc, -} - -/// Exact-match registry from inbound `model` values to components-v2 profiles. -#[derive(Clone, Default)] -pub struct ProfileRegistry { - entries: Vec, -} - -impl ProfileRegistry { - /// Builds a registry from already-built exact-match profile runtimes. - pub fn from_profiles( - entries: impl IntoIterator, String)>, - ) -> Result { - let mut registry = Self::default(); - for (model_id, profile, display_name) in entries { - registry.insert(model_id, profile, display_name)?; - } - Ok(registry) - } - - /// Builds a registry from a resolved profile config plan. - pub fn from_plan(plan: &ProfileConfigPlan) -> Result { - let mut registry = Self::default(); - - for profile_id in plan.profile_ids() { - let model_id = ModelId::new(profile_id.as_str())?; - let profile: Arc = Arc::from(plan.build_profile(profile_id)?); - let display_name = plan.profile_type(profile_id).unwrap_or("profile"); - registry.insert(model_id, profile, display_name)?; - } - - for (_target_id, target) in plan.targets() { - let profile: Arc = Arc::from( - PassthroughProfileConfig { - target: target.clone(), - } - .build_boxed()?, - ); - registry.insert( - ModelId::new(target.id.as_str())?, - Arc::clone(&profile), - target.model.as_str(), - )?; - } - - Ok(registry) - } - - /// Returns the profile for an inbound request model. - pub fn lookup(&self, model: Option<&str>) -> Result> { - let Some(model) = model else { - return Err(SwitchyardError::InvalidRequest( - "request body must include a non-empty string `model`".to_string(), - )); - }; - let model = ModelId::new(model)?; - self.entries - .iter() - .find(|entry| entry.model.id == model) - .map(|entry| Arc::clone(&entry.profile)) - .ok_or(SwitchyardError::ModelNotFound { model }) - } - - /// Returns model entries in deterministic registration order. - pub fn served_models(&self) -> Vec { - self.entries - .iter() - .map(|entry| entry.model.clone()) - .collect() - } - - fn insert( - &mut self, - model_id: ModelId, - profile: Arc, - display_name: impl Into, - ) -> Result<()> { - if self.entries.iter().any(|entry| entry.model.id == model_id) { - return Err(SwitchyardError::DuplicateRegistration { - kind: "model", - id: model_id.to_string(), - }); - } - self.entries.push(RegistryEntry { - model: ServedModel { - id: model_id, - display_name: display_name.into(), - }, - profile, - }); - Ok(()) - } -} diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rs index 0a27718e..3db288aa 100644 --- a/crates/switchyard-server/src/response.rs +++ b/crates/switchyard-server/src/response.rs @@ -1,19 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Response translation glue for Rust server endpoints. +//! Response encoding glue for libsy server endpoints. -use std::sync::Arc; +use std::error::Error; use axum::response::sse::Sse; +use libsy::{LlmResponse, Response}; use serde_json::Value; -use switchyard_core::{BoxResponseStream, ChatResponse, Result, StreamEvent, SwitchyardError}; -use switchyard_translation::{ - StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat, -}; +use switchyard_translation::{encode_aggregated_response, encode_stream, WireFormat}; use crate::sse::{frame_stream, SseFrameStream}; +type BoxError = Box; + pub(crate) enum TranslatedResponse { /// Complete JSON response body ready for an Axum JSON response. Buffered(Value), @@ -21,124 +21,21 @@ pub(crate) enum TranslatedResponse { Stream(Sse), } -/// Translates a profile response into the endpoint's expected response shape. -pub(crate) fn translate_chain_response( - response: ChatResponse, +/// Encodes a libsy response into the endpoint's wire format. +pub(crate) fn translate_response( + response: Response, target_format: WireFormat, - translation: Arc, - policy: TranslationPolicy, -) -> Result { - match response { - ChatResponse::OpenAiCompletion(response) => translate_buffered_response( - response.into_body(), - WireFormat::OpenAiChat, - target_format, - translation, - &policy, - ), - ChatResponse::OpenAiResponsesCompletion(response) => translate_buffered_response( - response.into_body(), - WireFormat::OpenAiResponses, + requested_model: Option, +) -> Result { + match response.llm_response { + LlmResponse::Agg(response) => Ok(TranslatedResponse::Buffered(encode_aggregated_response( + &response, target_format, - translation, - &policy, - ), - ChatResponse::AnthropicCompletion(response) => translate_buffered_response( - response.into_body(), - WireFormat::AnthropicMessages, + requested_model.as_deref(), + )?)), + LlmResponse::Stream(stream) => Ok(TranslatedResponse::Stream(frame_stream( + encode_stream(stream, target_format, requested_model)?, target_format, - translation, - &policy, - ), - ChatResponse::OpenAiStream(stream) => Ok(TranslatedResponse::Stream( - translate_stream_response(stream, WireFormat::OpenAiChat, target_format, translation), - )), - ChatResponse::OpenAiResponsesStream(stream) => { - Ok(TranslatedResponse::Stream(translate_stream_response( - stream, - WireFormat::OpenAiResponses, - target_format, - translation, - ))) - } - ChatResponse::AnthropicStream(stream) => { - Ok(TranslatedResponse::Stream(translate_stream_response( - stream, - WireFormat::AnthropicMessages, - target_format, - translation, - ))) - } - } -} - -fn translate_buffered_response( - body: Value, - source_format: WireFormat, - target_format: WireFormat, - translation: Arc, - policy: &TranslationPolicy, -) -> Result { - if source_format == target_format { - return Ok(TranslatedResponse::Buffered(body)); + ))), } - - let output = translation - .translate_response(source_format, target_format, &body, policy) - .map_err(|error| { - SwitchyardError::Other(format!( - "failed to translate {source_format} response to {target_format}: {error}" - )) - })?; - Ok(TranslatedResponse::Buffered(output.body)) -} - -fn translate_stream_response( - mut stream: BoxResponseStream, - source_format: WireFormat, - target_format: WireFormat, - translation: Arc, -) -> Sse { - let events = async_stream::try_stream! { - let mut translation_state = StreamTranslationState::new(source_format, target_format); - while let Some(event) = futures_util::StreamExt::next(&mut stream).await { - let event = event?; - match event { - StreamEvent::Json(value) => { - if source_format == target_format { - yield value; - } else { - for translated in translation.translate_event( - &mut translation_state, - source_format, - target_format, - &value, - ).map_err(|error| { - SwitchyardError::Other(format!( - "failed to translate {source_format} stream event to {target_format}: {error}" - )) - })? { - yield translated; - } - } - } - StreamEvent::Text(text) => { - yield Value::String(text); - } - } - } - - if source_format != target_format { - for translated in translation.finish_stream(&mut translation_state, target_format) - .map_err(|error| { - SwitchyardError::Other(format!( - "failed to finish {target_format} stream translation: {error}" - )) - })? { - yield translated; - } - } - }; - - frame_stream(events, target_format) } diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rs index 3d4f0e38..d1c25ce4 100644 --- a/crates/switchyard-server/src/sse.rs +++ b/crates/switchyard-server/src/sse.rs @@ -3,35 +3,47 @@ //! SSE framing helpers for OpenAI, Anthropic, and Responses endpoints. +use std::convert::Infallible; + use axum::response::sse::{Event, Sse}; use futures_util::Stream; use serde_json::{json, Value}; -use switchyard_core::{Result, SwitchyardError}; -use switchyard_translation::WireFormat; +use switchyard_translation::{RawEventStream, WireFormat}; /// Boxed stream type accepted by Axum's SSE response wrapper. pub(crate) type SseFrameStream = - std::pin::Pin> + Send>>; + std::pin::Pin> + Send>>; /// Converts translated JSON events into endpoint-specific SSE frames. pub(crate) fn frame_stream( - stream: impl Stream> + Send + 'static, + stream: RawEventStream, target_format: WireFormat, ) -> Sse { let framed = async_stream::stream! { - let mut stream = Box::pin(stream); + let mut stream = stream; + let mut failed = false; while let Some(item) = futures_util::StreamExt::next(&mut stream).await { - match item { - Ok(value) => yield frame_event(target_format, value), + let event = match item { + Ok(value) => match frame_event(target_format, value) { + Ok(event) => event, + Err(error) => { + failed = true; + error_event(target_format, error.to_string()) + } + }, Err(error) => { tracing::warn!(error = %error, "stream iteration failed"); - yield Ok(error_event(target_format, error.to_string())); - return; + failed = true; + error_event(target_format, error.to_string()) } + }; + yield Ok(event); + if failed { + break; } } - if target_format == WireFormat::OpenAiChat { + if !failed && target_format == WireFormat::OpenAiChat { yield Ok(Event::default().data("[DONE]")); } }; @@ -39,24 +51,16 @@ pub(crate) fn frame_stream( Sse::new(Box::pin(framed) as SseFrameStream) } -fn frame_event( - target_format: WireFormat, - value: Value, -) -> std::result::Result { +fn frame_event(target_format: WireFormat, value: Value) -> Result { match target_format { - WireFormat::OpenAiChat => Event::default() - .json_data(value) - .map_err(|error| SwitchyardError::Other(error.to_string())), + WireFormat::OpenAiChat => Event::default().json_data(value), WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => { let event_type = value .get("type") .and_then(Value::as_str) .unwrap_or("message") .to_string(); - Event::default() - .event(event_type) - .json_data(value) - .map_err(|error| SwitchyardError::Other(error.to_string())) + Event::default().event(event_type).json_data(value) } } } @@ -86,3 +90,35 @@ fn error_event(target_format: WireFormat, message: String) -> Event { } } } + +#[cfg(test)] +mod tests { + use std::{error::Error, io}; + + use axum::{body::to_bytes, response::IntoResponse}; + use futures_util::stream; + + use super::*; + + type TestResult = Result<(), Box>; + + #[tokio::test] + async fn stream_error_terminates_without_done_marker() -> TestResult { + let failure: Box = Box::new(io::Error::other("boom")); + let stream: RawEventStream = Box::pin(stream::iter(vec![ + Ok(json!({"id": "before"})), + Err(failure), + Ok(json!({"id": "after"})), + ])); + + let response = frame_stream(stream, WireFormat::OpenAiChat).into_response(); + let body = String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?; + + // A stream error is terminal: later chunks and success markers must not be emitted. + assert!(body.contains("before")); + assert!(body.contains("boom")); + assert!(!body.contains("after")); + assert!(!body.contains("[DONE]")); + Ok(()) + } +} diff --git a/crates/switchyard-server/tests/profile_migration.rs b/crates/switchyard-server/tests/profile_migration.rs deleted file mode 100644 index abcb5cd3..00000000 --- a/crates/switchyard-server/tests/profile_migration.rs +++ /dev/null @@ -1,837 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Migration tests proving components-v2 profiles cover legacy serving contracts. - -use std::io::ErrorKind; -use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream}; -use std::sync::{Arc, Mutex}; -use std::thread; - -use axum::body::Body; -use axum::http::{Request, StatusCode}; -use http_body_util::BodyExt; -use serde_json::{json, Value}; -use switchyard_components_v2::{ - parse_profile_config_str, profile_stats_accumulator, ProfileConfigFormat, -}; -use switchyard_server::{build_switchyard_router, ServerState}; -use tokio::sync::Mutex as TokioMutex; -use tower::ServiceExt; - -static STATS_TEST_LOCK: TokioMutex<()> = TokioMutex::const_new(()); - -#[tokio::test] -async fn noop_profile_serves_all_inbound_formats_without_upstream() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let models = app - .clone() - .oneshot(request("GET", "/v1/models", None)?) - .await?; - assert_eq!(models.status(), StatusCode::OK); - assert_eq!(json_body(models).await?["model_pool"], json!(["bench"])); - - let chat = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("bench")), - )?) - .await?; - assert_eq!(chat.status(), StatusCode::OK); - assert_eq!( - json_body(chat).await?["choices"][0]["message"]["content"], - "ok" - ); - - let messages = app - .clone() - .oneshot(request( - "POST", - "/v1/messages", - Some(json!({ - "model": "bench", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(messages.status(), StatusCode::OK); - assert_eq!(json_body(messages).await?["content"][0]["text"], "ok"); - - let responses = app - .oneshot(request( - "POST", - "/v1/responses", - Some(json!({"model": "bench", "input": "hi"})), - )?) - .await?; - assert_eq!(responses.status(), StatusCode::OK); - assert_eq!( - json_body(responses).await?["output"][0]["content"][0]["text"], - "ok" - ); - Ok(()) -} - -#[tokio::test] -async fn passthrough_profile_routes_all_inbound_formats_to_configured_target() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![ - StubResponse::ok(), - StubResponse::ok(), - StubResponse::ok(), - ])? - else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - direct: - model: provider/direct - format: openai - base_url: {base_url} -profiles: - direct-profile: - type: passthrough - target: direct -"#, - base_url = stub.base_url - ))?); - - let chat = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("direct-profile")), - )?) - .await?; - assert_eq!(chat.status(), StatusCode::OK); - let body = json_body(chat).await?; - assert_eq!(body["choices"][0]["message"]["content"], "stub-ok"); - - let messages = app - .clone() - .oneshot(request( - "POST", - "/v1/messages", - Some(json!({ - "model": "direct-profile", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(messages.status(), StatusCode::OK); - assert_eq!(json_body(messages).await?["content"][0]["text"], "stub-ok"); - - let responses = app - .oneshot(request( - "POST", - "/v1/responses", - Some(json!({"model": "direct-profile", "input": "hi"})), - )?) - .await?; - assert_eq!(responses.status(), StatusCode::OK); - assert_eq!( - json_body(responses).await?["output"][0]["content"][0]["text"], - "stub-ok" - ); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 3); - for request in seen { - assert_eq!(request["method"], "POST"); - assert_eq!(request["path"], "/v1/chat/completions"); - assert_eq!(request["body"]["model"], "provider/direct"); - } - Ok(()) -} - -#[tokio::test] -async fn random_routing_profile_covers_strong_and_weak_selection() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let cases = [(1.0, "provider/strong"), (0.0, "provider/weak")]; - for (strong_probability, expected_model) in cases { - let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - strong: - model: provider/strong - format: openai - base_url: {base_url} - weak: - model: provider/weak - format: openai - base_url: {base_url} -profiles: - random-profile: - type: random-routing - strong: strong - weak: weak - strong_probability: {strong_probability} - rng_seed: 7 -"#, - base_url = stub.base_url - ))?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("random-profile")), - )?) - .await?; - - assert_eq!(response.status(), StatusCode::OK); - let seen = stub.requests()?; - assert_eq!(seen.len(), 1); - assert_eq!(seen[0]["body"]["model"], expected_model); - } - Ok(()) -} - -#[tokio::test] -async fn stage_router_profile_threshold_zero_uses_dimensions_signal_path() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![StubResponse::ok()])? else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - strong: - model: provider/strong - format: openai - base_url: {base_url} - weak: - model: provider/weak - format: openai - base_url: {base_url} -profiles: - smart-stage_router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.0 -"#, - base_url = stub.base_url - ))?); - - let response = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("smart-stage_router")), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 1); - assert_eq!(seen[0]["body"]["model"], "provider/weak"); - - let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?; - let stats = json_body(stats).await?; - assert_eq!(stats["routing_decisions"]["stage_router"]["dimensions"], 1); - assert_eq!(stats["classifier"]["total_requests"], 0); - assert_eq!(stats["models"]["provider/weak"]["tier"], "efficient"); - Ok(()) -} - -#[tokio::test] -async fn stage_router_profile_threshold_one_uses_llm_classifier_path() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![ - StubResponse::classifier("capable"), - StubResponse::ok(), - ])? - else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - strong: - model: provider/strong - format: openai - base_url: {base_url} - weak: - model: provider/weak - format: openai - base_url: {base_url} -profiles: - smart-stage_router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 1.0 - classifier: - model: classifier/model - api_key: test-key - base_url: {base_url} - timeout_secs: 5.0 -"#, - base_url = stub.base_url - ))?); - - let response = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("smart-stage_router")), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 2); - assert_eq!(seen[0]["body"]["model"], "classifier/model"); - assert_eq!(seen[1]["body"]["model"], "provider/strong"); - - let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?; - let stats = json_body(stats).await?; - assert_eq!( - stats["routing_decisions"]["stage_router"]["llm-classifier"], - 1 - ); - assert_eq!(stats["classifier"]["total_requests"], 1); - assert_eq!(stats["models"]["provider/strong"]["tier"], "capable"); - Ok(()) -} - -#[tokio::test] -async fn stage_router_profile_retries_fallback_after_context_overflow() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![StubResponse::context_overflow(), StubResponse::ok()])? - else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - strong: - model: provider/strong - format: openai - base_url: {base_url} - weak: - model: provider/weak - format: openai - base_url: {base_url} -profiles: - smart-stage_router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: efficient_first - confidence_threshold: 0.7 -"#, - base_url = stub.base_url - ))?); - - let response = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("smart-stage_router")), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 2); - assert_eq!(seen[0]["body"]["model"], "provider/weak"); - assert_eq!(seen[1]["body"]["model"], "provider/strong"); - - let stats = app.oneshot(request("GET", "/v1/stats", None)?).await?; - let stats = json_body(stats).await?; - assert_eq!(stats["routing_decisions"]["stage_router"]["fall_open"], 1); - assert_eq!(stats["models"]["provider/strong"]["tier"], "capable"); - Ok(()) -} - -#[tokio::test] -async fn latency_service_profile_uses_health_selection_and_retries_failed_target() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![ - StubResponse::latency_health(), - StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})), - StubResponse::ok(), - ])? - else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("latency-profile")), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - json_body(response).await?["choices"][0]["message"]["content"], - "stub-ok" - ); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 3); - assert_eq!(seen[0]["method"], "GET"); - assert!(seen[0]["path"] - .as_str() - .unwrap_or_default() - .starts_with("/v1/endpoints/health?")); - assert!(seen[0]["path"] - .as_str() - .unwrap_or_default() - .contains("endpoint_ids=fast")); - assert!(seen[0]["path"] - .as_str() - .unwrap_or_default() - .contains("endpoint_ids=slow")); - assert_eq!(seen[1]["path"], "/fast/v1/chat/completions"); - assert_eq!(seen[1]["body"]["model"], "provider/fast"); - assert_eq!(seen[2]["path"], "/slow/v1/chat/completions"); - assert_eq!(seen[2]["body"]["model"], "provider/slow"); - Ok(()) -} - -#[tokio::test] -async fn latency_service_profile_propagates_last_upstream_error_after_retries() -> TestResult { - let _stats_guard = STATS_TEST_LOCK.lock().await; - reset_stats()?; - let Some(stub) = HttpStub::start(vec![ - StubResponse::latency_health(), - StubResponse::error(503, json!({"error": {"message": "fast unavailable"}})), - StubResponse::error(502, json!({"error": {"message": "slow unavailable"}})), - ])? - else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&latency_profile_yaml(&stub.origin, 1))?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("latency-profile")), - )?) - .await?; - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - let body = json_body(response).await?; - assert_eq!(body["error"]["type"], "upstream_error"); - assert!(body["error"]["message"] - .as_str() - .unwrap_or_default() - .contains("slow unavailable")); - - let seen = stub.requests()?; - assert_eq!(seen.len(), 3); - assert_eq!(seen[1]["body"]["model"], "provider/fast"); - assert_eq!(seen[2]["body"]["model"], "provider/slow"); - Ok(()) -} - -#[tokio::test] -async fn profile_config_negative_cases_fail_before_legacy_deletion() -> TestResult { - let unknown_profile_type = state_from_yaml( - r#" -profiles: - bad: - type: not-a-profile -"#, - ) - .err() - .ok_or("unknown profile type should fail")? - .to_string(); - assert!(unknown_profile_type.contains("unknown profile type")); - assert!(unknown_profile_type.contains("not-a-profile")); - - let unknown_target = state_from_yaml( - r#" -targets: {} -profiles: - bad: - type: passthrough - target: missing -"#, - ) - .err() - .ok_or("unknown target should fail")? - .to_string(); - assert!(unknown_target.contains("unknown target missing")); - - let invalid_config = state_from_yaml( - r#" -targets: - strong: - model: provider/strong - format: openai - weak: - model: provider/weak - format: openai -profiles: - random-profile: - type: random-routing - strong: strong - weak: weak - strong_probability: 2.0 -"#, - ) - .err() - .ok_or("invalid random-routing probability should fail")? - .to_string(); - assert!(invalid_config.contains("strong_probability")); - Ok(()) -} - -#[tokio::test] -async fn unknown_inbound_model_stays_a_client_error() -> TestResult { - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(chat_body("missing")), - )?) - .await?; - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!( - json_body(response).await?["error"]["code"], - "model_not_found" - ); - Ok(()) -} - -#[derive(Clone)] -struct StubResponse { - status: u16, - body: Value, -} - -impl StubResponse { - fn ok() -> Self { - Self { - status: 200, - body: json!({ - "id": "chatcmpl-stub", - "object": "chat.completion", - "model": "stub", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "stub-ok"}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 1, - "completion_tokens": 1, - "total_tokens": 2, - }, - }), - } - } - - fn latency_health() -> Self { - Self { - status: 200, - body: json!({ - "endpoint_health": { - "fast": {"status": "healthy", "last_latency_ms": 10.0}, - "slow": {"status": "degraded", "last_latency_ms": 1.0}, - }, - }), - } - } - - fn error(status: u16, body: Value) -> Self { - Self { status, body } - } - - fn classifier(tier: &str) -> Self { - Self { - status: 200, - body: json!({ - "id": "chatcmpl-classifier", - "object": "chat.completion", - "model": "classifier/model", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": json!({"tier": tier}).to_string()}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 3, - "completion_tokens": 1, - "total_tokens": 4, - }, - }), - } - } - - fn context_overflow() -> Self { - Self { - status: 400, - body: json!({ - "error": { - "code": "context_length_exceeded", - "message": "maximum context length exceeded", - }, - }), - } - } -} - -/// In-process HTTP stub that serves one response per accepted request. -struct HttpStub { - /// Server origin used by latency-service health polling. - origin: String, - /// OpenAI-compatible base URL ending in `/v1`. - base_url: String, - /// Bound address used to wake blocked accepts during drop. - addr: SocketAddr, - /// Number of requests the stub thread expects before exiting. - expected_requests: usize, - /// Captured method, path, and JSON body for each real request. - requests: Arc>>, - /// Background accept loop joined on drop after wake-up connects. - handle: Option>, -} - -impl HttpStub { - /// Binds an ephemeral port and returns `None` when loopback is sandbox-denied. - fn start(responses: Vec) -> TestResult> { - let listener = match StdTcpListener::bind("127.0.0.1:0") { - Ok(listener) => listener, - Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None), - Err(error) => return Err(error.into()), - }; - let addr = listener.local_addr()?; - let expected_requests = responses.len(); - let requests = Arc::new(Mutex::new(Vec::new())); - let thread_requests = Arc::clone(&requests); - let handle = thread::spawn(move || { - for response in responses { - let Ok((mut stream, _addr)) = listener.accept() else { - return; - }; - if let Ok((method, path, body)) = read_http_request(&mut stream) { - if let Ok(mut requests) = thread_requests.lock() { - requests.push(json!({"method": method, "path": path, "body": body})); - } - } - let body = response.body.to_string(); - let reason = if response.status == 200 { - "OK" - } else { - "ERROR" - }; - let _ = write!( - stream, - "HTTP/1.1 {} {}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response.status, - reason, - body.len(), - body - ); - } - }); - - Ok(Some(Self { - origin: format!("http://{addr}"), - base_url: format!("http://{addr}/v1"), - addr, - expected_requests, - requests, - handle: Some(handle), - })) - } - - fn requests(&self) -> TestResult> { - self.requests - .lock() - .map(|requests| requests.clone()) - .map_err(|_| "stub request mutex poisoned".into()) - } -} - -impl Drop for HttpStub { - fn drop(&mut self) { - // Wake pending accepts before joining the stub thread. - for _ in 0..self.expected_requests { - let _ = TcpStream::connect(self.addr); - } - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - } -} - -/// Reads one small HTTP request, tolerating partial reads until headers and -/// the declared `content-length` body are available. -fn read_http_request(stream: &mut TcpStream) -> TestResult<(String, String, Value)> { - let mut buffer = Vec::new(); - let mut header_end = None; - while header_end.is_none() { - let mut chunk = [0; 1024]; - let read = stream.read(&mut chunk)?; - if read == 0 { - break; - } - buffer.extend_from_slice(&chunk[..read]); - header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4); - } - - let Some(body_start) = header_end else { - return Err("HTTP request headers were incomplete".into()); - }; - let headers = std::str::from_utf8(&buffer[..body_start])?; - let mut request_line = headers - .lines() - .next() - .ok_or("HTTP request line was missing")? - .split_whitespace(); - let method = request_line - .next() - .ok_or("HTTP request line was missing a method")? - .to_string(); - let path = request_line - .next() - .ok_or("HTTP request line was missing a path")? - .to_string(); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - - while buffer.len() < body_start + content_length { - let mut chunk = [0; 1024]; - let read = stream.read(&mut chunk)?; - if read == 0 { - break; - } - buffer.extend_from_slice(&chunk[..read]); - } - - if buffer.len() < body_start + content_length { - return Err("HTTP request body was incomplete".into()); - } - let body = if content_length == 0 { - Value::Null - } else { - serde_json::from_slice(&buffer[body_start..body_start + content_length])? - }; - Ok((method, path, body)) -} - -fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn state_from_yaml(input: &str) -> TestResult { - let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?; - Ok(ServerState::from_plan(&plan)?) -} - -fn chat_body(model: &str) -> Value { - json!({"model": model, "messages": [{"role": "user", "content": "hi"}]}) -} - -fn latency_profile_yaml(origin: &str, max_retries: usize) -> String { - format!( - r#" -targets: - fast: - model: provider/fast - format: openai - base_url: {origin}/fast/v1 - slow: - model: provider/slow - format: openai - base_url: {origin}/slow/v1 -profiles: - latency-profile: - type: latency-service - latency_service_url: {origin} - targets: [fast, slow] - max_retries: {max_retries} -"# - ) -} - -fn request(method: &str, uri: &str, body: Option) -> TestResult> { - let builder = Request::builder() - .method(method) - .uri(uri) - .header("content-type", "application/json"); - let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string())); - Ok(builder.body(body)?) -} - -async fn json_body(response: axum::response::Response) -> TestResult { - let bytes = response.into_body().collect().await?.to_bytes(); - Ok(serde_json::from_slice(&bytes)?) -} - -fn reset_stats() -> switchyard_core::Result<()> { - profile_stats_accumulator().reset() -} - -fn log_loopback_bind_skip() { - eprintln!("SKIP: permission denied binding loopback socket"); -} - -type TestResult = std::result::Result>; diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 90ee30b1..e4d5886f 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1,841 +1,349 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Integration tests for the components-v2 Rust profile server. - -use std::io::ErrorKind; -use std::io::{Read, Write}; -use std::net::{SocketAddr, TcpListener as StdTcpListener, TcpStream}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::thread; - -use async_trait::async_trait; -use axum::body::Body; -use axum::http::{Request, StatusCode}; +//! Integration tests for the libsy Rust server. + +use std::collections::{BTreeMap, HashSet}; +use std::convert::Infallible; +use std::error::Error; +use std::sync::Arc; + +use axum::body::{Body, Bytes}; +use axum::extract::State; +use axum::http::{Request as HttpRequest, StatusCode}; +use axum::response::sse::{Event, Sse}; +use axum::response::{IntoResponse, Response as HttpResponse}; +use axum::routing::post; +use axum::{Json, Router}; use http_body_util::BodyExt; +use libsy::{Algorithm, LlmTarget, LlmTargetSet, RandomAlgo, RoutedLlmClient}; use serde_json::{json, Value}; -use switchyard_components_v2::{ - parse_profile_config_str, profile_stats_accumulator, Profile, ProfileConfigFormat, - ProfileInput, ProfileResponse, RoutingMetadata, -}; -use switchyard_core::{ - ChatRequestType, ChatResponse, ModelId, Result, StreamEvent, SwitchyardError, -}; -use switchyard_server::{build_switchyard_router, ProfileRegistry, ServerState}; +use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; +use switchyard_server::{build_switchyard_router, ServerState}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; use tower::ServiceExt; -#[tokio::test] -async fn minimal_noop_config_boots_and_serves_core_routes() -> TestResult { - let _stats_guard = stats_guard().await; - reset_stats()?; - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let health = app - .clone() - .oneshot(request("GET", "/health", None)?) - .await?; - assert_eq!(health.status(), StatusCode::OK); - assert_eq!(json_body(health).await?, json!({"status": "ok"})); +type TestError = Box; +type TestResult = Result; - let models = app - .clone() - .oneshot(request("GET", "/v1/models", None)?) - .await?; - assert_eq!(models.status(), StatusCode::OK); - let models = json_body(models).await?; - assert_eq!(models["object"], "list"); - assert_eq!(models["data"][0]["id"], "bench"); - assert_eq!(models["default_model"], "bench"); - assert_eq!(models["model_pool"], json!(["bench"])); - - let chat = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({ - "model": "bench", - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(chat.status(), StatusCode::OK); - assert_eq!( - json_body(chat).await?["choices"][0]["message"]["content"], - "ok" - ); - Ok(()) -} +const ROUTE_MODEL: &str = "switchyard/random"; -#[tokio::test] -async fn all_request_endpoints_route_through_selected_profile() -> TestResult { - let _stats_guard = stats_guard().await; - reset_stats()?; - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let anthropic = app - .clone() - .oneshot(request( - "POST", - "/v1/messages", - Some(json!({ - "model": "bench", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(anthropic.status(), StatusCode::OK); - let anthropic = json_body(anthropic).await?; - assert_eq!(anthropic["type"], "message"); - assert_eq!(anthropic["content"][0]["text"], "ok"); - - let responses = app - .oneshot(request( - "POST", - "/v1/responses", - Some(json!({"model": "bench", "input": "hi"})), - )?) - .await?; - assert_eq!(responses.status(), StatusCode::OK); - let responses = json_body(responses).await?; - assert_eq!(responses["object"], "response"); - assert_eq!(responses["output"][0]["content"][0]["text"], "ok"); - Ok(()) +struct MockUpstream { + base_url: String, + calls: Arc>>, + task: JoinHandle<()>, } -#[tokio::test] -async fn missing_and_unknown_models_return_client_errors() -> TestResult { - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let missing = app - .clone() - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({"messages": [{"role": "user", "content": "hi"}]})), - )?) - .await?; - assert_eq!(missing.status(), StatusCode::BAD_REQUEST); - assert_eq!( - json_body(missing).await?["error"]["type"], - "invalid_request_error" - ); +impl MockUpstream { + async fn start() -> TestResult { + let calls = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route("/v1/chat/completions", post(upstream_chat)) + .with_state(Arc::clone(&calls)); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let task = tokio::spawn(async move { + if let Err(error) = axum::serve(listener, app).await { + tracing::error!(error = %error, "mock upstream stopped"); + } + }); + Ok(Self { + base_url: format!("http://{addr}/v1"), + calls, + task, + }) + } +} - let unknown = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({ - "model": "missing-route", - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(unknown.status(), StatusCode::NOT_FOUND); - assert_eq!( - json_body(unknown).await?["error"]["type"], - "model_not_found" - ); - Ok(()) +impl Drop for MockUpstream { + fn drop(&mut self) { + self.task.abort(); + } } -#[tokio::test] -async fn malformed_and_non_object_json_return_shared_client_errors() -> TestResult { - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - for uri in ["/v1/chat/completions", "/v1/messages", "/v1/responses"] { - let malformed = app.clone().oneshot(raw_request("POST", uri, "{")?).await?; - assert_eq!(malformed.status(), StatusCode::BAD_REQUEST); - assert_eq!(json_body(malformed).await?["error"]["code"], "invalid_body"); - - let non_object = app.clone().oneshot(raw_request("POST", uri, "[]")?).await?; - assert_eq!(non_object.status(), StatusCode::BAD_REQUEST); - assert_eq!( - json_body(non_object).await?["error"]["code"], - "invalid_body" +async fn upstream_chat( + State(calls): State>>>, + Json(body): Json, +) -> HttpResponse { + calls.lock().await.push(body.clone()); + if body["messages"][0]["content"] == "fail" { + return ( + StatusCode::IM_A_TEAPOT, + Json(json!({"error": {"message": "upstream rejected request"}})), + ) + .into_response(); + } + + let model = body["model"].as_str().unwrap_or("unknown").to_string(); + if body["stream"].as_bool() == Some(true) { + let events = [ + json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(), + json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(), + json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(), + "[DONE]".to_string(), + ]; + let stream = futures_util::stream::iter( + events + .into_iter() + .map(|data| Ok::(Event::default().data(data))), ); + return Sse::new(stream).into_response(); } - Ok(()) + + Json(json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + })) + .into_response() +} + +fn random_state(base_url: &str, targets: &[&str]) -> TestResult { + let backend = Backend::OpenAiChat(HttpBackendConfig { + base_url: base_url.to_string(), + api_key: Some("test-key".to_string()), + extra_headers: BTreeMap::new(), + }); + let model_configs = targets + .iter() + .map(|model| ModelConfig::new(*model, backend.clone(), None)) + .collect::>(); + let client: Arc = Arc::new(TranslatingLlmClient::new(&model_configs)?); + let target_set = LlmTargetSet::new( + targets + .iter() + .map(|model| LlmTarget { + semantic_name: (*model).to_string(), + llm_client: Some(Arc::clone(&client)), + }) + .collect(), + ); + let algorithm: Arc = Arc::new(RandomAlgo::new(target_set)); + Ok(ServerState::new( + ROUTE_MODEL, + "uniform random routing", + algorithm, + )?) +} + +async fn send(app: &Router, method: &str, path: &str, body: Option) -> TestResult { + let mut builder = HttpRequest::builder().method(method).uri(path); + let request_body = if let Some(body) = body { + builder = builder.header("content-type", "application/json"); + Body::from(serde_json::to_vec(&body)?) + } else { + Body::empty() + }; + let response = app.clone().oneshot(builder.body(request_body)?).await?; + let status = response.status(); + let headers = response.headers().clone(); + let bytes = response.into_body().collect().await?.to_bytes(); + Ok(Response { + status, + headers, + bytes, + }) } -#[tokio::test] -async fn translation_errors_do_not_emit_routing_metadata_headers() -> TestResult { - let app = build_switchyard_router(state_from_profile("bad", Arc::new(BadTranslationProfile))?); +struct Response { + status: StatusCode, + headers: axum::http::HeaderMap, + bytes: Bytes, +} - let response = app - .oneshot(request( - "POST", - "/v1/messages", - Some(json!({ - "model": "bad", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 8, - })), - )?) - .await?; +impl Response { + fn json(&self) -> TestResult { + Ok(serde_json::from_slice(&self.bytes)?) + } - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert!(!response - .headers() - .keys() - .any(|name| name.as_str().starts_with("x-model-router-"))); - Ok(()) + fn text(&self) -> TestResult<&str> { + Ok(std::str::from_utf8(&self.bytes)?) + } } #[tokio::test] -async fn target_with_same_id_and_model_is_registered_once() -> TestResult { - let _stats_guard = stats_guard().await; - let Some(stub) = HttpStub::start(1)? else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - upstream-direct: - model: upstream-direct - format: openai - base_url: {base_url} -profiles: - direct-profile: - type: passthrough - target: upstream-direct -"#, - base_url = stub.base_url - ))?); - - let models = app - .clone() - .oneshot(request("GET", "/v1/models", None)?) - .await?; - assert_eq!( - json_body(models).await?["model_pool"], - json!(["direct-profile", "upstream-direct"]) - ); +async fn health_models_and_unknown_paths_have_stable_shapes() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({ - "model": "upstream-direct", - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(stub.requests()?[0]["model"], "upstream-direct"); - Ok(()) -} + let health = send(&app, "GET", "/health", None).await?; + assert_eq!(health.status, StatusCode::OK); + assert_eq!(health.json()?, json!({"status": "ok"})); -#[tokio::test] -async fn random_routing_profile_reaches_selected_backend_path() -> TestResult { - let _stats_guard = stats_guard().await; - reset_stats()?; - let Some(stub) = HttpStub::start(1)? else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - strong: - model: upstream-strong - format: openai - base_url: {base_url} - weak: - model: upstream-weak - format: openai - base_url: {base_url} -profiles: - random: - type: random-routing - strong: strong - weak: weak - strong_probability: 0.0000004 - rng_seed: 7 -"#, - base_url = stub.base_url - ))?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({ - "model": "random", - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - for (name, expected) in [ - ("x-model-router-selected-model", "upstream-weak"), - ("x-model-router-selected-tier", "weak"), - ("x-model-router-version", "random-routing:v1"), - ("x-model-router-tolerance", "0.0000004"), - ] { - assert_eq!(header(&response, name), Some(expected)); - } - assert!(header(&response, "x-model-router-rationale") - .is_some_and(|value| value.contains("strong_probability 0.0000004; selected weak"))); + let models = send(&app, "GET", "/v1/models", None).await?; + assert_eq!(models.status, StatusCode::OK); + assert_eq!(models.json()?["model_pool"], json!([ROUTE_MODEL])); - let seen = stub.requests()?; - assert_eq!(seen.len(), 1); - assert_eq!(seen[0]["model"], "upstream-weak"); + let missing = send(&app, "GET", "/missing", None).await?; + assert_eq!(missing.status, StatusCode::NOT_FOUND); + assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found"); Ok(()) } #[tokio::test] -async fn latency_service_profile_reaches_configured_backend_path() -> TestResult { - let _stats_guard = stats_guard().await; - reset_stats()?; - let Some(stub) = HttpStub::start(1)? else { - log_loopback_bind_skip(); - return Ok(()); - }; - let app = build_switchyard_router(state_from_yaml(&format!( - r#" -targets: - fast: - model: upstream-fast - format: openai - base_url: {base_url} -profiles: - latency: - type: latency-service - latency_service_url: http://latency.local - targets: [fast] -"#, - base_url = stub.base_url - ))?); - - let response = app - .oneshot(request( - "POST", +async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + + let cases = [ + ( "/v1/chat/completions", - Some(json!({ - "model": "latency", - "messages": [{"role": "user", "content": "hi"}], - })), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); + json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "hi"}] + }), + ), + ( + "/v1/messages", + json!({ + "model": ROUTE_MODEL, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}] + }), + ), + ( + "/v1/responses", + json!({"model": ROUTE_MODEL, "input": "hi"}), + ), + ]; - let seen = stub.requests()?; - assert_eq!(seen.len(), 1); - assert_eq!(seen[0]["model"], "upstream-fast"); - Ok(()) -} + let mut responses = Vec::new(); + for (path, body) in cases { + responses.push(send(&app, "POST", path, Some(body)).await?); + } -#[tokio::test] -async fn stats_endpoints_use_components_v2_global_accumulator() -> TestResult { - let _stats_guard = stats_guard().await; - reset_stats()?; - profile_stats_accumulator().record_success("served-model", Some(12.0), Some("strong"))?; - let app = build_switchyard_router(state_from_yaml( - r#" -profiles: - bench: - type: noop -"#, - )?); - - let stats = app - .clone() - .oneshot(request("GET", "/v1/routing/stats", None)?) - .await?; - assert_eq!(stats.status(), StatusCode::OK); + assert!(responses + .iter() + .all(|response| response.status == StatusCode::OK)); assert_eq!( - json_body(stats).await?["models"]["served-model"]["calls"], - 1 + responses[0].json()?["choices"][0]["message"]["content"], + "ok" ); + assert_eq!(responses[1].json()?["content"][0]["text"], "ok"); + assert_eq!( + responses[2].json()?["output"][0]["content"][0]["text"], + "ok" + ); + for response in &responses { + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/a") + ); + } - let reset = app - .clone() - .oneshot(request("POST", "/v1/routing/stats/reset", None)?) - .await?; - assert_eq!(reset.status(), StatusCode::OK); - assert_eq!(json_body(reset).await?, json!({"status": "reset"})); - - let after = app.oneshot(request("GET", "/v1/stats", None)?).await?; - assert_eq!(json_body(after).await?["models"], json!({})); - Ok(()) -} - -#[tokio::test] -async fn openai_streams_are_sse_framed_with_done() -> TestResult { - let app = build_switchyard_router(state_from_profile( - "stream", - Arc::new(StreamProfile { - kind: StreamKind::OpenAi, - routing_metadata: None, - }), - )?); - - let response = app - .oneshot(request( - "POST", - "/v1/chat/completions", - Some(json!({ - "model": "stream", - "messages": [{"role": "user", "content": "hi"}], - "stream": true, - })), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - let body = text_body(response).await?; - assert!(body.contains("data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}")); - assert!(body.contains("data: [DONE]")); + let calls = upstream.calls.lock().await; + assert_eq!(calls.len(), 3); + assert!(calls.iter().all(|call| call["model"] == "model/a")); Ok(()) } #[tokio::test] -async fn openai_streams_include_routing_metadata_headers() -> TestResult { - let app = build_switchyard_router(state_from_profile( - "stream", - Arc::new(StreamProfile { - kind: StreamKind::OpenAi, - routing_metadata: Some(RoutingMetadata { - selected_model: Some("served-model".to_string()), - selected_tier: Some("weak".to_string()), - confidence: Some(0.0000004), - router_version: Some("test-router:v1".to_string()), - tolerance: Some(0.0000004), - rationale: Some("line\nbreak".to_string()), - }), - }), - )?); - - let response = app - .oneshot(request( +async fn random_routing_only_calls_configured_targets() -> TestResult { + let upstream = MockUpstream::start().await?; + let targets = ["model/a", "model/b", "model/c"]; + let app = build_switchyard_router(random_state(&upstream.base_url, &targets)?); + + for _ in 0..20 { + let response = send( + &app, "POST", "/v1/chat/completions", Some(json!({ - "model": "stream", - "messages": [{"role": "user", "content": "hi"}], - "stream": true, + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "hi"}] })), - )?) + ) .await?; - assert_eq!(response.status(), StatusCode::OK); - for (name, expected) in [ - ("x-model-router-selected-model", "served-model"), - ("x-model-router-confidence", "0.0000004"), - ("x-model-router-rationale", "line break"), - ] { - assert_eq!(header(&response, name), Some(expected)); + assert_eq!(response.status, StatusCode::OK); } - let body = text_body(response).await?; - assert!(body.contains("data: [DONE]")); + + let configured = targets.into_iter().collect::>(); + let calls = upstream.calls.lock().await; + assert!(calls.iter().all(|call| { + call["model"] + .as_str() + .map(|model| configured.contains(model)) + .unwrap_or(false) + })); Ok(()) } #[tokio::test] -async fn anthropic_streams_are_named_sse_without_done() -> TestResult { - let app = build_switchyard_router(state_from_profile( - "stream", - Arc::new(StreamProfile { - kind: StreamKind::Anthropic, - routing_metadata: None, - }), - )?); - - let response = app - .oneshot(request( - "POST", - "/v1/messages", - Some(json!({ - "model": "stream", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - "stream": true, - })), - )?) - .await?; - assert_eq!(response.status(), StatusCode::OK); - let body = text_body(response).await?; - assert!(body.contains("event: message_start")); - assert!(body.contains("\"type\":\"message_start\"")); - assert!(!body.contains("[DONE]")); +async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "hi"}], + "stream": true + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert!(response.text()?.contains("hello")); + assert!(response.text()?.contains("data: [DONE]")); Ok(()) } #[tokio::test] -async fn endpoint_metadata_is_passed_to_profiles() -> TestResult { - let captured = Arc::new(Mutex::new(None)); - let app = build_switchyard_router(state_from_profile( - "capture", - Arc::new(CaptureProfile { - captured: Arc::clone(&captured), - }), - )?); - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("content-type", "application/json") - .header("X-Request-ID", "req-123") - .header("X-Switchyard-Trace", "trace-a") - .body(Body::from( - json!({ - "model": "capture", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hi"}], - }) - .to_string(), - ))?, - ) - .await?; - assert_eq!(response.status(), StatusCode::OK); - - let input = captured - .lock() - .map_err(|_| "captured input mutex poisoned")? - .clone() - .ok_or("profile should have received input")?; - assert_eq!(input.request.model(), Some("capture")); +async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + + let unknown = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "other", + "messages": [{"role": "user", "content": "hi"}] + })), + ) + .await?; + assert_eq!(unknown.status, StatusCode::NOT_FOUND); + assert_eq!(unknown.json()?["error"]["code"], "model_not_found"); + + let missing_model = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({"messages": [{"role": "user", "content": "hi"}]})), + ) + .await?; + assert_eq!(missing_model.status, StatusCode::BAD_REQUEST); assert_eq!( - input.metadata.request_id.as_ref().map(|id| id.as_str()), - Some("req-123") - ); - assert_eq!( - input.metadata.inbound_format, - Some(ChatRequestType::Anthropic) - ); - assert_eq!( - input - .metadata - .headers - .get("x-switchyard-trace") - .map(Vec::as_slice), - Some(&["trace-a".to_string()][..]) + missing_model.json()?["error"]["code"], + "invalid_request_error" ); - Ok(()) -} - -/// Test profile that emits one deterministic stream event for SSE framing checks. -#[derive(Clone)] -struct StreamProfile { - kind: StreamKind, - routing_metadata: Option, -} - -/// Stream format variant emitted by `StreamProfile`. -#[derive(Clone, Copy)] -enum StreamKind { - OpenAi, - Anthropic, -} - -#[async_trait] -impl Profile for StreamProfile { - async fn run(&self, _input: ProfileInput) -> Result { - let response = match self.kind { - StreamKind::OpenAi => ChatResponse::OpenAiStream(Box::pin(futures_util::stream::iter( - [Ok(StreamEvent::Json(json!({ - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "served-model", - "choices": [{ - "index": 0, - "delta": {"content": "hello"}, - "finish_reason": null, - }], - })))], - ))), - StreamKind::Anthropic => ChatResponse::AnthropicStream(Box::pin( - futures_util::stream::iter([Ok(StreamEvent::Json(json!({ - "type": "message_start", - "message": { - "id": "msg-test", - "type": "message", - "role": "assistant", - "content": [], - "model": "claude-test", - "stop_reason": null, - "stop_sequence": null, - "usage": {"input_tokens": 1, "output_tokens": 0}, - }, - })))]), - )), - }; - Ok(match &self.routing_metadata { - Some(metadata) => ProfileResponse::with_routing_metadata(response, metadata.clone()), - None => ProfileResponse::from(response), - }) - } -} - -struct CaptureProfile { - captured: Arc>>, -} - -struct BadTranslationProfile; - -#[async_trait] -impl Profile for BadTranslationProfile { - async fn run(&self, _input: ProfileInput) -> Result { - Ok(ProfileResponse::with_routing_metadata( - ChatResponse::openai_completion(json!("not an OpenAI response object")), - RoutingMetadata { - selected_model: Some("bad-upstream".to_string()), - selected_tier: Some("weak".to_string()), - router_version: Some("test-router:v1".to_string()), - ..RoutingMetadata::default() - }, - )) - } -} -#[async_trait] -impl Profile for CaptureProfile { - async fn run(&self, input: ProfileInput) -> Result { - { - let mut captured = self - .captured - .lock() - .map_err(|_| SwitchyardError::Other("captured input mutex poisoned".to_string()))?; - *captured = Some(input); - } - Ok(ChatResponse::anthropic_completion(json!({ - "id": "msg-capture", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "captured"}], - "model": "capture", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": {"input_tokens": 1, "output_tokens": 1}, - })) - .into()) - } -} - -/// In-process HTTP stub that records JSON request bodies from backend calls. -struct HttpStub { - base_url: String, - addr: SocketAddr, - expected_requests: usize, - requests: Arc>>, - handle: Option>, -} - -impl HttpStub { - /// Binds an ephemeral port and accepts the expected number of stub requests. - fn start(expected_requests: usize) -> TestResult> { - let listener = match StdTcpListener::bind("127.0.0.1:0") { - Ok(listener) => listener, - Err(error) if error.kind() == ErrorKind::PermissionDenied => return Ok(None), - Err(error) => return Err(error.into()), - }; - let addr = listener.local_addr()?; - let requests = Arc::new(Mutex::new(Vec::new())); - let thread_requests = Arc::clone(&requests); - let handle = thread::spawn(move || { - for _ in 0..expected_requests { - let Ok((mut stream, _addr)) = listener.accept() else { - return; - }; - if let Ok(body) = read_http_body(&mut stream) { - if let Ok(value) = serde_json::from_slice::(&body) { - if let Ok(mut requests) = thread_requests.lock() { - requests.push(value); - } - } - } - let response = json!({ - "id": "chatcmpl-stub", - "object": "chat.completion", - "model": "stub", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "stub-ok"}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 1, - "completion_tokens": 1, - "total_tokens": 2, - }, - }) - .to_string(); - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - response.len(), - response - ); - } - }); - - Ok(Some(Self { - base_url: format!("http://{addr}/v1"), - addr, - expected_requests, - requests, - handle: Some(handle), - })) - } - - /// Returns the JSON request bodies captured by the stub thread. - fn requests(&self) -> TestResult> { - self.requests - .lock() - .map(|requests| requests.clone()) - .map_err(|_| "stub request mutex poisoned".into()) - } -} - -impl Drop for HttpStub { - fn drop(&mut self) { - // Wake pending accepts before joining the stub thread. - for _ in 0..self.expected_requests { - let _ = TcpStream::connect(self.addr); - } - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - } -} - -fn read_http_body(stream: &mut std::net::TcpStream) -> TestResult> { - let mut buffer = Vec::new(); - let mut header_end = None; - while header_end.is_none() { - let mut chunk = [0; 1024]; - let read = stream.read(&mut chunk)?; - if read == 0 { - break; - } - buffer.extend_from_slice(&chunk[..read]); - header_end = find_bytes(&buffer, b"\r\n\r\n").map(|index| index + 4); - } - - let Some(body_start) = header_end else { - return Err("HTTP request headers were incomplete".into()); - }; - let headers = std::str::from_utf8(&buffer[..body_start])?; - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - - while buffer.len() < body_start + content_length { - let mut chunk = [0; 1024]; - let read = stream.read(&mut chunk)?; - if read == 0 { - break; - } - buffer.extend_from_slice(&chunk[..read]); - } - - Ok(buffer[body_start..body_start + content_length].to_vec()) -} - -fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn state_from_yaml(input: &str) -> TestResult { - let plan = parse_profile_config_str(input, ProfileConfigFormat::Yaml)?.resolve()?; - Ok(ServerState::from_plan(&plan)?) -} - -fn state_from_profile(model: &'static str, profile: Arc) -> TestResult { - let registry = ProfileRegistry::from_profiles([( - ModelId::from_static(model), - profile, - model.to_string(), - )])?; - Ok(ServerState::new(registry)) -} - -fn request(method: &str, uri: &str, body: Option) -> TestResult> { - let builder = Request::builder() - .method(method) - .uri(uri) - .header("content-type", "application/json"); - let body = body.map_or_else(Body::empty, |body| Body::from(body.to_string())); - Ok(builder.body(body)?) -} - -fn raw_request(method: &str, uri: &str, body: &str) -> TestResult> { - let builder = Request::builder() - .method(method) - .uri(uri) - .header("content-type", "application/json"); - Ok(builder.body(Body::from(body.to_string()))?) -} - -async fn json_body(response: axum::response::Response) -> TestResult { - let bytes = response.into_body().collect().await?.to_bytes(); - Ok(serde_json::from_slice(&bytes)?) -} - -async fn text_body(response: axum::response::Response) -> TestResult { - let bytes = response.into_body().collect().await?.to_bytes(); - Ok(String::from_utf8(bytes.to_vec())?) -} - -fn header<'a>(response: &'a axum::response::Response, name: &str) -> Option<&'a str> { - response - .headers() - .get(name) - .and_then(|value| value.to_str().ok()) -} - -fn reset_stats() -> Result<()> { - profile_stats_accumulator().reset() -} - -/// Serializes tests that touch the global profile stats accumulator. -/// -/// The lock is initialized once for the test process and held by each caller -/// until the returned guard is dropped. -async fn stats_guard() -> tokio::sync::MutexGuard<'static, ()> { - static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| tokio::sync::Mutex::new(())) - .lock() - .await -} - -fn log_loopback_bind_skip() { - eprintln!("SKIP: permission denied binding loopback socket"); + let upstream_error = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": ROUTE_MODEL, + "messages": [{"role": "user", "content": "fail"}] + })), + ) + .await?; + assert_eq!(upstream_error.status, StatusCode::IM_A_TEAPOT); + assert_eq!(upstream_error.json()?["error"]["code"], "upstream_error"); + Ok(()) } - -type TestResult = std::result::Result>; diff --git a/docs/cli_reference.md b/docs/cli_reference.md index c9b47cda..4c3b98f4 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -21,7 +21,7 @@ These apply to the top-level `switchyard` command, before any verb. |---|---| | `--version` | Print the installed Switchyard version (`switchyard X.Y.Z`) and exit. Reads the version from the installed package metadata. | | `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. | -| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config` (the Rust profile server has no Python processor chain). | +| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config`. | | `--rl-log-dir DIR` | Output directory for `--enable-rl-logging` traces (default: `./rl_data`). No effect without `--enable-rl-logging`. | ## Cross-cutting flag families @@ -147,8 +147,7 @@ matches the pre-1.0 trace format: ``` Turns without an assistant choice (e.g. upstream errors) are skipped. The flag -works on `launch` and on `serve` with a route bundle; `serve --config` (the Rust -profile server) rejects it, since it has no Python processor chain to attach to. +works on `launch` and on `serve` with a route bundle; `serve --config` rejects it. ### Transport (server verbs) @@ -205,7 +204,7 @@ because the built-in default is `https://openrouter.ai/api/v1`. ## `switchyard serve` -Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Files whose profiles are all Rust-defined use the Rust profile server. Files that include Python-defined profiles use the Python FastAPI adapter, while keeping the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id. +Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Rust-defined and Python-defined profiles run through the same Python HTTP application and expose the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id. The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropic Messages (`/v1/messages`), and OpenAI Responses (`/v1/responses`) APIs on the same host and port. diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index 859c3a9f..3dcdc18f 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -16,7 +16,7 @@ a drop-in scrape config and starter alert rules. !!! note "Availability: legacy route-bundle serve only" `GET /metrics` is served by the **legacy route-bundle path** - (`switchyard --routing-profiles PATH serve`). The recommended v2 profile server + (`switchyard --routing-profiles PATH serve`). The v2 profile-config path (`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats` (or its alias `GET /v1/routing/stats`) on that path instead. diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py index cac0ff3d..a7fb6ebc 100644 --- a/switchyard/cli/switchyard_cli.py +++ b/switchyard/cli/switchyard_cli.py @@ -99,7 +99,6 @@ add_transport_args, build_and_serve, load_secrets, - resolve_port, resolve_rl_log_dir, ) @@ -144,17 +143,6 @@ def _warn_deprecated_saved_route_bundle() -> None: ) -def _warn_deprecated_python_profile_server(python_profiles: Sequence[str]) -> None: - _print_deprecation_warning( - "Python-defined profile serving", - details=( - "This config is using the Python FastAPI adapter.", - f"Python profile(s): {', '.join(python_profiles)}.", - "Prefer Rust-defined components-v2 profiles for new serve configs.", - ), - ) - - class _IntakeEnabledAction(argparse.Action): """Store the normalized Intake enable flag and warn on the deprecated alias.""" @@ -468,7 +456,7 @@ def _cmd_serve(args: argparse.Namespace) -> None: def _cmd_serve_profile_config(args: argparse.Namespace) -> None: - """Serve a components-v2 profile config.""" + """Serve a v2 profile config through the Python HTTP application.""" if args.routing_profiles: raise SystemExit( "serve --config cannot be combined with --routing-profiles; " @@ -494,47 +482,14 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None: raise SystemExit("serve --config does not support Intake options yet.") if getattr(args, "enable_rl_logging", False): raise SystemExit( - "serve --config does not support --enable-rl-logging: the Rust " - "profile server has no Python processor chain to attach the trace " - "logger to. Use serve --routing-profiles for local RL trace logging." + "serve --config does not support --enable-rl-logging. " + "Use serve --routing-profiles for local RL trace logging." ) - # Inspect first so files containing only Rust-defined profiles can use the - # Rust server path, while files with Python-defined profiles use FastAPI. - from switchyard.lib.profiles.loader import python_profile_ids - - try: - python_profiles = python_profile_ids(args.config) - except Exception as exc: - raise SystemExit( - "serve --config: failed to inspect profile config for " - f"Python-defined profiles: {exc}" - ) from exc - if python_profiles: - _cmd_serve_mixed_profile_config(args, python_profiles) - return - - from switchyard_rust.server import run_profile_server - - port = args.port if isinstance(args.port, int) else resolve_port() - logger.info( - "Switchyard components-v2 profile config loaded from %s", - args.config, - ) - run_profile_server(args.config, args.host, port) - - -def _cmd_serve_mixed_profile_config( - args: argparse.Namespace, - python_profiles: list[str], -) -> None: - """Serve a config containing Python-defined profiles through FastAPI.""" - _warn_deprecated_python_profile_server(python_profiles) table = _profile_config_route_table(args.config) logger.info( - "Switchyard profile config loaded from %s with Python-defined profile(s): %s", + "Switchyard profile config loaded from %s", args.config, - ", ".join(python_profiles), ) build_and_serve(args, table, inbound_default="both") @@ -928,10 +883,7 @@ def _build_parser() -> argparse.ArgumentParser: default=None, metavar="PATH", help=( - "Path to a Switchyard v2 profile config (YAML, JSON, or TOML). " - "Files containing only Rust-defined profiles use the Rust profile " - "server; files with Python-defined profiles use the Python FastAPI " - "adapter." + "Path to a Switchyard v2 profile config (YAML, JSON, or TOML)." ), ) serve.add_argument( diff --git a/switchyard/lib/processors/random_routing_request_processor.py b/switchyard/lib/processors/random_routing_request_processor.py index 48e2e8ab..e9320ef8 100644 --- a/switchyard/lib/processors/random_routing_request_processor.py +++ b/switchyard/lib/processors/random_routing_request_processor.py @@ -15,8 +15,6 @@ class RandomRoutingRequestProcessor: """Rewrite requests to the randomly selected strong or weak target. This is a compatibility component for the current Python profile chain. - New Rust serving code uses components-v2 profiles directly instead of - exporting a Rust request-processor object. """ def __init__(self, config: RandomRoutingProcessorConfig) -> None: diff --git a/switchyard/lib/profiles/loader.py b/switchyard/lib/profiles/loader.py index 713615be..ee56925b 100644 --- a/switchyard/lib/profiles/loader.py +++ b/switchyard/lib/profiles/loader.py @@ -29,7 +29,7 @@ parse_profile_config_path, ) -__all__ = ["load_profiles", "load_profiles_and_targets", "python_profile_ids"] +__all__ = ["load_profiles", "load_profiles_and_targets"] _PROFILE_TARGET = "profile_target" @@ -66,17 +66,6 @@ def load_profiles_and_targets( return built, _targets(plan) -def python_profile_ids(path: str | Path) -> list[str]: - """Return the ids of Python-defined profiles declared in ``path``. - - Used by ``serve --config`` to select the Rust server for files containing - only Rust-defined profiles and the Python FastAPI adapter for files that - include Python-defined profiles. - """ - _register_shipped_python_profiles() - return sorted(_python_profiles(parse_profile_config_path(path))) - - class _RustProfileRunner: """Adapts an erased Rust ``Profile`` to the ``run(input)`` runner contract.""" diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py index 3633cfc2..a7859405 100644 --- a/switchyard_rust/core.py +++ b/switchyard_rust/core.py @@ -252,15 +252,6 @@ class _NativeModule(Protocol): SessionCache: type[Any] session_key_from_body: Any - def run_profile_server( - self, - config_path: str, - host: str, - port: int, - backlog: int, - dry_run: bool, - ) -> None: ... - def _ensure_switchyard_version_env() -> None: if os.environ.get("SWITCHYARD_VERSION", "").strip(): diff --git a/switchyard_rust/server.py b/switchyard_rust/server.py deleted file mode 100644 index 7ad6c82c..00000000 --- a/switchyard_rust/server.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rust components-v2 profile server bindings.""" - -from switchyard_rust.core import _load_native - - -def run_profile_server( - config_path: str, - host: str = "127.0.0.1", - port: int = 4000, - backlog: int = 65_535, - dry_run: bool = False, -) -> None: - """Run the Rust components-v2 profile server from an installed package.""" - _load_native().run_profile_server(config_path, host, port, backlog, dry_run) - - -__all__ = ["run_profile_server"] diff --git a/tests/test_profile_migration.py b/tests/test_profile_migration.py index e0a14d94..58446567 100644 --- a/tests/test_profile_migration.py +++ b/tests/test_profile_migration.py @@ -3,13 +3,11 @@ """Migration tests for replacing legacy serving paths with components-v2 profiles.""" -import argparse import errno import json import threading from collections.abc import Iterator from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path from typing import Any import pytest @@ -535,73 +533,3 @@ async def test_profile_runner_uses_same_profile_config( assert response.body["model"] == "provider/direct" assert response.body["mock_path"] == "/v2-direct/v1/chat/completions" assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/direct"] - - -def test_cli_serve_config_delegates_to_rust_profile_server( - mocker: MockerFixture, - tmp_path: Path, -) -> None: - import switchyard.cli.switchyard_cli as cli - import switchyard_rust.server as rust_server - - config_path = tmp_path / "profiles.yaml" - config_path.write_text("profiles:\n bench:\n type: noop\n", encoding="utf-8") - captured: dict[str, object] = {} - - def _fake_run_profile_server( - config_path: str, - host: str = "127.0.0.1", - port: int = 4000, - backlog: int = 65_535, - dry_run: bool = False, - ) -> None: - captured["config_path"] = config_path - captured["host"] = host - captured["port"] = port - captured["backlog"] = backlog - captured["dry_run"] = dry_run - - mocker.patch.object( - rust_server, - "run_profile_server", - side_effect=_fake_run_profile_server, - ) - - def _fail_build_and_serve( - args: argparse.Namespace, - switchyard: object, - inbound_default: str = "openai", - disable_backend_streaming: bool = False, - extra_endpoints: list[object] | None = None, - ) -> None: - _ = ( - args, - switchyard, - inbound_default, - disable_backend_streaming, - extra_endpoints, - ) - pytest.fail("route-bundle server should not run") - - mocker.patch.object(cli, "build_and_serve", side_effect=_fail_build_and_serve) - - args = cli._build_parser().parse_args( - [ - "serve", - "--config", - str(config_path), - "--host", - "127.0.0.1", - "--port", - "4555", - ] - ) - args.func(args) - - assert captured == { - "config_path": str(config_path), - "host": "127.0.0.1", - "port": 4555, - "backlog": 65_535, - "dry_run": False, - } diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 86860c72..cafe39e1 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -421,73 +421,6 @@ def _fake_serve( assert captured["inbound_default"] == "both" -def test_serve_config_delegates_to_rust_profile_server( - mocker: Any, - tmp_path, -) -> None: - import switchyard.cli.switchyard_cli as cli - import switchyard_rust.server as rust_server - - config_path = tmp_path / "profiles.yaml" - config_path.write_text("profiles:\n bench:\n type: noop\n") - captured: dict[str, Any] = {} - - def _fake_run_profile_server( - config_path: str, - host: str = "127.0.0.1", - port: int = 4000, - backlog: int = 65_535, - dry_run: bool = False, - ) -> None: - captured["config_path"] = config_path - captured["host"] = host - captured["port"] = port - captured["backlog"] = backlog - captured["dry_run"] = dry_run - - mocker.patch.object( - rust_server, - "run_profile_server", - side_effect=_fake_run_profile_server, - ) - - def _fail_build_and_serve( - args: argparse.Namespace, - switchyard: object, - *, - inbound_default: str = "openai", - disable_backend_streaming: bool = False, - **_kwargs: object, - ) -> None: - _ = (args, switchyard, inbound_default, disable_backend_streaming) - pytest.fail("route-bundle server should not run") - - mocker.patch.object( - cli, - "build_and_serve", - side_effect=_fail_build_and_serve, - ) - - args = cli._build_parser().parse_args([ - "serve", - "--config", - str(config_path), - "--host", - "127.0.0.1", - "--port", - "4555", - ]) - args.func(args) - - assert captured == { - "config_path": str(config_path), - "host": "127.0.0.1", - "port": 4555, - "backlog": 65_535, - "dry_run": False, - } - - def test_serve_config_and_routing_profiles_are_mutually_exclusive( tmp_path, ) -> None: diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py index 29809fe3..299cb4ef 100644 --- a/tests/test_serve_profile_config.py +++ b/tests/test_serve_profile_config.py @@ -1,13 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for serving a v2 profile config via `serve --config`. - -`dry_run` exercises the full load -> resolve -> build -> registry path of the -Rust profile server without binding a socket. Files with Python-defined profiles -use the Python FastAPI adapter so both profile implementations are routable on -the same paths. -""" +"""Tests for serving a v2 profile config via `serve --config`.""" import argparse import json @@ -20,36 +14,7 @@ from fastapi.testclient import TestClient from switchyard.cli.switchyard_cli import _cmd_serve_profile_config, _profile_config_route_table -from switchyard.lib.profiles.loader import python_profile_ids from switchyard.server.switchyard_app import build_switchyard_app -from switchyard_rust.core import SwitchyardConfigError -from switchyard_rust.server import run_profile_server - -_RUST_CONFIG = """ -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:9/strong/v1 - api_key: test-key - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:9/weak/v1 - api_key: test-key - -profiles: - fast: - type: passthrough - target: weak - smart-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: capable_first - confidence_threshold: 0.7 -""" _PYTHON_CONFIG = """ targets: @@ -155,36 +120,9 @@ def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path: return path -def test_dry_run_validates_rust_profiles_and_direct_targets(tmp_path: Path) -> None: - # passthrough + stage_router + the two targets (directly addressable). - path = _write(tmp_path, _RUST_CONFIG) - # dry_run loads, resolves, and builds the registry, then returns without - # binding a socket. No exception == a servable config. - run_profile_server(str(path), dry_run=True) - - -def test_dry_run_rejects_invalid_config(tmp_path: Path) -> None: - path = _write(tmp_path, "profiles:\n bad:\n type: passthrough\n target: ghost\n") - with pytest.raises(SwitchyardConfigError): - run_profile_server(str(path), dry_run=True) - - -def test_shipped_example_config_is_servable(monkeypatch: pytest.MonkeyPatch) -> None: - # Guards examples/profiles.yaml from rotting; dry_run never connects upstream. - monkeypatch.setenv("OPENROUTER_API_KEY", "dummy-key") - example = Path(__file__).resolve().parents[1] / "examples" / "profiles.yaml" - run_profile_server(str(example), dry_run=True) - - -def test_python_profile_ids_classifies_config(tmp_path: Path) -> None: - assert python_profile_ids(_write(tmp_path, _PYTHON_CONFIG)) == ["smart"] - assert python_profile_ids(_write(tmp_path, _RUST_CONFIG, name="rust.yaml")) == [] - - -def test_serve_config_uses_fastapi_for_python_profiles( +def test_serve_config_uses_fastapi_profile_table( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, - capsys: pytest.CaptureFixture[str], ) -> None: path = _write(tmp_path, _PYTHON_CONFIG) args = _serve_args(path) @@ -211,10 +149,6 @@ def capture_build_and_serve( assert captured["models"] == ["smart", "weak", "provider/weak"] assert captured["inbound_default"] == "both" - stderr = capsys.readouterr().err - assert "warning: Python-defined profile serving is deprecated." in stderr - assert "Python FastAPI adapter" in stderr - assert "Python profile(s): smart." in stderr def test_profile_config_route_table_serves_mixed_profiles( @@ -285,24 +219,6 @@ def test_profile_config_route_table_serves_mixed_profiles( ) -def test_serve_config_fails_closed_when_python_profile_inspection_fails( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - path = _write(tmp_path, _RUST_CONFIG) - - def fail_inspection(_path: str) -> list[str]: - raise RuntimeError("inspection exploded") - - monkeypatch.setattr( - "switchyard.lib.profiles.loader.python_profile_ids", - fail_inspection, - ) - - with pytest.raises(SystemExit, match="failed to inspect.*inspection exploded"): - _cmd_serve_profile_config(_serve_args(path)) - - def _serve_args(path: Path) -> argparse.Namespace: return argparse.Namespace( config=str(path), From 88ee03960199cffbc76abd6249c54cd2c96c14e1 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Mon, 20 Jul 2026 14:51:13 -0700 Subject: [PATCH 2/4] feat(server): serve multiple algorithms Signed-off-by: nachiketb --- Cargo.lock | 2 +- crates/switchyard-server/Cargo.toml | 2 +- crates/switchyard-server/src/cli.rs | 155 +++++++++++++++-------- crates/switchyard-server/src/lib.rs | 137 ++++++++++++-------- crates/switchyard-server/tests/server.rs | 122 ++++++++++++++---- 5 files changed, 286 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5a4d4e2..f45b55c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1763,9 +1763,9 @@ dependencies = [ "clap", "futures-util", "http-body-util", - "libsy", "rustls", "serde_json", + "switchyard-libsy", "switchyard-llm-client", "switchyard-translation", "tokio", diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml index 34e4c302..88e88ac0 100644 --- a/crates/switchyard-server/Cargo.toml +++ b/crates/switchyard-server/Cargo.toml @@ -18,7 +18,7 @@ axum-server = { version = "0.8", features = ["tls-rustls"] } rustls = { version = "0.23" } clap = { version = "4", features = ["derive", "env"] } futures-util = "0.3" -libsy = { path = "../libsy" } +libsy = { package = "switchyard-libsy", path = "../libsy" } switchyard-llm-client = { path = "../libsy-llm-client" } serde_json = "1" switchyard-translation = { path = "../switchyard-translation" } diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 17760208..41dbf1d2 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -6,19 +6,55 @@ use std::collections::{BTreeMap, BTreeSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use clap::{Parser, ValueEnum}; -use libsy::{Algorithm, LlmTarget, LlmTargetSet, RandomAlgo, RoutedLlmClient}; +use libsy::algorithms::Random; +use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient}; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; use switchyard_server::{ - run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions, + run_server, ServedModel, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions, DEFAULT_LISTEN_BACKLOG, }; const DEFAULT_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED); const DEFAULT_PORT: u16 = 4000; -const DEFAULT_ROUTE_MODEL: &str = "switchyard/random"; + +#[derive(Clone, Debug)] +struct RandomRouteSpec { + model: String, + targets: Vec, +} + +impl FromStr for RandomRouteSpec { + type Err = String; + + fn from_str(value: &str) -> Result { + let (model, target_list) = value + .split_once('=') + .ok_or_else(|| "route must use MODEL=TARGET[,TARGET...]".to_string())?; + let model = model.trim(); + if model.is_empty() { + return Err("route model must not be empty".to_string()); + } + let targets = target_list + .split(',') + .map(str::trim) + .map(str::to_string) + .collect::>(); + if targets.is_empty() || targets.iter().any(String::is_empty) { + return Err(format!("route {model} must contain non-empty targets")); + } + if targets.iter().collect::>().len() != targets.len() { + return Err(format!("route {model} must contain unique targets")); + } + Ok(Self { + model: model.to_string(), + targets, + }) + } +} #[derive(Clone, Copy, Debug, ValueEnum)] enum UpstreamFormat { @@ -48,13 +84,13 @@ impl UpstreamFormat { version )] pub(crate) struct ServerArgs { - /// Public model id clients send to this server. - #[arg(long, default_value = DEFAULT_ROUTE_MODEL)] - route_model: String, - - /// Upstream model id eligible for random selection. Repeat for each target. - #[arg(long = "target", required = true)] - targets: Vec, + /// Random route as MODEL=TARGET[,TARGET...]. Repeat to serve multiple routes. + #[arg( + long = "route", + required = true, + value_name = "MODEL=TARGET[,TARGET...]" + )] + routes: Vec, /// Base URL shared by the configured upstream targets. #[arg(long, env = "SWITCHYARD_UPSTREAM_BASE_URL")] @@ -100,7 +136,6 @@ impl ServerArgs { } fn into_runtime(self) -> ServerResult<(ServerState, ServerRunOptions)> { - let targets = validated_targets(self.targets)?; if self.base_url.trim().is_empty() { return Err(ServerError::new("--base-url must not be empty")); } @@ -110,29 +145,38 @@ impl ServerArgs { api_key: self.api_key, extra_headers: BTreeMap::new(), }); - let model_configs = targets + let target_models = self + .routes .iter() - .map(|model| ModelConfig::new(model, backend.clone(), None)) + .flat_map(|route| route.targets.iter()) + .collect::>(); + let model_configs = target_models + .into_iter() + .map(|model| ModelConfig::new(model.as_str(), backend.clone(), None)) .collect::>(); let client: Arc = Arc::new( TranslatingLlmClient::new(&model_configs) .map_err(|error| ServerError::new(error.to_string()))?, ); - let target_set = LlmTargetSet::new( - targets - .iter() - .map(|model| LlmTarget { - semantic_name: model.clone(), - llm_client: Some(Arc::clone(&client)), - }) - .collect(), - ); - let algorithm: Arc = Arc::new(RandomAlgo::new(target_set)); - let state = ServerState::new( - self.route_model, - format!("uniform random routing across {}", targets.join(", ")), - algorithm, - )?; + let routes = self.routes.into_iter().map(|route| { + let target_set = LlmTargetSet::new( + route + .targets + .iter() + .map(|model| LlmTarget { + semantic_name: model.clone(), + llm_client: Some(Arc::clone(&client)), + }) + .collect(), + ); + let algorithm: Arc = Arc::new(Random::new(target_set)); + let model = ServedModel { + id: route.model, + display_name: format!("uniform random routing across {}", route.targets.join(", ")), + }; + (model, algorithm) + }); + let state = ServerState::new(routes)?; let tls = match (self.tls_cert, self.tls_key) { (Some(cert), Some(key)) => { @@ -157,17 +201,6 @@ impl ServerArgs { } } -fn validated_targets(targets: Vec) -> ServerResult> { - if targets.iter().any(|target| target.trim().is_empty()) { - return Err(ServerError::new("--target values must not be empty")); - } - let unique = targets.iter().collect::>(); - if unique.len() != targets.len() { - return Err(ServerError::new("--target values must be unique")); - } - Ok(targets) -} - /// Builds the random algorithm and starts the server. pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> { let (state, options) = args.into_runtime()?; @@ -179,13 +212,13 @@ mod tests { use super::*; #[test] - fn repeated_targets_build_random_runtime() -> ServerResult<()> { + fn repeated_routes_build_independent_random_algorithms() -> ServerResult<()> { let args = ServerArgs::try_parse_from([ "switchyard-server", - "--target", - "model/a", - "--target", - "model/b", + "--route", + "switchyard/general=model/a,model/b", + "--route", + "switchyard/coding=model/c,model/d", "--base-url", "http://127.0.0.1:9/v1", "--dry-run", @@ -194,23 +227,39 @@ mod tests { let (state, options) = args.into_runtime()?; - assert_eq!(state.served_model().id, DEFAULT_ROUTE_MODEL); - assert!(state - .served_model() - .display_name - .contains("model/a, model/b")); + assert_eq!( + state + .served_models() + .map(|model| model.id.as_str()) + .collect::>(), + ["switchyard/coding", "switchyard/general"] + ); assert!(options.dry_run); Ok(()) } #[test] fn duplicate_targets_are_rejected() -> ServerResult<()> { + let result = ServerArgs::try_parse_from([ + "switchyard-server", + "--route", + "switchyard/general=model/a,model/a", + "--base-url", + "http://127.0.0.1:9/v1", + ]); + + assert!(result.is_err()); + Ok(()) + } + + #[test] + fn duplicate_route_models_are_rejected() -> ServerResult<()> { let args = ServerArgs::try_parse_from([ "switchyard-server", - "--target", - "model/a", - "--target", - "model/a", + "--route", + "switchyard/general=model/a", + "--route", + "switchyard/general=model/b", "--base-url", "http://127.0.0.1:9/v1", ]) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 54527bc1..bce23e31 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -6,6 +6,7 @@ mod response; mod sse; +use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::error::Error; use std::fmt::{Display, Formatter}; @@ -76,36 +77,54 @@ pub struct ServedModel { /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { + routes: Arc>, +} + +struct AlgorithmEntry { algorithm: Arc, - served_model: Arc, + model: ServedModel, } impl ServerState { - /// Creates server state for one public route backed by a libsy algorithm. + /// Creates server state from public models and their libsy algorithms. pub fn new( - model: impl Into, - display_name: impl Into, - algorithm: Arc, + routes: impl IntoIterator)>, ) -> ServerResult { - let model = model.into(); - if model.trim().is_empty() { - return Err(ServerError::new("route model must not be empty")); + let mut entries = BTreeMap::new(); + for (model, algorithm) in routes { + if model.id.trim().is_empty() { + return Err(ServerError::new("route model must not be empty")); + } + let id = model.id.clone(); + match entries.entry(id) { + Entry::Vacant(entry) => { + entry.insert(AlgorithmEntry { algorithm, model }); + } + Entry::Occupied(entry) => { + return Err(ServerError::new(format!( + "duplicate route model {}", + entry.key() + ))); + } + } + } + if entries.is_empty() { + return Err(ServerError::new("at least one algorithm route is required")); } Ok(Self { - algorithm, - served_model: Arc::new(ServedModel { - id: model, - display_name: display_name.into(), - }), + routes: Arc::new(entries), }) } - /// Returns the public model served by this algorithm. - pub fn served_model(&self) -> &ServedModel { - self.served_model.as_ref() + /// Returns the public models served by the configured algorithms. + pub fn served_models(&self) -> impl Iterator { + self.routes.values().map(|entry| &entry.model) } - fn validate_model(&self, model: Option<&str>) -> std::result::Result<(), Box> { + fn algorithm_for_model( + &self, + model: Option<&str>, + ) -> std::result::Result, Box> { let Some(model) = model.filter(|model| !model.trim().is_empty()) else { return Err(Box::new(error_response( StatusCode::BAD_REQUEST, @@ -114,15 +133,15 @@ impl ServerState { "invalid_request_error", ))); }; - if model != self.served_model.id { + let Some(entry) = self.routes.get(model) else { return Err(Box::new(error_response( StatusCode::NOT_FOUND, format!("No route registered for model {model}"), "model_not_found", "model_not_found", ))); - } - Ok(()) + }; + Ok(Arc::clone(&entry.algorithm)) } } @@ -309,19 +328,17 @@ async fn handle_llm_request( Err(error) => return invalid_body_error(error.to_string()), }; let requested_model = llm_request.model.clone(); - if let Err(response) = state.validate_model(requested_model.as_deref()) { - return *response; - } + let algorithm = match state.algorithm_for_model(requested_model.as_deref()) { + Ok(algorithm) => algorithm, + Err(response) => return *response, + }; let request = Request { llm_request, raw_request: Some(body), metadata: Some(metadata_from_headers(&headers)), }; - let (trace, response) = match Arc::clone(&state.algorithm) - .run(Context::default(), request) - .await - { + let (trace, response) = match algorithm.run(Context::default(), request).await { Ok(result) => result, Err(error) => return algorithm_error(error), }; @@ -477,7 +494,7 @@ fn error_response( } async fn models(State(state): State) -> Json { - Json(model_list_payload(state.served_model())) + Json(model_list_payload(state.served_models())) } async fn health() -> Json { @@ -493,15 +510,22 @@ async fn not_found() -> Response { ) } -fn model_list_payload(entry: &ServedModel) -> Value { +fn model_list_payload<'a>(entries: impl IntoIterator) -> Value { + let entries = entries.into_iter().collect::>(); + let model_ids = entries + .iter() + .map(|entry| entry.id.clone()) + .collect::>(); + let first_id = model_ids.first().cloned(); + let last_id = model_ids.last().cloned(); json!({ "object": "list", - "data": [model_entry_json(entry)], - "first_id": entry.id, - "last_id": entry.id, + "data": entries.into_iter().map(model_entry_json).collect::>(), + "first_id": first_id, + "last_id": last_id, "has_more": false, - "default_model": entry.id, - "model_pool": [entry.id], + "default_model": first_id, + "model_pool": model_ids, }) } @@ -533,11 +557,6 @@ fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String { let mut output = String::new(); push_line(&mut output, "Switchyard libsy server"); - push_line(&mut output, format!(" model: {}", state.served_model.id)); - push_line( - &mut output, - format!(" algorithm: {}", state.served_model.display_name), - ); push_line(&mut output, format!(" listening: {listen_url}")); if local_url != listen_url { push_line(&mut output, format!(" local: {local_url}")); @@ -550,17 +569,27 @@ fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String { push_line(&mut output, " POST /v1/messages"); push_line(&mut output, " POST /v1/responses"); push_line(&mut output, ""); + push_line(&mut output, " routes:"); + for model in state.served_models() { + push_line( + &mut output, + format!(" - {} ({})", model.id, model.display_name), + ); + } + push_line(&mut output, ""); push_line(&mut output, " try:"); push_line(&mut output, format!(" curl -s {local_url}/health")); - let chat_payload = json!({ - "model": state.served_model.id, - "messages": [{"role": "user", "content": "Say OK"}], - "max_tokens": 256, - }); - push_line( - &mut output, - format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'"), - ); + if let Some(model) = state.served_models().next() { + let chat_payload = json!({ + "model": model.id, + "messages": [{"role": "user", "content": "Say OK"}], + "max_tokens": 256, + }); + push_line( + &mut output, + format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'"), + ); + } if options.is_tls() { push_line( &mut output, @@ -573,10 +602,14 @@ fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String { } fn dry_run_summary(state: &ServerState) -> String { - format!( - "server OK: model={}, algorithm={}\n", - state.served_model.id, state.served_model.display_name - ) + let mut output = String::from("server OK:\n"); + for model in state.served_models() { + push_line( + &mut output, + format!(" - {} ({})", model.id, model.display_name), + ); + } + output } fn push_line(output: &mut String, line: impl AsRef) { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index e4d5886f..c76b941f 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -16,10 +16,11 @@ use axum::response::{IntoResponse, Response as HttpResponse}; use axum::routing::post; use axum::{Json, Router}; use http_body_util::BodyExt; -use libsy::{Algorithm, LlmTarget, LlmTargetSet, RandomAlgo, RoutedLlmClient}; +use libsy::algorithms::Random; +use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient}; use serde_json::{json, Value}; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; -use switchyard_server::{build_switchyard_router, ServerState}; +use switchyard_server::{build_switchyard_router, ServedModel, ServerState}; use tokio::net::TcpListener; use tokio::sync::Mutex; use tokio::task::JoinHandle; @@ -106,32 +107,41 @@ async fn upstream_chat( .into_response() } -fn random_state(base_url: &str, targets: &[&str]) -> TestResult { +fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult { let backend = Backend::OpenAiChat(HttpBackendConfig { base_url: base_url.to_string(), api_key: Some("test-key".to_string()), extra_headers: BTreeMap::new(), }); - let model_configs = targets + let target_models = routes .iter() - .map(|model| ModelConfig::new(*model, backend.clone(), None)) + .flat_map(|(_, targets)| targets.iter().copied()) + .collect::>(); + let model_configs = target_models + .into_iter() + .map(|model| ModelConfig::new(model, backend.clone(), None)) .collect::>(); let client: Arc = Arc::new(TranslatingLlmClient::new(&model_configs)?); - let target_set = LlmTargetSet::new( - targets - .iter() - .map(|model| LlmTarget { - semantic_name: (*model).to_string(), - llm_client: Some(Arc::clone(&client)), - }) - .collect(), - ); - let algorithm: Arc = Arc::new(RandomAlgo::new(target_set)); - Ok(ServerState::new( - ROUTE_MODEL, - "uniform random routing", - algorithm, - )?) + let entries = routes.iter().map(|(route_model, targets)| { + let target_set = LlmTargetSet::new( + targets + .iter() + .map(|model| LlmTarget { + semantic_name: (*model).to_string(), + llm_client: Some(Arc::clone(&client)), + }) + .collect(), + ); + let algorithm: Arc = Arc::new(Random::new(target_set)); + ( + ServedModel { + id: (*route_model).to_string(), + display_name: "uniform random routing".to_string(), + }, + algorithm, + ) + }); + Ok(ServerState::new(entries)?) } async fn send(app: &Router, method: &str, path: &str, body: Option) -> TestResult { @@ -172,7 +182,10 @@ impl Response { #[tokio::test] async fn health_models_and_unknown_paths_have_stable_shapes() -> TestResult { let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[(ROUTE_MODEL, &["model/a"])], + )?); let health = send(&app, "GET", "/health", None).await?; assert_eq!(health.status, StatusCode::OK); @@ -188,10 +201,60 @@ async fn health_models_and_unknown_paths_have_stable_shapes() -> TestResult { Ok(()) } +#[tokio::test] +async fn multiple_algorithms_dispatch_by_route_model() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[ + ("switchyard/coding", &["model/code"]), + ("switchyard/general", &["model/general"]), + ], + )?); + + let models = send(&app, "GET", "/v1/models", None).await?; + assert_eq!( + models.json()?["model_pool"], + json!(["switchyard/coding", "switchyard/general"]) + ); + + for (route_model, target_model) in [ + ("switchyard/general", "model/general"), + ("switchyard/coding", "model/code"), + ] { + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": route_model, + "messages": [{"role": "user", "content": "hi"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some(target_model) + ); + } + + let calls = upstream.calls.lock().await; + assert_eq!(calls[0]["model"], "model/general"); + assert_eq!(calls[1]["model"], "model/code"); + Ok(()) +} + #[tokio::test] async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[(ROUTE_MODEL, &["model/a"])], + )?); let cases = [ ( @@ -252,7 +315,10 @@ async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestRes async fn random_routing_only_calls_configured_targets() -> TestResult { let upstream = MockUpstream::start().await?; let targets = ["model/a", "model/b", "model/c"]; - let app = build_switchyard_router(random_state(&upstream.base_url, &targets)?); + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[(ROUTE_MODEL, &targets)], + )?); for _ in 0..20 { let response = send( @@ -282,7 +348,10 @@ async fn random_routing_only_calls_configured_targets() -> TestResult { #[tokio::test] async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[(ROUTE_MODEL, &["model/a"])], + )?); let response = send( &app, @@ -305,7 +374,10 @@ async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { #[tokio::test] async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult { let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state(&upstream.base_url, &["model/a"])?); + let app = build_switchyard_router(random_state( + &upstream.base_url, + &[(ROUTE_MODEL, &["model/a"])], + )?); let unknown = send( &app, From 764f185fe1693ae7bb9fdbc0e0110284168d928d Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 21 Jul 2026 14:25:36 -0700 Subject: [PATCH 3/4] refactor(server): simplify libsy host Signed-off-by: nachiketb --- crates/switchyard-server/README.md | 15 ++ crates/switchyard-server/src/cli.rs | 77 +++------ crates/switchyard-server/src/lib.rs | 204 ++++++----------------- crates/switchyard-server/src/response.rs | 31 ++-- crates/switchyard-server/tests/server.rs | 104 +++--------- 5 files changed, 127 insertions(+), 304 deletions(-) create mode 100644 crates/switchyard-server/README.md diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md new file mode 100644 index 00000000..4bed0fd0 --- /dev/null +++ b/crates/switchyard-server/README.md @@ -0,0 +1,15 @@ +# switchyard-server + +`switchyard-server` exposes libsy algorithms through OpenAI Chat Completions, OpenAI Responses, +and Anthropic Messages endpoints. The current CLI builds uniform-random routes over one shared +upstream backend. + +```bash +cargo run -p switchyard-server -- \ + --route switchyard/general=model/a,model/b \ + --route switchyard/coding=model/c,model/d \ + --base-url https://example.com/v1 \ + --api-key "$API_KEY" +``` + +Select a route by sending its `switchyard/*` ID as the request's `model`. diff --git a/crates/switchyard-server/src/cli.rs b/crates/switchyard-server/src/cli.rs index 41dbf1d2..63e88345 100644 --- a/crates/switchyard-server/src/cli.rs +++ b/crates/switchyard-server/src/cli.rs @@ -14,7 +14,7 @@ use libsy::algorithms::Random; use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient}; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; use switchyard_server::{ - run_server, ServedModel, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions, + run_server, ServerError, ServerResult, ServerRunOptions, ServerState, TlsOptions, DEFAULT_LISTEN_BACKLOG, }; @@ -43,7 +43,7 @@ impl FromStr for RandomRouteSpec { .map(str::trim) .map(str::to_string) .collect::>(); - if targets.is_empty() || targets.iter().any(String::is_empty) { + if targets.iter().any(String::is_empty) { return Err(format!("route {model} must contain non-empty targets")); } if targets.iter().collect::>().len() != targets.len() { @@ -170,11 +170,7 @@ impl ServerArgs { .collect(), ); let algorithm: Arc = Arc::new(Random::new(target_set)); - let model = ServedModel { - id: route.model, - display_name: format!("uniform random routing across {}", route.targets.join(", ")), - }; - (model, algorithm) + (route.model, algorithm) }); let state = ServerState::new(routes)?; @@ -211,61 +207,36 @@ pub(crate) async fn run(args: ServerArgs) -> ServerResult<()> { mod tests { use super::*; + fn runtime(routes: &[&str]) -> ServerResult { + let mut args = vec!["switchyard-server", "--base-url", "http://127.0.0.1:9/v1"]; + for route in routes { + args.extend(["--route", route]); + } + let args = ServerArgs::try_parse_from(args) + .map_err(|error| ServerError::new(error.to_string()))?; + args.into_runtime().map(|(state, _)| state) + } + #[test] - fn repeated_routes_build_independent_random_algorithms() -> ServerResult<()> { - let args = ServerArgs::try_parse_from([ - "switchyard-server", - "--route", + fn routes_build_independent_algorithms() -> ServerResult<()> { + let state = runtime(&[ "switchyard/general=model/a,model/b", - "--route", "switchyard/coding=model/c,model/d", - "--base-url", - "http://127.0.0.1:9/v1", - "--dry-run", - ]) - .map_err(|error| ServerError::new(error.to_string()))?; - - let (state, options) = args.into_runtime()?; - + ])?; assert_eq!( - state - .served_models() - .map(|model| model.id.as_str()) - .collect::>(), + state.models().collect::>(), ["switchyard/coding", "switchyard/general"] ); - assert!(options.dry_run); Ok(()) } #[test] - fn duplicate_targets_are_rejected() -> ServerResult<()> { - let result = ServerArgs::try_parse_from([ - "switchyard-server", - "--route", - "switchyard/general=model/a,model/a", - "--base-url", - "http://127.0.0.1:9/v1", - ]); - - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn duplicate_route_models_are_rejected() -> ServerResult<()> { - let args = ServerArgs::try_parse_from([ - "switchyard-server", - "--route", - "switchyard/general=model/a", - "--route", - "switchyard/general=model/b", - "--base-url", - "http://127.0.0.1:9/v1", - ]) - .map_err(|error| ServerError::new(error.to_string()))?; - - assert!(args.into_runtime().is_err()); - Ok(()) + fn invalid_routes_are_rejected() { + for routes in [ + &["switchyard/general=model/a,model/a"][..], + &["switchyard/general=model/a", "switchyard/general=model/b"], + ] { + assert!(runtime(routes).is_err()); + } } } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index bce23e31..3fc2c979 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -6,7 +6,6 @@ mod response; mod sse; -use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::error::Error; use std::fmt::{Display, Formatter}; @@ -28,7 +27,7 @@ use tokio::net::{TcpListener, TcpSocket}; use switchyard_translation::{decode_request, WireFormat}; -use crate::response::{translate_response, TranslatedResponse}; +use crate::response::into_http_response; /// Default TCP listen backlog used by the Rust server. pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535; @@ -65,47 +64,25 @@ impl Error for ServerError {} /// Result returned by server setup and lifecycle operations. pub type ServerResult = std::result::Result; -/// Public model entry advertised by `/v1/models`. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ServedModel { - /// Public model or route ID accepted in inbound request bodies. - pub id: String, - /// Human-readable label shown by startup logs and `/v1/models`. - pub display_name: String, -} - /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { - routes: Arc>, -} - -struct AlgorithmEntry { - algorithm: Arc, - model: ServedModel, + routes: Arc>>, } impl ServerState { - /// Creates server state from public models and their libsy algorithms. + /// Creates server state from route model IDs and their libsy algorithms. pub fn new( - routes: impl IntoIterator)>, + routes: impl IntoIterator)>, ) -> ServerResult { let mut entries = BTreeMap::new(); for (model, algorithm) in routes { - if model.id.trim().is_empty() { + let model = model.trim(); + if model.is_empty() { return Err(ServerError::new("route model must not be empty")); } - let id = model.id.clone(); - match entries.entry(id) { - Entry::Vacant(entry) => { - entry.insert(AlgorithmEntry { algorithm, model }); - } - Entry::Occupied(entry) => { - return Err(ServerError::new(format!( - "duplicate route model {}", - entry.key() - ))); - } + if entries.insert(model.to_string(), algorithm).is_some() { + return Err(ServerError::new(format!("duplicate route model {model}"))); } } if entries.is_empty() { @@ -116,32 +93,13 @@ impl ServerState { }) } - /// Returns the public models served by the configured algorithms. - pub fn served_models(&self) -> impl Iterator { - self.routes.values().map(|entry| &entry.model) + /// Returns the route model IDs served by the configured algorithms. + pub fn models(&self) -> impl Iterator { + self.routes.keys().map(String::as_str) } - fn algorithm_for_model( - &self, - model: Option<&str>, - ) -> std::result::Result, Box> { - let Some(model) = model.filter(|model| !model.trim().is_empty()) else { - return Err(Box::new(error_response( - StatusCode::BAD_REQUEST, - "request body must include a non-empty string `model`", - "invalid_request_error", - "invalid_request_error", - ))); - }; - let Some(entry) = self.routes.get(model) else { - return Err(Box::new(error_response( - StatusCode::NOT_FOUND, - format!("No route registered for model {model}"), - "model_not_found", - "model_not_found", - ))); - }; - Ok(Arc::clone(&entry.algorithm)) + fn algorithm_for_model(&self, model: &str) -> Option> { + self.routes.get(model).map(Arc::clone) } } @@ -298,22 +256,18 @@ async fn handle_endpoint( ) -> Response { let body = match llm_json_body(body) { Ok(body) => body, - Err(response) => return *response, + Err(message) => return invalid_body_error(message), }; handle_llm_request(state, headers, body, wire_format).await } fn llm_json_body( body: std::result::Result, JsonRejection>, -) -> std::result::Result> { +) -> std::result::Result { match body { Ok(Json(value)) if value.is_object() => Ok(value), - Ok(_) => Err(Box::new(invalid_body_error( - "Request body must be a JSON object", - ))), - Err(error) => Err(Box::new(invalid_body_error(format!( - "Request body must be valid JSON: {error}" - )))), + Ok(_) => Err("Request body must be a JSON object".to_string()), + Err(error) => Err(format!("Request body must be valid JSON: {error}")), } } @@ -327,10 +281,25 @@ async fn handle_llm_request( Ok(request) => request, Err(error) => return invalid_body_error(error.to_string()), }; - let requested_model = llm_request.model.clone(); - let algorithm = match state.algorithm_for_model(requested_model.as_deref()) { - Ok(algorithm) => algorithm, - Err(response) => return *response, + let Some(requested_model) = llm_request + .model + .clone() + .filter(|model| !model.trim().is_empty()) + else { + return error_response( + StatusCode::BAD_REQUEST, + "request body must include a non-empty string `model`", + "invalid_request_error", + "invalid_request_error", + ); + }; + let Some(algorithm) = state.algorithm_for_model(&requested_model) else { + return error_response( + StatusCode::NOT_FOUND, + format!("No route registered for model {requested_model}"), + "model_not_found", + "model_not_found", + ); }; let request = Request { @@ -343,9 +312,8 @@ async fn handle_llm_request( Err(error) => return algorithm_error(error), }; - let mut response = match translate_response(response, wire_format, requested_model) { - Ok(TranslatedResponse::Buffered(body)) => Json(body).into_response(), - Ok(TranslatedResponse::Stream(stream)) => stream.into_response(), + let mut response = match into_http_response(response, wire_format, Some(requested_model)) { + Ok(response) => response, Err(error) => return server_error(error.to_string()), }; if let Some(decision) = trace.last() { @@ -494,7 +462,7 @@ fn error_response( } async fn models(State(state): State) -> Json { - Json(model_list_payload(state.served_models())) + Json(model_list_payload(state.models())) } async fn health() -> Json { @@ -510,17 +478,13 @@ async fn not_found() -> Response { ) } -fn model_list_payload<'a>(entries: impl IntoIterator) -> Value { - let entries = entries.into_iter().collect::>(); - let model_ids = entries - .iter() - .map(|entry| entry.id.clone()) - .collect::>(); +fn model_list_payload<'a>(models: impl IntoIterator) -> Value { + let model_ids = models.into_iter().map(str::to_string).collect::>(); let first_id = model_ids.first().cloned(); let last_id = model_ids.last().cloned(); json!({ "object": "list", - "data": entries.into_iter().map(model_entry_json).collect::>(), + "data": model_ids.iter().map(|model| model_entry_json(model)).collect::>(), "first_id": first_id, "last_id": last_id, "has_more": false, @@ -529,14 +493,14 @@ fn model_list_payload<'a>(entries: impl IntoIterator) -> }) } -fn model_entry_json(entry: &ServedModel) -> Value { +fn model_entry_json(model: &str) -> Value { json!({ - "id": entry.id, + "id": model, "object": "model", "type": "model", "created": 0, "owned_by": "switchyard", - "display_name": entry.display_name, + "display_name": model, "capabilities": { "streaming": true, "tool_calling": null, @@ -552,84 +516,24 @@ fn model_entry_json(entry: &ServedModel) -> Value { fn startup_banner(options: &ServerRunOptions, state: &ServerState) -> String { let scheme = if options.is_tls() { "https" } else { "http" }; - let listen_url = url_for_addr(scheme, options.addr); - let local_url = local_url_for_addr(scheme, options.addr); - let mut output = String::new(); - - push_line(&mut output, "Switchyard libsy server"); - push_line(&mut output, format!(" listening: {listen_url}")); - if local_url != listen_url { - push_line(&mut output, format!(" local: {local_url}")); - } - push_line(&mut output, ""); - push_line(&mut output, " endpoints:"); - push_line(&mut output, " GET /health"); - push_line(&mut output, " GET /v1/models"); - push_line(&mut output, " POST /v1/chat/completions"); - push_line(&mut output, " POST /v1/messages"); - push_line(&mut output, " POST /v1/responses"); - push_line(&mut output, ""); - push_line(&mut output, " routes:"); - for model in state.served_models() { - push_line( - &mut output, - format!(" - {} ({})", model.id, model.display_name), - ); - } - push_line(&mut output, ""); - push_line(&mut output, " try:"); - push_line(&mut output, format!(" curl -s {local_url}/health")); - if let Some(model) = state.served_models().next() { - let chat_payload = json!({ - "model": model.id, - "messages": [{"role": "user", "content": "Say OK"}], - "max_tokens": 256, - }); - push_line( - &mut output, - format!(" curl -s {local_url}/v1/chat/completions -H 'content-type: application/json' -d '{chat_payload}'"), - ); - } - if options.is_tls() { - push_line( - &mut output, - " For self-signed certificates use `curl --insecure`.", - ); - } - push_line(&mut output, ""); - push_line(&mut output, " stop: Ctrl-C"); - output + format!( + "Switchyard libsy server\n listening: {}\n routes: {}", + url_for_addr(scheme, options.addr), + state.models().collect::>().join(", ") + ) } fn dry_run_summary(state: &ServerState) -> String { - let mut output = String::from("server OK:\n"); - for model in state.served_models() { - push_line( - &mut output, - format!(" - {} ({})", model.id, model.display_name), - ); - } - output -} - -fn push_line(output: &mut String, line: impl AsRef) { - output.push_str(line.as_ref()); - output.push('\n'); + format!( + "server OK: {}", + state.models().collect::>().join(", ") + ) } fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String { format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port()) } -fn local_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String { - let host = match addr.ip() { - std::net::IpAddr::V4(ip) if ip.is_unspecified() => "127.0.0.1".to_string(), - std::net::IpAddr::V6(ip) if ip.is_unspecified() => "[::1]".to_string(), - ip => host_for_url(ip), - }; - format!("{scheme}://{host}:{}", addr.port()) -} - fn host_for_url(ip: std::net::IpAddr) -> String { match ip { std::net::IpAddr::V4(ip) => ip.to_string(), diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rs index 3db288aa..9488d176 100644 --- a/crates/switchyard-server/src/response.rs +++ b/crates/switchyard-server/src/response.rs @@ -5,37 +5,32 @@ use std::error::Error; -use axum::response::sse::Sse; -use libsy::{LlmResponse, Response}; -use serde_json::Value; +use axum::response::{IntoResponse, Response as HttpResponse}; +use axum::Json; +use libsy::{LlmResponse, Response as AlgorithmResponse}; use switchyard_translation::{encode_aggregated_response, encode_stream, WireFormat}; -use crate::sse::{frame_stream, SseFrameStream}; +use crate::sse::frame_stream; type BoxError = Box; -pub(crate) enum TranslatedResponse { - /// Complete JSON response body ready for an Axum JSON response. - Buffered(Value), - /// Framed SSE response ready for Axum streaming. - Stream(Sse), -} - /// Encodes a libsy response into the endpoint's wire format. -pub(crate) fn translate_response( - response: Response, +pub(crate) fn into_http_response( + response: AlgorithmResponse, target_format: WireFormat, requested_model: Option, -) -> Result { +) -> Result { match response.llm_response { - LlmResponse::Agg(response) => Ok(TranslatedResponse::Buffered(encode_aggregated_response( + LlmResponse::Agg(response) => Ok(Json(encode_aggregated_response( &response, target_format, requested_model.as_deref(), - )?)), - LlmResponse::Stream(stream) => Ok(TranslatedResponse::Stream(frame_stream( + )?) + .into_response()), + LlmResponse::Stream(stream) => Ok(frame_stream( encode_stream(stream, target_format, requested_model)?, target_format, - ))), + ) + .into_response()), } } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index c76b941f..0db025d2 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -20,7 +20,7 @@ use libsy::algorithms::Random; use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient}; use serde_json::{json, Value}; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; -use switchyard_server::{build_switchyard_router, ServedModel, ServerState}; +use switchyard_server::{build_switchyard_router, ServerState}; use tokio::net::TcpListener; use tokio::sync::Mutex; use tokio::task::JoinHandle; @@ -133,17 +133,17 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult = Arc::new(Random::new(target_set)); - ( - ServedModel { - id: (*route_model).to_string(), - display_name: "uniform random routing".to_string(), - }, - algorithm, - ) + ((*route_model).to_string(), algorithm) }); Ok(ServerState::new(entries)?) } +async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Router)> { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(random_state(&upstream.base_url, routes)?); + Ok((upstream, app)) +} + async fn send(app: &Router, method: &str, path: &str, body: Option) -> TestResult { let mut builder = HttpRequest::builder().method(method).uri(path); let request_body = if let Some(body) = body { @@ -180,12 +180,12 @@ impl Response { } #[tokio::test] -async fn health_models_and_unknown_paths_have_stable_shapes() -> TestResult { - let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[(ROUTE_MODEL, &["model/a"])], - )?); +async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult { + let (upstream, app) = test_app(&[ + ("switchyard/coding", &["model/code"]), + ("switchyard/general", &["model/general"]), + ]) + .await?; let health = send(&app, "GET", "/health", None).await?; assert_eq!(health.status, StatusCode::OK); @@ -193,31 +193,15 @@ async fn health_models_and_unknown_paths_have_stable_shapes() -> TestResult { let models = send(&app, "GET", "/v1/models", None).await?; assert_eq!(models.status, StatusCode::OK); - assert_eq!(models.json()?["model_pool"], json!([ROUTE_MODEL])); - - let missing = send(&app, "GET", "/missing", None).await?; - assert_eq!(missing.status, StatusCode::NOT_FOUND); - assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found"); - Ok(()) -} - -#[tokio::test] -async fn multiple_algorithms_dispatch_by_route_model() -> TestResult { - let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[ - ("switchyard/coding", &["model/code"]), - ("switchyard/general", &["model/general"]), - ], - )?); - - let models = send(&app, "GET", "/v1/models", None).await?; assert_eq!( models.json()?["model_pool"], json!(["switchyard/coding", "switchyard/general"]) ); + let missing = send(&app, "GET", "/missing", None).await?; + assert_eq!(missing.status, StatusCode::NOT_FOUND); + assert_eq!(missing.json()?["error"]["code"], "endpoint_not_found"); + for (route_model, target_model) in [ ("switchyard/general", "model/general"), ("switchyard/coding", "model/code"), @@ -250,11 +234,7 @@ async fn multiple_algorithms_dispatch_by_route_model() -> TestResult { #[tokio::test] async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult { - let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[(ROUTE_MODEL, &["model/a"])], - )?); + let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; let cases = [ ( @@ -311,47 +291,9 @@ async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestRes Ok(()) } -#[tokio::test] -async fn random_routing_only_calls_configured_targets() -> TestResult { - let upstream = MockUpstream::start().await?; - let targets = ["model/a", "model/b", "model/c"]; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[(ROUTE_MODEL, &targets)], - )?); - - for _ in 0..20 { - let response = send( - &app, - "POST", - "/v1/chat/completions", - Some(json!({ - "model": ROUTE_MODEL, - "messages": [{"role": "user", "content": "hi"}] - })), - ) - .await?; - assert_eq!(response.status, StatusCode::OK); - } - - let configured = targets.into_iter().collect::>(); - let calls = upstream.calls.lock().await; - assert!(calls.iter().all(|call| { - call["model"] - .as_str() - .map(|model| configured.contains(model)) - .unwrap_or(false) - })); - Ok(()) -} - #[tokio::test] async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { - let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[(ROUTE_MODEL, &["model/a"])], - )?); + let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; let response = send( &app, @@ -373,11 +315,7 @@ async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult { #[tokio::test] async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult { - let upstream = MockUpstream::start().await?; - let app = build_switchyard_router(random_state( - &upstream.base_url, - &[(ROUTE_MODEL, &["model/a"])], - )?); + let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; let unknown = send( &app, From 2425ab78a806dd2d8734a53237e52b45ec62da26 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 21 Jul 2026 14:36:55 -0700 Subject: [PATCH 4/4] fix(server): use protocol metadata normalization Signed-off-by: nachiketb --- crates/switchyard-server/src/lib.rs | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 3fc2c979..a49e4e35 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -323,23 +323,10 @@ async fn handle_llm_request( } fn metadata_from_headers(headers: &HeaderMap) -> Metadata { - Metadata { - session_id: None, - agent_id: None, - task_id: None, - correlation_id: header_text(headers, "x-request-id"), - extra_metadata: None, - http_headers: Some(normalized_headers(headers)), - // Inbound format must not constrain the independently configured upstream. - wire_format: None, - } -} - -fn header_text(headers: &HeaderMap, name: &str) -> Option { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::to_string) + let headers = normalized_headers(headers); + let mut metadata = Metadata::from_headers(&headers); + metadata.http_headers = Some(headers); + metadata } fn normalized_headers(headers: &HeaderMap) -> BTreeMap {