From 2e2c77367d20a1a659cde33ee9e8e0fca4504771 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:32 -0700 Subject: [PATCH 1/9] fix(libsy): validate classifier scores and drain streamed verdicts Signed-off-by: Elyas Mehtabuddin --- crates/libsy/src/algorithms/llm_class.rs | 98 +++++++++++++++++++++--- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index aa10a9fe..9b27d3e3 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -179,16 +179,19 @@ impl Algorithm for LlmClassifier { classify_decision, ) .await?; - let score = classify_response - .llm_response - .as_agg() - .map(completion_text) - .unwrap_or_default() + // Drain a streamed classifier response to its aggregate so a streamed + // score is read instead of silently dropped. Keep only a valid + // probability in [0.0, 1.0]: NaN, ±inf, and out-of-range values parse + // as f64 but are not usable scores, so they collapse to None and fail + // open to strong below rather than being treated as a real verdict. + let score = completion_text(&classify_response.llm_response.into_agg().await?) .trim() .parse::() - .ok(); + .ok() + .filter(|s| (0.0..=1.0).contains(s)); - // 2. Route: pick strong/weak. Fail open — an unparseable score routes strong. + // 2. Route: pick strong/weak. Fail open — an unparseable or out-of-range + // score routes strong. let (tier, model) = match score { Some(s) if s >= self.threshold => (ClassifierTier::Strong, self.strong_model.clone()), Some(_) => (ClassifierTier::Weak, self.weak_model.clone()), @@ -215,7 +218,8 @@ impl Algorithm for LlmClassifier { #[cfg(test)] mod tests { use super::*; - use crate::{LlmResponse, LlmTarget, Response, RoutedLlmClient}; + use crate::{LlmResponse, LlmResponseChunk, LlmTarget, Response, RoutedLlmClient}; + use futures::StreamExt; use std::sync::Mutex; use switchyard_protocol::text_response; @@ -225,6 +229,9 @@ mod tests { struct ScoringClient { classifier_model: String, score: String, + /// When true, the classifier target returns its score as a live stream + /// rather than a buffered aggregate, exercising the drain-the-stream path. + stream: bool, seen: Arc>>, } @@ -237,14 +244,26 @@ mod tests { decision: Arc, ) -> Result> { let name = decision.selected_model().to_string(); - let completion = if name == self.classifier_model { + let is_classifier = name == self.classifier_model; + let completion = if is_classifier { self.score.clone() } else { format!("answer from {name}") }; self.seen.lock().map_err(|_| "lock poisoned")?.push(request); + // A streaming classifier target emits its score as a live TextDelta + // stream; the algorithm must drain it (into_agg) to read the score. + let llm_response = if is_classifier && self.stream { + let chunks = vec![Ok(LlmResponseChunk::TextDelta { + index: 0, + text: completion, + })]; + LlmResponse::Stream(futures::stream::iter(chunks).boxed()) + } else { + LlmResponse::Agg(text_response(None, completion)) + }; Ok(Response { - llm_response: LlmResponse::Agg(text_response(None, completion)), + llm_response, metadata: None, }) } @@ -252,10 +271,24 @@ mod tests { /// Build a classifier algo whose three targets share a scoring client. fn algo(threshold: f64, score: &str) -> (LlmClassifier, Arc>>) { + build_algo(threshold, score, false) + } + + /// Like [`algo`], but the classifier target returns its score as a live stream. + fn algo_streaming(threshold: f64, score: &str) -> (LlmClassifier, Arc>>) { + build_algo(threshold, score, true) + } + + fn build_algo( + threshold: f64, + score: &str, + stream: bool, + ) -> (LlmClassifier, Arc>>) { let seen = Arc::new(Mutex::new(Vec::new())); let client = Arc::new(ScoringClient { classifier_model: "router/classifier".to_string(), score: score.to_string(), + stream, seen: Arc::clone(&seen), }) as Arc; let target = |name: &str| LlmTarget { @@ -378,4 +411,49 @@ mod tests { assert_eq!(routed.score, None); Ok(()) } + + #[tokio::test] + async fn out_of_range_score_defaults_to_strong() -> Result<(), Box> { + // "NaN" parses as an f64 but is not a usable probability, so it must fail + // open to strong rather than be treated as a below-threshold verdict and + // silently downgraded to weak (NvBug 6485976). + let (algo, _) = algo(0.5, "NaN"); + let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?; + assert_eq!( + response + .llm_response + .as_agg() + .map(completion_text) + .unwrap_or_default(), + "answer from frontier/model" + ); + let routed = as_classifier(&trace[1])?; + assert_eq!(routed.tier, Some(ClassifierTier::Strong)); + assert_eq!(routed.score, None); + Ok(()) + } + + #[tokio::test] + async fn streamed_score_below_threshold_routes_weak() -> Result<(), Box> + { + // A streaming classifier target must have its score drained and read, not + // dropped — otherwise every streamed verdict falls open to strong + // regardless of value (NvBug 6485975). + let (algo, _) = algo_streaming(0.5, "0.2"); + let (trace, response) = orch(algo) + .run(Context::default(), request("say hello")) + .await?; + assert_eq!( + response + .llm_response + .as_agg() + .map(completion_text) + .unwrap_or_default(), + "answer from cheap/model" + ); + let routed = as_classifier(&trace[1])?; + assert_eq!(routed.tier, Some(ClassifierTier::Weak)); + assert_eq!(routed.score, Some(0.2)); + Ok(()) + } } From 0467a07bf3b7b84935f92b4548f4795ef4f03713 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:32 -0700 Subject: [PATCH 2/9] build(libsy): pin protocol dep version and fix strict-rustdoc links Signed-off-by: Elyas Mehtabuddin --- crates/libsy/Cargo.toml | 2 +- crates/libsy/src/core/algorithm.rs | 4 ++-- crates/libsy/src/lib.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index fca1dcf8..dd76c0a2 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -16,7 +16,7 @@ async-trait = "0.1" serde_json = "1" futures = "0.3" rand = "0.8" -switchyard-protocol = { path = "../protocol" } +switchyard-protocol = { path = "../protocol", version = "0.1.0" } tokio = { version = "1", features = ["full"] } tokio-stream = "0.1" diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 621fb980..69ce22a2 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -55,7 +55,7 @@ pub struct RoutedRequest { /// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`]. /// -/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the +/// Wraps a `DriverRequest` whose payload is a [`RoutedRequest`]. The host reads the /// routed request ([`get_routed`](Self::get_routed)) and the decision behind it /// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and /// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's @@ -196,7 +196,7 @@ impl Default for Driver { } } -/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`]. +/// One item in the stream returned by `Driver::stream` / [`Algorithm::run_stream`]. pub enum Step { /// The algorithm needs this model call performed. The host serves it (optionally /// via [`RoutedRequest::default_client`]) and fulfills it with diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 2f8eebca..eec60fb7 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -56,9 +56,9 @@ //! //! Concrete algorithms live in [`algorithms`]: //! -//! [`algorithms::Random`](crate::algorithms::Random) provides uniform random routing. +//! [`algorithms::Random`] provides uniform random routing. //! -//! [`algorithms::LlmClassifier`](crate::algorithms::LlmClassifier) classifies +//! [`algorithms::LlmClassifier`] classifies //! with one model, then routes to a strong/weak model depending on the classifier's choice. mod core; From 1658f2ba59a79b80d13bdd2d99ccb82265613ecf Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:32 -0700 Subject: [PATCH 3/9] fix(server): expose /v1/models capabilities and return 400 on context overflow Signed-off-by: Elyas Mehtabuddin --- crates/switchyard-server/src/lib.rs | 40 +++++++++++++++++++- crates/switchyard-server/tests/server.rs | 48 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 50595aaa..b9abee5e 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -446,6 +446,21 @@ fn llm_error(error: SwitchyardError) -> Response { })), ) .into_response(), + // Context-window overflow is a client-side problem (prompt too long), so + // surface it as a 400 with the OpenAI-compatible `context_length_exceeded` + // code rather than an opaque 500. + err @ (SwitchyardError::ContextWindowExceeded { .. } + | SwitchyardError::ContextPoolExhausted { .. }) => ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": err.to_string(), + "type": "invalid_request_error", + "code": "context_length_exceeded", + } + })), + ) + .into_response(), error => server_error(error.to_string()), } } @@ -541,8 +556,8 @@ fn model_entry_json(entry: &ServedModel) -> Value { "display_name": entry.display_name, "capabilities": { "streaming": true, - "tool_calling": null, - "context_window": null, + "tool_calling": true, + "context_window": inferred_context_window(&entry.display_name), "supported_inbound_formats": [ "openai-chat-completions", "openai-responses", @@ -552,6 +567,27 @@ fn model_entry_json(entry: &ServedModel) -> Value { }) } +/// Infers an advertised context window from a model's display name. +/// +/// Mirrors the Python `model_listing` fragment table: the first matching +/// substring wins, falling back to a conservative 128k default. +fn inferred_context_window(model: &str) -> u32 { + const TABLE: &[(&str, u32)] = &[ + ("deepseek-v4", 1_000_000), + ("nemotron-3-super", 1_000_000), + ("kimi-k2", 256_000), + ("nemotron-3-nano", 262_000), + ("claude", 200_000), + ]; + let lower = model.to_ascii_lowercase(); + for (frag, ctx) in TABLE { + if lower.contains(frag) { + return *ctx; + } + } + 128_000 +} + fn startup_banner(options: &ServerRunOptions, registry: &ProfileRegistry) -> String { let entries = registry.served_models(); let scheme = if options.is_tls() { "https" } else { "http" }; diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 90ee30b1..7f29c3cb 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -51,6 +51,14 @@ profiles: let models = json_body(models).await?; assert_eq!(models["object"], "list"); assert_eq!(models["data"][0]["id"], "bench"); + assert_eq!( + models["data"][0]["capabilities"]["tool_calling"], + json!(true) + ); + assert_eq!( + models["data"][0]["capabilities"]["context_window"], + json!(128000) + ); assert_eq!(models["default_model"], "bench"); assert_eq!(models["model_pool"], json!(["bench"])); @@ -206,6 +214,32 @@ async fn translation_errors_do_not_emit_routing_metadata_headers() -> TestResult Ok(()) } +#[tokio::test] +async fn context_window_overflow_maps_to_client_error() -> TestResult { + let app = build_switchyard_router(state_from_profile( + "overflow", + Arc::new(ContextOverflowProfile), + )?); + + let response = app + .oneshot(request( + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "overflow", + "messages": [{"role": "user", "content": "hi"}], + })), + )?) + .await?; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + json_body(response).await?["error"]["code"], + "context_length_exceeded" + ); + Ok(()) +} + #[tokio::test] async fn target_with_same_id_and_model_is_registered_once() -> TestResult { let _stats_guard = stats_guard().await; @@ -600,6 +634,20 @@ struct CaptureProfile { struct BadTranslationProfile; +/// Test profile that fails with a context-window overflow error. +struct ContextOverflowProfile; + +#[async_trait] +impl Profile for ContextOverflowProfile { + async fn run(&self, _input: ProfileInput) -> Result { + Err(SwitchyardError::ContextWindowExceeded { + target_id: "overflow".to_string(), + model: "overflow".to_string(), + message: "prompt is too long".to_string(), + }) + } +} + #[async_trait] impl Profile for BadTranslationProfile { async fn run(&self, _input: ProfileInput) -> Result { From 0276280f556d1e8088cee3b0fbd444e72ea26a01 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:32 -0700 Subject: [PATCH 4/9] feat(routing): add context-window fallback retry to v2 random-routing Signed-off-by: Elyas Mehtabuddin --- .../src/profiles/random_routing.rs | 308 +++++++++++++++--- .../tests/random_routing_profile.rs | 2 + 2 files changed, 270 insertions(+), 40 deletions(-) diff --git a/crates/switchyard-components-v2/src/profiles/random_routing.rs b/crates/switchyard-components-v2/src/profiles/random_routing.rs index dea63c67..a18130fc 100644 --- a/crates/switchyard-components-v2/src/profiles/random_routing.rs +++ b/crates/switchyard-components-v2/src/profiles/random_routing.rs @@ -10,7 +10,7 @@ use switchyard_components::request_processors::{ RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier, }; use switchyard_components::StatsAccumulator; -use switchyard_core::{ChatResponse, LlmTarget, Result, SwitchyardError}; +use switchyard_core::{ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError}; use crate::backend::{native_target_backend, TargetBackend}; use crate::profile_stats_accumulator; @@ -35,6 +35,9 @@ pub struct RandomRoutingProfileConfig { /// Optional deterministic RNG seed for reproducible routing. #[serde(default)] pub rng_seed: Option, + /// Target used for one retry after context-window overflow; defaults to the strong target. + #[serde(default)] + pub fallback_target_on_evict: Option, } impl ProfileConfig for RandomRoutingProfileConfig { @@ -42,6 +45,18 @@ impl ProfileConfig for RandomRoutingProfileConfig { /// Builds the runtime profile using existing native backend construction. fn build(&self) -> Result { + // Resolve the evict fallback to the strong target when unset, then validate it + // names one of this profile's two configured targets. + let fallback_target_on_evict = self + .fallback_target_on_evict + .clone() + .unwrap_or_else(|| self.strong.id.clone()); + if fallback_target_on_evict != self.strong.id && fallback_target_on_evict != self.weak.id { + return Err(SwitchyardError::InvalidConfig(format!( + "fallback_target_on_evict={} must match one of [{}, {}]", + fallback_target_on_evict, self.weak.id, self.strong.id + ))); + } let router_config = RandomRoutingProcessorConfig::new(self.strong.clone(), self.weak.clone()) .with_strong_probability(self.strong_probability)? @@ -50,6 +65,7 @@ impl ProfileConfig for RandomRoutingProfileConfig { router: RandomRoutingEngine::new(router_config)?, strong_backend: native_target_backend(self.strong.clone())?, weak_backend: native_target_backend(self.weak.clone())?, + fallback_target_on_evict, stats: profile_stats_accumulator(), }) } @@ -60,6 +76,7 @@ pub struct RandomRoutingProfile { router: RandomRoutingEngine, strong_backend: TargetBackend, weak_backend: TargetBackend, + fallback_target_on_evict: LlmTargetId, stats: StatsAccumulator, } @@ -85,19 +102,94 @@ impl RandomRoutingProfile { } // Finds the routed backend by the target ID emitted by the routing engine. - fn selected_backend(&self, decision: &RandomRoutingDecision) -> Result<&TargetBackend> { - if decision.selected_target == self.strong_backend.target().id { + fn backend_for_target(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> { + if *target_id == self.strong_backend.target().id { Ok(&self.strong_backend) - } else if decision.selected_target == self.weak_backend.target().id { + } else if *target_id == self.weak_backend.target().id { Ok(&self.weak_backend) } else { Err(SwitchyardError::InvalidConfig(format!( - "router selected target {} that is not configured for this profile", - decision.selected_target + "random-routing selected target {target_id} that is not configured for this profile" ))) } } + // Maps a configured target ID back to its random-routing tier. + fn tier_for_target(&self, target_id: &LlmTargetId) -> Result { + if *target_id == self.strong_backend.target().id { + Ok(RandomRoutingTier::Strong) + } else if *target_id == self.weak_backend.target().id { + Ok(RandomRoutingTier::Weak) + } else { + Err(SwitchyardError::InvalidConfig(format!( + "random-routing target {target_id} is not configured for this profile" + ))) + } + } + + // Rewrites the processed request to dispatch against the evict fallback target. + fn fallback_processed_request( + &self, + processed: &RandomRoutingProcessedRequest, + ) -> Result { + let backend = self.backend_for_target(&self.fallback_target_on_evict)?; + let target = backend.target(); + let mut profile_input = processed.profile_input.clone(); + profile_input.request.set_model(target.model.as_str()); + let mut decision = processed.decision.clone(); + decision.tier = self.tier_for_target(&target.id)?; + decision.selected_target = target.id.clone(); + decision.selected_model = target.model.clone(); + Ok(RandomRoutingProcessedRequest { + profile_input, + decision, + }) + } + + // Dispatches the routed request to its backend and reports the backend latency. + async fn call_selected( + &self, + processed: &RandomRoutingProcessedRequest, + ) -> (Result, f64) { + let started_at = Instant::now(); + let backend = match self.backend_for_target(&processed.decision.selected_target) { + Ok(backend) => backend, + Err(error) => return (Err(error), 0.0), + }; + let result = backend.call(&processed.profile_input.request).await; + let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0; + (result, latency_ms) + } + + fn record_success( + &self, + decision: &RandomRoutingDecision, + response: ChatResponse, + profile_started_at: Instant, + backend_latency_ms: f64, + ) -> Result { + self.stats.record_success( + decision.selected_model.as_str(), + Some(backend_latency_ms), + Some(decision.tier.as_str()), + )?; + record_usage_or_wrap_stream( + &self.stats, + decision.selected_model.as_str(), + Some(decision.tier.as_str()), + profile_started_at, + backend_latency_ms, + response, + ) + } + + fn record_error(&self, decision: &RandomRoutingDecision) -> Result<()> { + self.stats.record_error( + decision.selected_model.as_str(), + Some(decision.tier.as_str()), + ) + } + fn routing_metadata(&self, decision: &RandomRoutingDecision) -> RoutingMetadata { let comparison = if decision.tier == RandomRoutingTier::Strong { "<" @@ -141,45 +233,61 @@ impl ProfileHooks for RandomRoutingProfile { #[async_trait] impl Profile for RandomRoutingProfile { - /// Executes random routing while keeping selected-target state local to this call. + /// Executes random routing with one context-window fallback retry. async fn run(&self, input: ProfileInput) -> Result { let profile_started_at = Instant::now(); let processed = self.process(input).await?; - let decision = &processed.decision; - let selected_backend = self.selected_backend(decision)?; - let backend_started_at = Instant::now(); - let response = match selected_backend - .call(&processed.profile_input.request) - .await - { - Ok(response) => response, - Err(error) => { - self.stats.record_error( - decision.selected_model.as_str(), - Some(decision.tier.as_str()), + let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await; + match first_result { + Ok(response) => { + let response = self.record_success( + &processed.decision, + response, + profile_started_at, + first_backend_latency_ms, )?; - return Err(error); + let response = self.rprocess(&processed, response).await?; + Ok(ProfileResponse::with_routing_metadata( + response, + self.routing_metadata(&processed.decision), + )) } - }; - let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0; - self.stats.record_success( - decision.selected_model.as_str(), - Some(backend_latency_ms), - Some(decision.tier.as_str()), - )?; - let response = record_usage_or_wrap_stream( - &self.stats, - decision.selected_model.as_str(), - Some(decision.tier.as_str()), - profile_started_at, - backend_latency_ms, - response, - )?; - let response = self.rprocess(&processed, response).await?; - Ok(ProfileResponse::with_routing_metadata( - response, - self.routing_metadata(decision), - )) + Err(SwitchyardError::ContextWindowExceeded { .. }) => { + let retry = self.fallback_processed_request(&processed)?; + let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await; + match retry_result { + Ok(response) => { + let response = self.record_success( + &retry.decision, + response, + profile_started_at, + retry_backend_latency_ms, + )?; + let response = self.rprocess(&retry, response).await?; + Ok(ProfileResponse::with_routing_metadata( + response, + self.routing_metadata(&retry.decision), + )) + } + Err(SwitchyardError::ContextWindowExceeded { target_id, .. }) => { + self.record_error(&retry.decision)?; + Err(SwitchyardError::ContextPoolExhausted { + last_target_id: target_id, + reason: "all attempted targets returned context-window overflow" + .to_string(), + }) + } + Err(error) => { + self.record_error(&retry.decision)?; + Err(error) + } + } + } + Err(error) => { + self.record_error(&processed.decision)?; + Err(error) + } + } } } @@ -217,6 +325,13 @@ mod tests { struct StreamingUsageBackend; + // Records the call, then reports a context-window overflow for evict/retry tests. + struct ContextOverflowBackend { + name: &'static str, + target_id: String, + calls: Arc>>, + } + #[async_trait] impl ProfileBackend for TestBackend { async fn call(&self, request: &ChatRequest) -> Result { @@ -259,6 +374,24 @@ mod tests { } } + #[async_trait] + impl ProfileBackend for ContextOverflowBackend { + async fn call(&self, request: &ChatRequest) -> Result { + self.calls + .lock() + .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))? + .push(ObservedCall { + backend: self.name, + body: request.body().clone(), + }); + Err(SwitchyardError::ContextWindowExceeded { + target_id: self.target_id.clone(), + model: request.model().unwrap_or("").to_string(), + message: "prompt is too long".to_string(), + }) + } + } + fn target(id: &str, model: &str) -> Result { let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?); target.format = BackendFormat::OpenAi; @@ -271,6 +404,7 @@ mod tests { weak, strong_probability: probability, rng_seed: Some(7), + fallback_target_on_evict: None, } } @@ -327,6 +461,7 @@ mod tests { router: RandomRoutingEngine::new(router_config)?, strong_backend, weak_backend, + fallback_target_on_evict: strong.id.clone(), stats: StatsAccumulator::new(), }; Ok((profile, calls)) @@ -416,6 +551,7 @@ mod tests { let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone()) .with_strong_probability(0.0)? .with_rng_seed(Some(7)); + let strong_id = strong.id.clone(); let profile = RandomRoutingProfile { router: RandomRoutingEngine::new(router_config)?, strong_backend: TargetBackend::new( @@ -426,6 +562,7 @@ mod tests { }), ), weak_backend: TargetBackend::new(weak, Arc::new(StreamingUsageBackend)), + fallback_target_on_evict: strong_id, stats: StatsAccumulator::new(), }; @@ -556,4 +693,95 @@ mod tests { assert!(observed(&calls)?.is_empty()); Ok(()) } + + #[tokio::test] + async fn run_retries_fallback_target_after_context_overflow() -> Result<()> { + let strong = target("strong", "frontier/model")?; + let weak = target("weak", "cheap/model")?; + let calls = Arc::new(Mutex::new(Vec::new())); + // strong_probability 0.0 selects the weak target first; the default fallback is strong. + let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone()) + .with_strong_probability(0.0)? + .with_rng_seed(Some(7)); + let profile = RandomRoutingProfile { + router: RandomRoutingEngine::new(router_config)?, + strong_backend: TargetBackend::new( + strong.clone(), + Arc::new(TestBackend { + name: "strong-backend", + calls: calls.clone(), + }), + ), + weak_backend: TargetBackend::new( + weak.clone(), + Arc::new(ContextOverflowBackend { + name: "weak-backend", + target_id: weak.id.to_string(), + calls: calls.clone(), + }), + ), + fallback_target_on_evict: strong.id.clone(), + stats: StatsAccumulator::new(), + }; + + let response = profile + .run(profile_input(ChatRequest::openai_chat(json!({ + "model": "client/model", + "messages": [{"role": "user", "content": "continue"}], + })))) + .await?; + + let routing_metadata = response + .routing_metadata + .as_ref() + .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?; + assert_eq!( + routing_metadata.selected_model.as_deref(), + Some("frontier/model") + ); + assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong")); + let response = response.response; + + let calls = observed(&calls)?; + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].backend, "weak-backend"); + assert_eq!(calls[0].body["model"], "cheap/model"); + assert_eq!(calls[1].backend, "strong-backend"); + assert_eq!(calls[1].body["model"], "frontier/model"); + match response { + ChatResponse::OpenAiCompletion(body) => { + assert_eq!(body.body()["served_by"], "strong-backend"); + assert_eq!(body.body()["model"], "frontier/model"); + } + _ => return Err(SwitchyardError::Other("unexpected response shape".into())), + } + Ok(()) + } + + #[test] + fn build_rejects_fallback_target_not_matching_strong_or_weak() -> Result<()> { + let mut config = config( + target("strong", "frontier/model")?, + target("weak", "cheap/model")?, + 0.5, + ); + config.fallback_target_on_evict = Some(LlmTargetId::new("ghost")?); + + match config.build() { + Err(SwitchyardError::InvalidConfig(message)) => { + assert!(message.contains("fallback_target_on_evict")); + } + Ok(_) => { + return Err(SwitchyardError::Other( + "unknown fallback target should reject profile construction".into(), + )); + } + Err(other) => { + return Err(SwitchyardError::Other(format!( + "expected InvalidConfig, got {other}" + ))); + } + } + Ok(()) + } } diff --git a/crates/switchyard-components-v2/tests/random_routing_profile.rs b/crates/switchyard-components-v2/tests/random_routing_profile.rs index 66fd2cb5..963d5291 100644 --- a/crates/switchyard-components-v2/tests/random_routing_profile.rs +++ b/crates/switchyard-components-v2/tests/random_routing_profile.rs @@ -19,6 +19,7 @@ fn config(strong: LlmTarget, weak: LlmTarget, probability: f64) -> RandomRouting weak, strong_probability: probability, rng_seed: Some(7), + fallback_target_on_evict: None, } } @@ -50,6 +51,7 @@ fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> { "weak": config.weak, "strong_probability": 0.5, "rng_seed": 7, + "fallback_target_on_evict": "strong", "stats": false, }); let error = serde_json::from_value::(old_stats_toggle) From 967f72bf2eb3695e6b9ffe13da2729d32d5b5cc8 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:32 -0700 Subject: [PATCH 5/9] fix(cli): forward harness args after --, honor saved provider/env key, clarify serve errors Signed-off-by: Elyas Mehtabuddin --- switchyard/cli/configure_command.py | 17 ++++++- switchyard/cli/switchyard_cli.py | 26 ++++++---- tests/test_launch_claude.py | 50 ++++++++++++++++++- tests/test_serve_profile_config.py | 42 ++++++++++++++++ tests/test_user_config.py | 76 +++++++++++++++++++++++++++++ 5 files changed, 199 insertions(+), 12 deletions(-) diff --git a/switchyard/cli/configure_command.py b/switchyard/cli/configure_command.py index 20eda6eb..fc8e7645 100644 --- a/switchyard/cli/configure_command.py +++ b/switchyard/cli/configure_command.py @@ -367,13 +367,15 @@ def cmd_configure(request: ConfigureRequest) -> None: print("No Switchyard user config found.") return - provider = request.provider target_scope = request.target configure_claude = target_scope in ("all", "claude") configure_codex = target_scope in ("all", "codex") configure_openclaw = target_scope in ("all", "openclaw") existing_config = load_user_config() existing_credentials = load_user_credentials() + # A CLI --provider wins; otherwise honor the saved default_provider so the + # save path doesn't overwrite it with the argparse fallback. + provider = request.provider or existing_config.default_provider skill_distillation = _apply_skill_distillation_args( existing_config.skill_distillation, request, @@ -417,9 +419,22 @@ def cmd_configure(request: ConfigureRequest) -> None: api_key = request.api_key if not api_key: existing_api_key = existing_credentials.api_key(provider) + # Fall back to an env/secrets key so non-interactive `configure` (no + # --api-key, no saved key) picks up e.g. OPENROUTER_API_KEY instead of + # erroring out that it "requires an API key". + env_api_key = resolve_provider_connectivity( + cli_api_key=None, + cli_base_url=None, + api_key_env_vars=("OPENROUTER_API_KEY", "NVIDIA_API_KEY", "OPENAI_API_KEY"), + base_url_env_vars=("OPENROUTER_BASE_URL", "NVIDIA_BASE_URL", "OPENAI_BASE_URL"), + secrets=load_secrets(), + secrets_section_priority=DEFAULT_SECRETS_SECTION_PRIORITY, + default_provider=provider, + ).api_key prompt_default_api_key = ( existing_api_key or request.prompt_default_api_key + or env_api_key ) prompt_default_api_key_source = request.prompt_default_api_key_source if reuse_existing_provider and existing_api_key: diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py index cac0ff3d..b7feb509 100644 --- a/switchyard/cli/switchyard_cli.py +++ b/switchyard/cli/switchyard_cli.py @@ -434,9 +434,9 @@ def _cmd_serve(args: argparse.Namespace) -> None: saved = load_user_config().routing_profiles if not saved: raise SystemExit( - "serve: no routing-profiles given. Pass --routing-profiles " - "PATH or run `switchyard configure --routing-profiles PATH` " - "to save one." + "serve: no config given. Pass --config PATH for a v2 profile " + "config (recommended), or --routing-profiles PATH for a legacy " + "route bundle." ) # Saved bundles are stored as parsed dicts, so we can skip the # YAML parse step and feed them straight into the dict-driven @@ -514,6 +514,7 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None: _cmd_serve_mixed_profile_config(args, python_profiles) return + from switchyard_rust import SwitchyardDuplicateRegistrationError from switchyard_rust.server import run_profile_server port = args.port if isinstance(args.port, int) else resolve_port() @@ -521,7 +522,10 @@ def _cmd_serve_profile_config(args: argparse.Namespace) -> None: "Switchyard components-v2 profile config loaded from %s", args.config, ) - run_profile_server(args.config, args.host, port) + try: + run_profile_server(args.config, args.host, port) + except SwitchyardDuplicateRegistrationError as exc: + raise SystemExit(f"serve --config: {exc}") from exc def _cmd_serve_mixed_profile_config( @@ -985,7 +989,7 @@ def _build_parser() -> argparse.ArgumentParser: ), ) cfg.add_argument( - "--provider", type=str, default=DEFAULT_PROVIDER, + "--provider", type=str, default=None, help=f"Provider id to configure (default: {DEFAULT_PROVIDER})", ) cfg.add_argument( @@ -1456,10 +1460,14 @@ def main() -> None: # switchyard --routing-profiles dev.yaml -- launch claude # The '--' is purely visual — argparse doesn't need it. argv = list(sys.argv[1:]) - try: - argv.pop(argv.index("--")) - except ValueError: - pass + # Only strip a '--' that precedes the subcommand token; a '--' after it is + # the harness separator (e.g. `launch claude ... -- --version`) and must + # survive so forwarded args reach the launcher instead of tripping argparse. + _SUBCOMMANDS = ("serve", "configure", "launch", "verify") + sep_idx = argv.index("--") if "--" in argv else len(argv) + cmd_idx = next((i for i, t in enumerate(argv) if t in _SUBCOMMANDS), len(argv)) + if sep_idx < cmd_idx: + argv.pop(sep_idx) args = parser.parse_args(argv) if args.routing_profiles is not None: diff --git a/tests/test_launch_claude.py b/tests/test_launch_claude.py index 3acc9679..64a6cff8 100644 --- a/tests/test_launch_claude.py +++ b/tests/test_launch_claude.py @@ -140,17 +140,63 @@ def test_configure_parser_builds_a_complete_request() -> None: # A real `switchyard configure` parse must fill every ConfigureRequest field # through the one from_namespace boundary. This is the check the original bug # lacked: disable_skill_distillation is the field that crashed first-run. - from switchyard.cli.config.user_config import DEFAULT_PROVIDER from switchyard.cli.switchyard_cli import _build_parser namespace = _build_parser().parse_args(["configure"]) request = ConfigureRequest.from_namespace(namespace) - assert request.provider == DEFAULT_PROVIDER + # --provider defaults to None so it can't shadow a saved default_provider; + # cmd_configure resolves the effective provider from the saved config. + assert request.provider is None assert request.disable_skill_distillation is False assert request.routing_profiles is None # the global flag is merged in +def test_main_forwards_harness_args_after_separator(monkeypatch, tmp_path) -> None: + """``launch claude ... -- --version`` forwards harness args past the ``--``. + + main() must only strip a ``--`` that occurs before the subcommand token, + so the launcher-side ``--`` separating claude args survives argparse and + the forwarded args reach ``launch_claude`` intact. Before the fix the first + ``--`` was popped unconditionally, so ``--version`` hit argparse as an + unrecognized argument (``SystemExit(2)``). + """ + import sys + + from switchyard.cli import switchyard_cli as cli + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr( + sys, "argv", + [ + "switchyard", "launch", "claude", + "--model", "nvidia/x", "--api-key", "sk-test", + "--", "--version", + ], + ) + monkeypatch.setattr( + "switchyard.cli.launch_command.resolve_launch_connectivity", + lambda args, **_kw: ("sk-test", "https://inference-api.nvidia.com/v1"), + ) + + captured: dict = {} + + def fake_launch(**kwargs): + captured.update(kwargs) + raise SystemExit(0) + + monkeypatch.setattr( + "switchyard.cli.launchers.claude_code_launcher.launch_claude", + fake_launch, + ) + + with pytest.raises(SystemExit) as excinfo: + cli.main() + + assert excinfo.value.code == 0 + assert captured["claude_args"] == ["--version"] + + # --------------------------------------------------------------------------- # _ModelRewriteRequestProcessor # --------------------------------------------------------------------------- diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py index 29809fe3..3e1cba63 100644 --- a/tests/test_serve_profile_config.py +++ b/tests/test_serve_profile_config.py @@ -303,6 +303,48 @@ def fail_inspection(_path: str) -> list[str]: _cmd_serve_profile_config(_serve_args(path)) +_DUPLICATE_ID_CONFIG = """ +targets: + shared: + model: provider/shared + format: openai + base_url: http://127.0.0.1:9/shared/v1 + api_key: test-key + +profiles: + shared: + type: passthrough + target: shared +""" + + +def test_serve_no_config_error_mentions_config_flag( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """`serve` with neither --config nor a saved bundle points at --config.""" + from switchyard.cli.switchyard_cli import _build_parser, _cmd_serve + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + parser = _build_parser() + args = parser.parse_args(["serve", "--port", "4000"]) + with pytest.raises(SystemExit) as excinfo: + _cmd_serve(args) + assert "--config" in str(excinfo.value) + + +def test_serve_config_duplicate_model_id_exits_cleanly(tmp_path: Path) -> None: + """A profile id colliding with a target id surfaces as a clean SystemExit. + + The Rust profile server raises SwitchyardDuplicateRegistrationError while + registering routes (before binding a socket); `serve --config` maps it to a + one-line diagnostic instead of a raw traceback. + """ + path = _write(tmp_path, _DUPLICATE_ID_CONFIG) + with pytest.raises(SystemExit, match="already registered"): + _cmd_serve_profile_config(_serve_args(path)) + + def _serve_args(path: Path) -> argparse.Namespace: return argparse.Namespace( config=str(path), diff --git a/tests/test_user_config.py b/tests/test_user_config.py index ec5a57da..752ae844 100644 --- a/tests/test_user_config.py +++ b/tests/test_user_config.py @@ -333,6 +333,82 @@ def test_configure_invalid_skill_namespace_reports_user_error( assert "letters, numbers, dot, underscore, and hyphen" in message +def test_configure_list_models_honors_saved_default_provider( + monkeypatch, + tmp_path, +): + """`configure --list-models` uses the saved default_provider, not the + argparse --provider default (which now defaults to None so it can't shadow + the saved value).""" + from switchyard.cli.configure_command import _list_models + from switchyard.cli.switchyard_cli import _build_parser + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + save_user_config( + UserConfig( + default_provider="nvidia", + providers={"nvidia": ProviderConfig(base_url="https://nvidia.test/v1")}, + ), + config_dir=tmp_path, + ) + save_user_credentials( + UserCredentials(api_keys={"nvidia": "nvidia-key"}), + config_dir=tmp_path, + ) + monkeypatch.setattr("switchyard.server.server_util.load_secrets", lambda: {}) + + captured: dict = {} + monkeypatch.setattr( + "switchyard.cli.configure_command.fetch_model_ids", + lambda base_url, api_key: captured.update(base_url=base_url, api_key=api_key) or [], + ) + monkeypatch.setattr( + "switchyard.cli.configure_command.render_models", + lambda model_ids, request: "", + ) + + parser = _build_parser() + assert parser.parse_args(["configure"]).provider is None + args = parser.parse_args(["configure", "--list-models"]) + + _list_models(args) + + assert captured["base_url"] == "https://nvidia.test/v1" + assert captured["api_key"] == "nvidia-key" + + +def test_configure_non_interactive_uses_env_api_key( + monkeypatch, + tmp_path, +): + """Non-interactive `configure` falls back to an env/secrets API key instead + of erroring out when none is passed on the CLI.""" + from switchyard.cli.switchyard_cli import _build_parser, _cmd_configure + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("OPENROUTER_API_KEY", "env-or-key") + monkeypatch.setattr( + "switchyard.cli.command_utils.is_interactive_terminal", + lambda: False, + ) + monkeypatch.setattr( + "switchyard.cli.configure_command.is_interactive_terminal", + lambda: False, + ) + monkeypatch.setattr("switchyard.cli.configure_command.load_secrets", lambda: {}) + + parser = _build_parser() + args = parser.parse_args([ + "configure", + "--no-model-discovery", + "--target", "provider", + ]) + + _cmd_configure(args) + + assert load_user_credentials(tmp_path).api_key("openrouter") == "env-or-key" + + def test_redacted_snapshot_surfaces_only_route_ids(tmp_path): """The snapshot exposes route ids but never the full bundle (env-var references inside the bundle may resolve to secrets at run time).""" From 6109ae43f7bf95e67ea1a6858d4146ce59498340 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:33 -0700 Subject: [PATCH 6/9] fix(launch): guard zero-config default trio and report routing bundle in --dry-run Signed-off-by: Elyas Mehtabuddin --- docs/guides/agent_launchers.md | 5 ++ switchyard/cli/launch_command.py | 72 +++++++++++++++++ switchyard/cli/output.py | 38 ++++++--- tests/test_launch_claude_deterministic.py | 98 ++++++++++++++++++++++- tests/test_launch_codex.py | 5 +- 5 files changed, 206 insertions(+), 12 deletions(-) diff --git a/docs/guides/agent_launchers.md b/docs/guides/agent_launchers.md index 50e86019..0434ec69 100644 --- a/docs/guides/agent_launchers.md +++ b/docs/guides/agent_launchers.md @@ -49,6 +49,11 @@ strong tier using the validated coding-agent trio: when route resolution selects an explicit or saved legacy route bundle or single model. +The default trio uses OpenRouter model ids and therefore requires OpenRouter +connectivity (`OPENROUTER_API_KEY`). If the resolved provider is not OpenRouter +and you pass no model overrides, the launcher fails fast — pass `--model` (and +optionally `--weak-model`/`--classifier-model`) or set `OPENROUTER_API_KEY`. + ## Override the default route Use `--model` for a single-model passthrough session: diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py index 5db565de..71c71ed1 100644 --- a/switchyard/cli/launch_command.py +++ b/switchyard/cli/launch_command.py @@ -97,6 +97,47 @@ def resolve_launch_connectivity( return connectivity.api_key, connectivity.base_url +# Marker for the OpenRouter gateway. The zero-flag deterministic default trio +# uses OpenRouter-only model ids, so a resolved endpoint that isn't OpenRouter +# cannot serve them. +_OPENROUTER_URL_MARKER = "openrouter.ai" + + +def _reject_openrouter_only_default_trio( + target: str, + args: argparse.Namespace, + api_key_env_vars: tuple[str, ...], + base_url: str, +) -> None: + """Fail fast when the zero-config default trio can't reach the provider. + + The deterministic zero-flag default uses OpenRouter-only model ids + (``anthropic/claude-opus-4.7`` / ``moonshotai/kimi-k2.6`` / + ``google/gemini-3.5-flash``). When the user supplied no model overrides and + the resolved endpoint is not OpenRouter, those ids are inaccessible to the + resolved key, so raise :class:`SystemExit` pointing the user at ``--model`` + or ``OPENROUTER_API_KEY`` instead of launching a trio that can't work. + """ + if ( + getattr(args, "model", None) + or getattr(args, "weak_model", None) + or getattr(args, "classifier_model", None) + ): + return + if _OPENROUTER_URL_MARKER in base_url: + return + provider = _resolve_launch_connectivity( + args, api_key_env_vars=api_key_env_vars, + ).provider + raise SystemExit( + f"launch {target}: the zero-config default trio " + f"(anthropic/claude-opus-4.7, moonshotai/kimi-k2.6, " + f"google/gemini-3.5-flash) is OpenRouter-only; the resolved provider " + f"is {provider!r}. Pass --model (and optionally " + f"--weak-model/--classifier-model), or set OPENROUTER_API_KEY." + ) + + def _is_deterministic_launch( target: str, args: argparse.Namespace, @@ -605,6 +646,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None: classifier_model=dry_run_classifier_model, profile=dry_run_profile, classifier_min_confidence=dry_run_min_confidence, + routing_profiles=routing_profiles, )) return @@ -614,6 +656,17 @@ def cmd_launch_claude(args: argparse.Namespace) -> None: startup_timing.mark("config + credentials resolved") if deterministic: + _reject_openrouter_only_default_trio( + target="claude", + args=args, + api_key_env_vars=( + "OPENROUTER_API_KEY", + "NVIDIA_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + ), + base_url=primary_connectivity.base_url, + ) primary_connectivity = require_launch_tier_key( target="claude", tier=PRIMARY_TIER, @@ -798,6 +851,7 @@ def cmd_launch_codex(args: argparse.Namespace) -> None: classifier_model=dry_run_classifier_model, profile=dry_run_profile, classifier_min_confidence=dry_run_min_confidence, + routing_profiles=routing_profiles, )) return @@ -806,6 +860,12 @@ def cmd_launch_codex(args: argparse.Namespace) -> None: ) if deterministic: + _reject_openrouter_only_default_trio( + target="codex", + args=args, + api_key_env_vars=("OPENROUTER_API_KEY", "NVIDIA_API_KEY", "OPENAI_API_KEY"), + base_url=primary_connectivity.base_url, + ) primary_connectivity = require_launch_tier_key( target="codex", tier=PRIMARY_TIER, @@ -1006,6 +1066,7 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None: classifier_model=dry_run_classifier_model, profile=dry_run_profile, classifier_min_confidence=dry_run_min_confidence, + routing_profiles=routing_profiles, )) return @@ -1014,6 +1075,17 @@ def cmd_launch_openclaw(args: argparse.Namespace) -> None: ) if deterministic: + _reject_openrouter_only_default_trio( + target="openclaw", + args=args, + api_key_env_vars=( + "OPENROUTER_API_KEY", + "NVIDIA_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + ), + base_url=primary_connectivity.base_url, + ) primary_connectivity = require_launch_tier_key( target="openclaw", tier=PRIMARY_TIER, diff --git a/switchyard/cli/output.py b/switchyard/cli/output.py index c2f614a7..887e96c5 100644 --- a/switchyard/cli/output.py +++ b/switchyard/cli/output.py @@ -143,12 +143,19 @@ def format_dry_run( classifier_model: str | None = None, profile: str | None = None, classifier_min_confidence: float | None = None, + routing_profiles: str | None = None, ) -> str: """Render resolved launch settings without exposing secrets. The optional ``classifier_*`` / ``profile`` kwargs are populated by the deterministic-routing dispatch path — the routing-policy fields that aren't carried on :class:`LaunchRouteConfig`. + + ``routing_profiles`` is the resolved routing-profiles YAML path for a + multi-chain bundle launch. When set, the summary reports ``route: bundle`` + plus the bundle path and its route ids instead of the single-tier + :func:`format_route_config` view (``route`` is a placeholder single-tier + config in that case). """ lines = [ @@ -158,15 +165,28 @@ def format_dry_run( f"port: {port if port is not None else 'auto'}", f"timeout: {timeout if timeout is not None else 'default'}", ] - for route_line in format_route_config(route): - lines.append(route_line) - if route.type == "deterministic": - if classifier_model: - lines.append(f"classifier model: {classifier_model}") - if profile: - lines.append(f"profile: {profile}") - if classifier_min_confidence is not None: - lines.append(f"min confidence: {classifier_min_confidence:.2f}") + if routing_profiles: + from switchyard.cli.route_bundle import ( + parse_routing_profiles_file, + routing_profile_model_ids, + ) + + lines.append("route: bundle") + lines.append(f"routing profiles: {routing_profiles}") + for route_id in routing_profile_model_ids( + parse_routing_profiles_file(routing_profiles) + ): + lines.append(f" {route_id}") + else: + for route_line in format_route_config(route): + lines.append(route_line) + if route.type == "deterministic": + if classifier_model: + lines.append(f"classifier model: {classifier_model}") + if profile: + lines.append(f"profile: {profile}") + if classifier_min_confidence is not None: + lines.append(f"min confidence: {classifier_min_confidence:.2f}") if forwarded_args: lines.append(f"forwarded args: {' '.join(forwarded_args)}") return "\n".join(lines) diff --git a/tests/test_launch_claude_deterministic.py b/tests/test_launch_claude_deterministic.py index c06a371a..a9d28784 100644 --- a/tests/test_launch_claude_deterministic.py +++ b/tests/test_launch_claude_deterministic.py @@ -10,8 +10,6 @@ ``--classifier-min-confidence``) still tune the default trio. """ -from __future__ import annotations - import pytest @@ -249,6 +247,102 @@ def fake_launch(**_kwargs): # Dry-run prints + returns without SystemExit. _cmd_launch_claude(args) + def test_dry_run_routing_profiles_reports_bundle( + self, monkeypatch, tmp_path, capsys, + ) -> None: + """A ``--routing-profiles`` dry run reports ``route: bundle``, not single.""" + from switchyard.cli.switchyard_cli import _build_parser, _cmd_launch_claude + + yaml_path = tmp_path / "routes.yaml" + yaml_path.write_text( + "routes:\n" + " my-route:\n" + " type: model\n" + " model: openai/gpt-4o-mini\n", + encoding="utf-8", + ) + + parser = _build_parser() + args = parser.parse_args([ + "--routing-profiles", str(yaml_path), + "--", "launch", "claude", "--dry-run", + ]) + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + monkeypatch.setattr( + "switchyard.cli.launch_command.resolve_launch_connectivity", + lambda args, **_kw: ("sk-test", "https://openrouter.ai/api/v1"), + ) + + _cmd_launch_claude(args) + + out = capsys.readouterr().out + assert "route: bundle" in out + assert "route: single" not in out + + def test_zero_config_non_openrouter_rejects_default_trio( + self, monkeypatch, tmp_path, + ) -> None: + """Zero-flag launch against a non-OpenRouter provider fails fast.""" + from switchyard.cli.switchyard_cli import _build_parser, _cmd_launch_claude + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("NVIDIA_API_KEY", "nvapi-test") + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setattr( + "switchyard.cli.launch_command.resolve_launch_connectivity", + lambda args, **_kw: ("nvapi-test", "https://inference-api.nvidia.com/v1"), + ) + + parser = _build_parser() + args = parser.parse_args(["launch", "claude"]) + + with pytest.raises(SystemExit) as exc_info: + _cmd_launch_claude(args) + + message = str(exc_info.value) + assert "OpenRouter-only" in message + assert "anthropic/claude-opus-4.7" in message + assert "moonshotai/kimi-k2.6" in message + assert "google/gemini-3.5-flash" in message + assert "'nvidia'" in message + + def test_model_flag_bypasses_non_openrouter_guard( + self, monkeypatch, tmp_path, + ) -> None: + """Passing --model opts out of the default trio, bypassing the guard.""" + from switchyard.cli.switchyard_cli import _build_parser, _cmd_launch_claude + + monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("NVIDIA_API_KEY", "nvapi-test") + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setattr( + "switchyard.cli.launch_command.resolve_launch_connectivity", + lambda args, **_kw: ("nvapi-test", "https://inference-api.nvidia.com/v1"), + ) + + captured: dict = {} + + def fake_passthrough(**kwargs): + captured.update(kwargs) + raise SystemExit(0) + + monkeypatch.setattr( + "switchyard.cli.launchers.claude_code_launcher.launch_claude", + fake_passthrough, + ) + + parser = _build_parser() + args = parser.parse_args([ + "launch", "claude", "--model", "nvidia/moonshotai/kimi-k2.5", + ]) + + with pytest.raises(SystemExit) as exc_info: + _cmd_launch_claude(args) + + assert "OpenRouter-only" not in str(exc_info.value) + assert captured["model"] == "nvidia/moonshotai/kimi-k2.5" + class TestRoutesByDefault: """The deterministic launch must boot claude on the *router*, not strong. diff --git a/tests/test_launch_codex.py b/tests/test_launch_codex.py index 1c3ab32b..40449dd3 100644 --- a/tests/test_launch_codex.py +++ b/tests/test_launch_codex.py @@ -506,9 +506,12 @@ def test_no_flags_dispatches_to_deterministic(self, monkeypatch, tmp_path): ) monkeypatch.setenv("SWITCHYARD_CONFIG_DIR", str(tmp_path)) + # OpenRouter endpoint: the zero-flag default trio is OpenRouter-only, so + # the deterministic default only dispatches when the resolved endpoint is + # OpenRouter (a non-OpenRouter provider is rejected — see NvBug 6401771). monkeypatch.setattr( "switchyard.cli.launch_command.resolve_launch_connectivity", - lambda args, **_kw: ("sk-test", "https://inference-api.nvidia.com/v1"), + lambda args, **_kw: ("sk-test", "https://openrouter.ai/api/v1"), ) captured: dict = {} From aad16ca23b7bc9bf058e2845ac2a21e92babc7bc Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:33 -0700 Subject: [PATCH 7/9] fix(cli): bypass env proxies for loopback readiness probes Signed-off-by: Elyas Mehtabuddin --- switchyard/cli/launchers/launcher_runtime.py | 12 ++++-- .../cli/launchers/proxy_health_monitor.py | 9 +++-- switchyard/server/verify.py | 4 +- tests/test_launcher_proxy_bypass.py | 39 +++++++++++++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 tests/test_launcher_proxy_bypass.py diff --git a/switchyard/cli/launchers/launcher_runtime.py b/switchyard/cli/launchers/launcher_runtime.py index 333d424c..e9cccccd 100644 --- a/switchyard/cli/launchers/launcher_runtime.py +++ b/switchyard/cli/launchers/launcher_runtime.py @@ -3,8 +3,6 @@ """Shared proxy/runtime helpers for one-command launchers.""" -from __future__ import annotations - import logging import os import socket @@ -27,6 +25,10 @@ _debug_file_handler: logging.FileHandler | None = None log = logging.getLogger(__name__) +#: Opener that ignores env proxies — loopback probes to the in-process proxy +#: must never be routed through a configured HTTP_PROXY. +_LOCAL_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + #: System CA bundle paths to try (Debian/Ubuntu, RHEL/CentOS/Fedora). _SYSTEM_CA_BUNDLE_CANDIDATES = ( @@ -75,7 +77,9 @@ def wait_for_proxy_ready(port: int, *, timeout_s: float) -> bool: deadline = time.monotonic() + timeout_s while time.monotonic() < deadline: try: - with urllib.request.urlopen(url, timeout=0.5): + # Loopback: bypass env proxies so a configured HTTP_PROXY can't + # intercept the 127.0.0.1 health probe. + with _LOCAL_OPENER.open(url, timeout=0.5): return True except Exception: time.sleep(0.05) @@ -283,7 +287,7 @@ def _classifier_part(r: _Mapping) -> str: # type: ignore[type-arg] return f"{route_type}: {route_key}" -def deterministic_strategy_summary(config: DeterministicRoutingConfig) -> str: +def deterministic_strategy_summary(config: "DeterministicRoutingConfig") -> str: """Return the strategy summary string for a deterministic (LLM-classifier) launch.""" return ( f"llm-classifier: classifier={config.classifier.model}, " diff --git a/switchyard/cli/launchers/proxy_health_monitor.py b/switchyard/cli/launchers/proxy_health_monitor.py index 92f8cde7..b5a12933 100644 --- a/switchyard/cli/launchers/proxy_health_monitor.py +++ b/switchyard/cli/launchers/proxy_health_monitor.py @@ -3,10 +3,9 @@ """Lightweight health indicator for launcher footers.""" -from __future__ import annotations - import time -import urllib.request + +from switchyard.cli.launchers.launcher_runtime import _LOCAL_OPENER class ProxyHealthMonitor: @@ -31,7 +30,9 @@ def poll(self) -> None: return self._last_check = now try: - with urllib.request.urlopen(self._url, timeout=0.5): + # Loopback: bypass env proxies so a configured HTTP_PROXY can't + # intercept the 127.0.0.1 health probe. + with _LOCAL_OPENER.open(self._url, timeout=0.5): self._healthy = True except Exception: self._healthy = False diff --git a/switchyard/server/verify.py b/switchyard/server/verify.py index d82b6e20..bb08dd65 100644 --- a/switchyard/server/verify.py +++ b/switchyard/server/verify.py @@ -486,7 +486,9 @@ def _proxy_roundtrip(port: int, model: str) -> str: "max_tokens": 2048, } start = time.monotonic() - with httpx.Client(timeout=_ROUNDTRIP_TIMEOUT_S) as client: + # Loopback probe to our own in-process proxy — bypass env proxies so a + # configured HTTP_PROXY doesn't intercept the 127.0.0.1 request. + with httpx.Client(timeout=_ROUNDTRIP_TIMEOUT_S, trust_env=False) as client: resp = client.post(url, json=body) elapsed_ms = (time.monotonic() - start) * 1000.0 if resp.status_code != 200: diff --git a/tests/test_launcher_proxy_bypass.py b/tests/test_launcher_proxy_bypass.py new file mode 100644 index 00000000..bb553ee8 --- /dev/null +++ b/tests/test_launcher_proxy_bypass.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Loopback readiness probes must ignore env proxies (HTTP_PROXY/NO_PROXY).""" + +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from switchyard.cli.launchers.launcher_runtime import wait_for_proxy_ready + + +class _HealthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + +def test_wait_for_proxy_ready_bypasses_env_proxy(monkeypatch): + """A configured HTTP_PROXY must not intercept the 127.0.0.1 health probe.""" + server = HTTPServer(("127.0.0.1", 0), _HealthHandler) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + # Point env proxy at a dead port and remove any NO_PROXY exemption; before + # the fix the loopback probe would route here and fail. + monkeypatch.setenv("HTTP_PROXY", "http://127.0.0.1:9") + monkeypatch.setenv("http_proxy", "http://127.0.0.1:9") + monkeypatch.delenv("NO_PROXY", raising=False) + monkeypatch.delenv("no_proxy", raising=False) + + try: + assert wait_for_proxy_ready(port, timeout_s=2) is True + finally: + server.shutdown() + server.server_close() From 1a0c5cff76b9dfc068ae93845664557783775715 Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:33 -0700 Subject: [PATCH 8/9] fix(stats): record noop routes in /v1/routing/stats Signed-off-by: Elyas Mehtabuddin --- switchyard/lib/profiles/noop.py | 6 ++++++ tests/test_noop_llm_backend.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/switchyard/lib/profiles/noop.py b/switchyard/lib/profiles/noop.py index 70ffc142..50ee698a 100644 --- a/switchyard/lib/profiles/noop.py +++ b/switchyard/lib/profiles/noop.py @@ -118,11 +118,13 @@ def __init__( *, request_processors: tuple[Any, ...] = (), response_processors: tuple[Any, ...] = (), + stats_accumulator: StatsAccumulator | None = None, ) -> None: """Create a no-op profile with a local translation helper.""" self._translation = TranslationEngine() self._request_processors = request_processors self._response_processors = response_processors + self._stats_accumulator = stats_accumulator def iter_components(self) -> list[object]: """Return lifecycle components in startup order.""" @@ -162,6 +164,7 @@ def with_runtime_components( return NoopProfile( request_processors=tuple(request_chain), response_processors=tuple(response_chain), + stats_accumulator=stats if enable_stats else None, ) async def process(self, input: ProfileInput) -> NoopProcessedRequest: @@ -217,6 +220,9 @@ async def run_with_context( ) -> ChatResponse: """Execute no-op response generation with an existing context.""" processed = await self.process_with_context(input, ctx) + ctx.selected_model = _NOOP_MODEL + if self._stats_accumulator is not None: + await self._stats_accumulator.record_success(_NOOP_MODEL, None, None) openai_request = self._translation.request_to_any_of( processed.request, _NOOP_SUPPORTED_TYPES, diff --git a/tests/test_noop_llm_backend.py b/tests/test_noop_llm_backend.py index 69521812..ab839f32 100644 --- a/tests/test_noop_llm_backend.py +++ b/tests/test_noop_llm_backend.py @@ -4,6 +4,7 @@ """Unit tests for the no-op profile.""" from switchyard.lib.profiles import NoopProfile, NoopProfileConfig, ProfileSwitchyard +from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest, ChatResponseType, response_type_matches from switchyard_rust.profiles import ProfileInput @@ -106,3 +107,18 @@ async def test_profile_process_and_rprocess_are_explicit_hooks(self) -> None: assert processed.request.model == "noop" assert processed_response is response + + async def test_noop_call_reports_routing_stats(self) -> None: + """The no-op profile self-reports its call and model to the stats accumulator.""" + stats = StatsAccumulator() + profile = NoopProfileConfig().build().with_runtime_components( + stats_accumulator=stats + ) + + await ProfileSwitchyard(profile).call(_openai_request(stream=False)) + + snap = stats.snapshot_sync() + assert snap["total_requests"] == 1 + assert "noop" in snap["models"] + assert snap["models"]["noop"]["calls"] == 1 + assert "" not in snap["models"] From 72f448e710343142c0c5d62d2f2664ba462d5a9d Mon Sep 17 00:00:00 2001 From: Elyas Mehtabuddin Date: Tue, 21 Jul 2026 11:53:33 -0700 Subject: [PATCH 9/9] docs: correct AUTO probe order, extras, saved-bundle note, and stale API references Signed-off-by: Elyas Mehtabuddin --- CHANGELOG.md | 7 ++++--- DEVELOPMENT.md | 2 +- INSTALLATION.md | 8 ++++---- README.md | 2 +- docs/cli_reference.md | 14 +++++++++++--- examples/minimal.py | 10 +++++----- examples/route.yaml | 2 ++ 7 files changed, 28 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07187269..b7e56952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ traffic that sits between client applications and LLM backends. - **Observability** — Prometheus `/metrics`, a JSON `/v1/stats` (`/v1/routing/stats` alias), and per-request cost/token/latency stats. See [Metrics Reference](docs/METRICS_REFERENCE.md). -- **Python library** — `SwitchyardRecipes` (`passthrough_recipe`, - `random_routing_recipe`, `stage_router_recipe`, `deterministic_routing_recipe`, - …) and typed `ChatRequest` / `ChatResponse` containers for in-process use. +- **Python library** — `ProfileSwitchyard` driven by typed profile configs + (`PassthroughProfileConfig`, `RandomRoutingProfileConfig`, + `StageRouterProfileConfig`, …) and typed `ChatRequest` / `ChatResponse` + containers for in-process use. - **Rust core** (PyO3) — chain execution, the latency-aware router, and the tool-result signal collector are implemented in Rust and re-exported to Python. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 12b1245f..7c92bb70 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -46,7 +46,7 @@ switchyard/ │ ├── lib/ # Core library │ │ ├── roles.py # RequestProcessor, LLMBackend, ResponseProcessor ABCs │ │ ├── switchyard.py # Switchyard chain executor -│ │ ├── recipes.py # SwitchyardRecipes (passthrough, random_routing, …) +│ │ ├── profiles/ # Profile configs/runtimes (passthrough, random_routing, …) │ │ ├── proxy_context.py # ProxyContext — per-request state │ │ ├── chat_request/ # Typed request hierarchy │ │ ├── chat_response/ # Typed response hierarchy diff --git a/INSTALLATION.md b/INSTALLATION.md index 270c450a..a6f03e42 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -67,7 +67,7 @@ All optional dependencies for complete feature set: pip install nemo-switchyard[all] ``` -Equivalent to: `switchyard[server,cli]` +Equivalent to: `nemo-switchyard[server,cli,tracing,intake,affinity-redis]` ### Common Combinations @@ -114,13 +114,13 @@ pip install nemo-switchyard[all] Embed Switchyard with minimal overhead: ```python -from switchyard import SwitchyardRecipes +from switchyard import PassthroughProfileConfig, ProfileSwitchyard # Core library only — no server/CLI dependencies -switchyard = SwitchyardRecipes.passthrough_recipe( +switchyard = ProfileSwitchyard(PassthroughProfileConfig( api_key="sk-...", base_url="https://api.openai.com/v1", -) +).build()) ``` ## Troubleshooting diff --git a/README.md b/README.md index 40a38911..6efdd447 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Optional extras: ```bash pip install "nemo-switchyard[server]" # FastAPI / Uvicorn HTTP endpoints pip install "nemo-switchyard[cli]" # Interactive CLI launchers (Claude / Codex) -pip install "nemo-switchyard[all]" # Server, CLI, and tracing extras +pip install "nemo-switchyard[all]" # Server, CLI, tracing, intake, and redis-affinity extras ``` See [Installation](INSTALLATION.md) for a full breakdown of what each extra adds. diff --git a/docs/cli_reference.md b/docs/cli_reference.md index c9b47cda..af3baf1d 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -50,9 +50,10 @@ upstream requests. Configuration files use these lowercase values: `auto` resolves formats in this order: -1. Probe `/v1/messages`; use `anthropic` when supported. -2. Probe `/v1/responses`; use `responses` when supported. -3. Fall back to `openai` and `/v1/chat/completions`. +1. Probe `/v1/chat/completions`; use `openai` when supported. +2. Probe `/v1/messages`; use `anthropic` when supported. +3. Probe `/v1/responses`; use `responses` when supported. +4. Fall back to `openai` (`/v1/chat/completions`). Single-model Claude Code and Codex launches use `auto`. OpenClaw is pinned to `openai`. Prefer an explicit format when the upstream contract is known so @@ -454,6 +455,13 @@ switchyard [--routing-profiles PATH] configure [--show [--check] [--json] | --re | `--no-tui` | Use plain text prompts instead of the TUI selector. | | `--check` | With `--show`, call `GET /models` against the resolved provider and report pass/fail in the output. | +> **Saved bundles keep `${VAR}` references literal.** A saved routing-profile +> bundle stores `${OPENROUTER_API_KEY}` (and any other `${VAR}`) verbatim — +> secrets are never baked in. The referenced environment variables must therefore +> be present in the environment at `serve` / `launch` time; on another machine or +> shell, export them again or Switchyard aborts with +> `missing environment variable(s): NAME`. + **Skill distillation config** ```json diff --git a/examples/minimal.py b/examples/minimal.py index a78abe05..26555b07 100644 --- a/examples/minimal.py +++ b/examples/minimal.py @@ -25,7 +25,7 @@ # Add package to path for development (not needed when installed via pip) sys.path.insert(0, str(Path(__file__).parent.parent)) -from switchyard import ChatRequest, SwitchyardRecipes +from switchyard import ChatRequest, PassthroughProfileConfig, ProfileSwitchyard async def main(): @@ -39,10 +39,10 @@ async def main(): return # Create a passthrough proxy that forwards to OpenAI - switchyard = SwitchyardRecipes.passthrough_recipe( + switchyard = ProfileSwitchyard(PassthroughProfileConfig( api_key=api_key, base_url="https://api.openai.com/v1", - ) + ).build()) print("=" * 60) print("Switchyard Minimal Example") @@ -64,8 +64,8 @@ async def main(): response = await switchyard.call(request) print("\nResponse:") - print(f" Content: {response.body['choices'][0]['message']['content']}") - print(f" Tokens: {response.body['usage']['total_tokens']}") + print(f" Content: {response['choices'][0]['message']['content']}") + print(f" Tokens: {response['usage']['total_tokens']}") print("\n" + "=" * 60) print("Example completed!") diff --git a/examples/route.yaml b/examples/route.yaml index 952f4190..9575ab5e 100644 --- a/examples/route.yaml +++ b/examples/route.yaml @@ -24,6 +24,8 @@ routes: llm-classifier: # `type: deterministic` — LLM-as-classifier type: deterministic profile: coding_agent # general | coding_agent | openclaw + fallback_target_on_evict: strong # tier to retry on context-window evict + classifier: model: google/gemini-3.5-flash api_key: ${OPENROUTER_API_KEY}