From 9089c49bc7760fe27fb90691af51a997582ddfd1 Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 16:11:41 +0100 Subject: [PATCH 1/2] fix(elevenlabs): degrade gracefully when /with-timestamps is rejected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The with-timestamps endpoint variant was appended unconditionally whenever boundaries were requested — its model coverage is not documented for eleven_v3, so a boundary-requesting call on v3 could hard-fail. Now a rejected variant retries the plain synthesis endpoint and flows into the streaming path with estimated boundaries (real failures — auth, quota — still surface from the retry). Also: the estimation plan is built from the caller-facing text, not the processed prompt, so injected audio tags are never estimated as words; and a boundaries-only degraded request now streams (previously the streaming branch required an on_audio callback, which would have delivered no boundaries at all). --- src/cloud_engine.rs | 230 ++++++++++++++++++++--------------- tests/live_cloud.rs.template | 9 +- 2 files changed, 141 insertions(+), 98 deletions(-) diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 4b36f62..677bd19 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -2315,105 +2315,131 @@ impl TtsEngine for CloudEngine { return Ok(()); } - let mut synth_url = self.config.synth_url.clone(); - if self.config.provider_id == "elevenlabs" && on_boundary.is_some() { - synth_url.push_str("/with-timestamps"); - } - let mut req = self.client.post(&synth_url); - - // Auth header - if !self.config.auth_header.is_empty() { - let val = format!("{}{}", self.config.auth_prefix, self.api_key); - req = req.header(&self.config.auth_header, val); - } + // ElevenLabs word timing comes from the /with-timestamps endpoint + // variant. Model support for that variant varies (not documented + // for eleven_v3) — if it rejects the request we degrade to a + // plain synthesis and estimated boundaries below rather than + // failing every boundary-requesting call on that model. + let wants_timestamps = self.config.provider_id == "elevenlabs" && on_boundary.is_some(); + + // Build and send the synthesis request. A closure so the + // with-timestamps attempt can be retried without the suffix. + let send_synthesis = |synth_url: &str| -> Result { + let mut req = self.client.post(synth_url); + + // Auth header + if !self.config.auth_header.is_empty() { + let val = format!("{}{}", self.config.auth_prefix, self.api_key); + req = req.header(&self.config.auth_header, val); + } - // Extra headers - for (k, v) in &self.config.extra_headers { - req = req.header(k.as_str(), v.as_str()); - } + // Extra headers + for (k, v) in &self.config.extra_headers { + req = req.header(k.as_str(), v.as_str()); + } - // Body depends on engine type - let resp = if self.config.body_is_ssml { - // Azure: send SSML XML body. When is_ssml=true, the text is - // already SSML — send it directly (don't escape/wrap with - // build_azure_ssml). Inject voice if the SSML lacks a - // tag, after completing the envelope and dropping unsupported - // elements (Azure accepts its documented - // here, so bookmarks are kept on this path). - let ssml = if is_ssml { - inject_voice_if_missing( - &strip_unsupported_marks(&normalize_ssml_envelope(&text, &voice_to_use), false), + // Body depends on engine type + let resp = if self.config.body_is_ssml { + // Azure: send SSML XML body. When is_ssml=true, the text is + // already SSML — send it directly (don't escape/wrap with + // build_azure_ssml). Inject voice if the SSML lacks a + // tag, after completing the envelope and dropping unsupported + // elements (Azure accepts its documented + // here, so bookmarks are kept on this path). + let ssml = if is_ssml { + inject_voice_if_missing( + &strip_unsupported_marks( + &normalize_ssml_envelope(&text, &voice_to_use), + false, + ), + &voice_to_use, + ) + } else { + build_azure_ssml(&text, &voice_to_use, rate, pitch, volume) + }; + let ct = self + .config + .content_type + .as_deref() + .unwrap_or("application/ssml+xml"); + req = req.header("Content-Type", ct); + req.body(ssml).send() + } else if self.config.provider_id == "google" { + // Google: build JSON body with proper structure + let (body, _words) = build_google_request( + &text, &voice_to_use, - ) - } else { - build_azure_ssml(&text, &voice_to_use, rate, pitch, volume) - }; - let ct = self - .config - .content_type - .as_deref() - .unwrap_or("application/ssml+xml"); - req = req.header("Content-Type", ct); - req.body(ssml).send() - } else if self.config.provider_id == "google" { - // Google: build JSON body with proper structure - let (body, _words) = build_google_request( - &text, - &voice_to_use, - on_boundary.is_some(), - google_ssml_override.as_deref(), - ); - req = req.json(&body); - req.send() - } else { - // Standard JSON body for all other engines - let mut body = serde_json::Map::new(); - if !self.config.text_field.is_empty() { - body.insert( - self.config.text_field.clone(), - serde_json::Value::String(text.clone()), - ); - } - if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() { - body.insert( - self.config.voice_param.clone(), - serde_json::Value::String(voice_to_use.clone()), + on_boundary.is_some(), + google_ssml_override.as_deref(), ); - } - if let Some(ref model_param) = self.config.model_param { - if let Some(ref model) = self.config.model_default { + req = req.json(&body); + req.send() + } else { + // Standard JSON body for all other engines + let mut body = serde_json::Map::new(); + if !self.config.text_field.is_empty() { body.insert( - model_param.clone(), - serde_json::Value::String(model.clone()), + self.config.text_field.clone(), + serde_json::Value::String(text.clone()), ); } - } - // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal) - // onto the deterministic voice_settings.speed API parameter - // (valid range 0.7–1.2; clamped). Only sent for explicit - // non-default rates. pitch/volume have no API equivalent - // (v3 models: use audio tags). Inserted before extra_body so - // a config-supplied voice_settings object (stability, - // similarity, …) takes precedence over the derived one. - if self.config.provider_id == "elevenlabs" - && rate > 0.0 - && (rate - 1.0).abs() > f32::EPSILON - && !self.config.extra_body.contains_key("voice_settings") - { - let speed = rate.clamp(0.7, 1.2); - body.insert( - "voice_settings".to_string(), - serde_json::json!({ "speed": speed }), - ); - } - for (k, v) in &self.config.extra_body { - body.insert(k.clone(), v.clone()); - } - req = req.json(&serde_json::Value::Object(body)); - req.send() + if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() { + body.insert( + self.config.voice_param.clone(), + serde_json::Value::String(voice_to_use.clone()), + ); + } + if let Some(ref model_param) = self.config.model_param { + if let Some(ref model) = self.config.model_default { + body.insert( + model_param.clone(), + serde_json::Value::String(model.clone()), + ); + } + } + // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal) + // onto the deterministic voice_settings.speed API parameter + // (valid range 0.7–1.2; clamped). Only sent for explicit + // non-default rates. pitch/volume have no API equivalent + // (v3 models: use audio tags). Inserted before extra_body so + // a config-supplied voice_settings object (stability, + // similarity, …) takes precedence over the derived one. + if self.config.provider_id == "elevenlabs" + && rate > 0.0 + && (rate - 1.0).abs() > f32::EPSILON + && !self.config.extra_body.contains_key("voice_settings") + { + let speed = rate.clamp(0.7, 1.2); + body.insert( + "voice_settings".to_string(), + serde_json::json!({ "speed": speed }), + ); + } + for (k, v) in &self.config.extra_body { + body.insert(k.clone(), v.clone()); + } + req = req.json(&serde_json::Value::Object(body)); + req.send() + }; + resp.map_err(|e| TtsError(format!("HTTP error: {e}"))) }; - let resp = resp.map_err(|e| TtsError(format!("HTTP error: {e}")))?; + let mut synth_url = self.config.synth_url.clone(); + if wants_timestamps { + synth_url.push_str("/with-timestamps"); + } + let mut resp = send_synthesis(&synth_url)?; + + let mut timestamps_degraded = false; + if wants_timestamps && !resp.status().is_success() { + // The /with-timestamps variant was rejected (likely a model + // that doesn't support it). Drop the suffix and fall back to + // streamed audio + estimated boundaries. A genuine failure + // (auth, quota, bad voice) fails the retry too and surfaces + // its error there. + timestamps_degraded = true; + resp = send_synthesis(&self.config.synth_url)?; + } if !resp.status().is_success() { let status = resp.status(); @@ -2427,7 +2453,11 @@ impl TtsEngine for CloudEngine { // silent success. let mut audio_total = 0usize; - if self.config.provider_id == "elevenlabs" && on_boundary.is_some() { + // The with-timestamps response is one JSON document (base64 audio + // + character alignment). When the variant was rejected above, the + // retry returned a plain streamed body — handled by the streaming + // branch below with estimated boundaries. + if wants_timestamps && !timestamps_degraded { let resp_text = resp .text() .map_err(|e| TtsError(format!("Read error: {e}")))?; @@ -2534,7 +2564,7 @@ impl TtsEngine for CloudEngine { } } } - } else if let Some(cb) = on_audio.as_mut() { + } else if on_audio.is_some() || (timestamps_degraded && on_boundary.is_some()) { // Most providers respond with an MP3 body (OpenAI, ElevenLabs, // Deepgram, Watson, …); a few return raw PCM natively (Azure via // X-Microsoft-OutputFormat, Cartesia). Stream the body as it @@ -2542,12 +2572,22 @@ impl TtsEngine for CloudEngine { // reaches on_audio before the response completes. // // Estimated word boundaries fire progressively, anchored to - // delivered audio, instead of all-at-once afterwards. - let plan = on_boundary.is_some().then(|| EstimatePlan::build(&text)); + // delivered audio, instead of all-at-once afterwards. The plan + // is built from the caller-facing text: for ElevenLabs + // SpeechMarkdown (and the /with-timestamps fallback), the + // processed prompt carries injected tags that must not become + // estimated "words". Also entered for a boundaries-only + // request when the timestamps variant degraded — otherwise + // those callers would get audio but no boundaries at all. + let plan = on_boundary + .is_some() + .then(|| EstimatePlan::build(boundary_search_text)); let mut on_event = |ev: StreamEvt<'_>| match ev { StreamEvt::Audio(bytes) => { audio_total += bytes.len(); - cb(bytes); + if let Some(cb) = on_audio.as_mut() { + cb(bytes); + } } StreamEvt::Boundary(word, start, end, offset, len) => { if let Some(bcb) = on_boundary.as_mut() { diff --git a/tests/live_cloud.rs.template b/tests/live_cloud.rs.template index 516e6b1..a3b1301 100644 --- a/tests/live_cloud.rs.template +++ b/tests/live_cloud.rs.template @@ -331,9 +331,12 @@ fn elevenlabs_v3_speechmarkdown_audio_tags_synthesise() { #[test] #[ignore] fn elevenlabs_v3_timestamps_with_audio_tags() { - // Word alignment through /with-timestamps while the text carries - // injected audio tags: boundary offsets must still resolve against - // the caller's input (words found, offsets >= 0). + // Word alignment while the text carries injected audio tags. If the + // model supports /with-timestamps, offsets come from the API's + // character alignment; if it rejects the variant, speak() degrades + // to streamed audio + estimated boundaries — either way boundaries + // must fire and offsets must resolve against the caller's input. + // (Which path ran is visible in the word count vs. text length.) let key = std::env::var("ELEVENLABS_API_KEY").unwrap_or_default(); if key.is_empty() { eprintln!("skipping: ELEVENLABS_API_KEY not set"); From d89a894dc083e88d3c565f5d83c70fe01e997d4b Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 16:29:44 +0100 Subject: [PATCH 2/2] test: offline mock-server coverage for the timestamps fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Std-TcpListener mock serving a 404 on /with-timestamps then a 200 MP3 (0.4s silence fixture, ffmpeg-generated): asserts the variant is attempted first, dropped on retry, and estimated boundaries fire with offsets resolving in the caller's text — the boundaries-only degraded entry included. No API key needed. --- src/cloud_engine.rs | 12 +-- tests/elevenlabs_timestamps_fallback.rs | 123 ++++++++++++++++++++++++ tests/fixtures/silence.mp3 | Bin 0 -> 1994 bytes 3 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 tests/elevenlabs_timestamps_fallback.rs create mode 100644 tests/fixtures/silence.mp3 diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 677bd19..6dd9694 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -2573,12 +2573,12 @@ impl TtsEngine for CloudEngine { // // Estimated word boundaries fire progressively, anchored to // delivered audio, instead of all-at-once afterwards. The plan - // is built from the caller-facing text: for ElevenLabs - // SpeechMarkdown (and the /with-timestamps fallback), the - // processed prompt carries injected tags that must not become - // estimated "words". Also entered for a boundaries-only - // request when the timestamps variant degraded — otherwise - // those callers would get audio but no boundaries at all. + // is built from the caller-facing text so formatter-injected + // tags are not estimated as words (user-authored markup in + // the SpeechMarkdown source still is — the estimator has no + // markup filter). Also entered for a boundaries-only request + // when the timestamps variant degraded — otherwise those + // callers would get audio but no boundaries at all. let plan = on_boundary .is_some() .then(|| EstimatePlan::build(boundary_search_text)); diff --git a/tests/elevenlabs_timestamps_fallback.rs b/tests/elevenlabs_timestamps_fallback.rs new file mode 100644 index 0000000..5fc58c2 --- /dev/null +++ b/tests/elevenlabs_timestamps_fallback.rs @@ -0,0 +1,123 @@ +//! Offline test for the ElevenLabs `/with-timestamps` degrade path: when +//! the endpoint variant is rejected (a model that doesn't support it), +//! speak() must retry the plain synthesis endpoint and deliver estimated +//! boundaries instead of failing the call. +//! +//! Uses a `std::net::TcpListener` mock so no network access or API key is +//! needed. The MP3 fixture is 0.4s of silence, regenerated with: +//! ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 0.4 -q:a 9 silence.mp3 + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; + +use rust_tts_wrapper::engine::TtsEngine; +use rust_tts_wrapper::factory::create_engine; + +const SILENCE_MP3: &[u8] = include_bytes!("fixtures/silence.mp3"); + +fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// Minimal HTTP/1.1 responder: drains exactly one request (headers plus +/// Content-Length body) and writes `status` with `body`. Returns the +/// request line so tests can assert on the path. +fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]) -> String { + let mut buf = vec![0u8; 16_384]; + let mut read = 0usize; + let header_end = loop { + let n = stream.read(&mut buf[read..]).expect("read request"); + assert!(n > 0, "client closed connection before sending a request"); + read += n; + if let Some(pos) = find_subsequence(&buf[..read], b"\r\n\r\n") { + break pos; + } + }; + let headers = String::from_utf8_lossy(&buf[..header_end]).to_string(); + let content_length: usize = headers + .lines() + .find(|l| l.to_ascii_lowercase().starts_with("content-length")) + .and_then(|l| l.split(':').nth(1)) + .map(|v| v.trim().parse().expect("content-length")) + .unwrap_or(0); + let mut received = read - header_end - 4; + while received < content_length { + let n = stream.read(&mut buf[read..]).expect("read body"); + read += n; + received += n; + } + + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream + .write_all(response.as_bytes()) + .expect("write headers"); + stream.write_all(body).expect("write body"); + stream.flush().expect("flush"); + headers.lines().next().unwrap_or_default().to_string() +} + +#[test] +fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let server = std::thread::spawn(move || { + let mut paths = Vec::new(); + // First attempt: the /with-timestamps variant is rejected. + let (mut stream, _) = listener.accept().expect("accept 1"); + paths.push(respond( + &mut stream, + "404 Not Found", + "application/json", + b"{\"detail\":{\"message\":\"not found\"}}", + )); + // Retry: plain synthesis endpoint with an MP3 body. + let (mut stream, _) = listener.accept().expect("accept 2"); + paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3)); + paths + }); + + let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine"); + + let mut words: Vec = Vec::new(); + let mut on_boundary = + |word: &str, _start: f32, _end: f32, offset: i32, _len: i32, _final: bool| { + assert!( + offset >= 0, + "estimated word {word:?} must resolve in caller text" + ); + words.push(word.to_string()); + }; + // Boundaries requested, no on_audio: exercises the degraded streaming + // entry that previously would have delivered no boundaries at all. + engine + .speak( + "Hello boundary fallback", + None, + 1.0, + 1.0, + 1.0, + None, + Some(&mut on_boundary), + None, + ) + .expect("degraded speak must succeed"); + + let paths = server.join().expect("server thread"); + assert!( + paths[0].contains("/with-timestamps"), + "first attempt must use the variant: {}", + paths[0] + ); + assert!( + !paths[1].contains("/with-timestamps"), + "retry must drop the variant: {}", + paths[1] + ); + assert_eq!(words, ["Hello", "boundary", "fallback"]); +} diff --git a/tests/fixtures/silence.mp3 b/tests/fixtures/silence.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..f8a759995480eca33e9b9dffab90a45b4a861402 GIT binary patch literal 1994 zcmeZtF=k-^0i}@OU{@f`$H2hslUSB!W~67VXJ}vmmV^-he>)sN;zF37d1?7T7C#UR zGBB{uV^9SFV=!<413xf`0fS61C3=Awjj=ruS?-?p+ zB&C)lDi~{qk^xXm0okM3jAjqnMx(d}!=7Mt|Io;u8anxB8J+xdo=*O`MkoJ_+Jll$ gaB(p5&kzqc6n}!s9Sr}VsGtsK)E@G}kzU~j0Du~8=Kufz literal 0 HcmV?d00001