ElevenLabs: degrade gracefully when /with-timestamps is rejected

src/cloud_engine.rs

@@ -2315,105 +2315,131 @@ impl TtsEngine for CloudEngine {
             return Ok(());
         }
 
-        let mut synth_url = self.config.synth_url.clone();
-        if self.config.provider_id == "elevenlabs" && on_boundary.is_some() {
-            synth_url.push_str("/with-timestamps");
-        }
-        let mut req = self.client.post(&synth_url);
-
-        // Auth header
-        if !self.config.auth_header.is_empty() {
-            let val = format!("{}{}", self.config.auth_prefix, self.api_key);
-            req = req.header(&self.config.auth_header, val);
-        }
+        // ElevenLabs word timing comes from the /with-timestamps endpoint
+        // variant. Model support for that variant varies (not documented
+        // for eleven_v3) — if it rejects the request we degrade to a
+        // plain synthesis and estimated boundaries below rather than
+        // failing every boundary-requesting call on that model.
+        let wants_timestamps = self.config.provider_id == "elevenlabs" && on_boundary.is_some();
+
+        // Build and send the synthesis request. A closure so the
+        // with-timestamps attempt can be retried without the suffix.
+        let send_synthesis = |synth_url: &str| -> Result<reqwest::blocking::Response, TtsError> {
+            let mut req = self.client.post(synth_url);
+
+            // Auth header
+            if !self.config.auth_header.is_empty() {
+                let val = format!("{}{}", self.config.auth_prefix, self.api_key);
+                req = req.header(&self.config.auth_header, val);
+            }
 
-        // Extra headers
-        for (k, v) in &self.config.extra_headers {
-            req = req.header(k.as_str(), v.as_str());
-        }
+            // Extra headers
+            for (k, v) in &self.config.extra_headers {
+                req = req.header(k.as_str(), v.as_str());
+            }
 
-        // Body depends on engine type
-        let resp = if self.config.body_is_ssml {
-            // Azure: send SSML XML body. When is_ssml=true, the text is
-            // already SSML — send it directly (don't escape/wrap with
-            // build_azure_ssml). Inject voice if the SSML lacks a <voice>
-            // tag, after completing the envelope and dropping unsupported
-            // <mark> elements (Azure accepts its documented <bookmark>
-            // here, so bookmarks are kept on this path).
-            let ssml = if is_ssml {
-                inject_voice_if_missing(
-                    &strip_unsupported_marks(&normalize_ssml_envelope(&text, &voice_to_use), false),
+            // Body depends on engine type
+            let resp = if self.config.body_is_ssml {
+                // Azure: send SSML XML body. When is_ssml=true, the text is
+                // already SSML — send it directly (don't escape/wrap with
+                // build_azure_ssml). Inject voice if the SSML lacks a <voice>
+                // tag, after completing the envelope and dropping unsupported
+                // <mark> elements (Azure accepts its documented <bookmark>
+                // here, so bookmarks are kept on this path).
+                let ssml = if is_ssml {
+                    inject_voice_if_missing(
+                        &strip_unsupported_marks(
+                            &normalize_ssml_envelope(&text, &voice_to_use),
+                            false,
+                        ),
+                        &voice_to_use,
+                    )
+                } else {
+                    build_azure_ssml(&text, &voice_to_use, rate, pitch, volume)
+                };
+                let ct = self
+                    .config
+                    .content_type
+                    .as_deref()
+                    .unwrap_or("application/ssml+xml");
+                req = req.header("Content-Type", ct);
+                req.body(ssml).send()
+            } else if self.config.provider_id == "google" {
+                // Google: build JSON body with proper structure
+                let (body, _words) = build_google_request(
+                    &text,
                     &voice_to_use,
-                )
-            } else {
-                build_azure_ssml(&text, &voice_to_use, rate, pitch, volume)
-            };
-            let ct = self
-                .config
-                .content_type
-                .as_deref()
-                .unwrap_or("application/ssml+xml");
-            req = req.header("Content-Type", ct);
-            req.body(ssml).send()
-        } else if self.config.provider_id == "google" {
-            // Google: build JSON body with proper structure
-            let (body, _words) = build_google_request(
-                &text,
-                &voice_to_use,
-                on_boundary.is_some(),
-                google_ssml_override.as_deref(),
-            );
-            req = req.json(&body);
-            req.send()
-        } else {
-            // Standard JSON body for all other engines
-            let mut body = serde_json::Map::new();
-            if !self.config.text_field.is_empty() {
-                body.insert(
-                    self.config.text_field.clone(),
-                    serde_json::Value::String(text.clone()),
-                );
-            }
-            if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() {
-                body.insert(
-                    self.config.voice_param.clone(),
-                    serde_json::Value::String(voice_to_use.clone()),
+                    on_boundary.is_some(),
+                    google_ssml_override.as_deref(),
                 );
-            }
-            if let Some(ref model_param) = self.config.model_param {
-                if let Some(ref model) = self.config.model_default {
+                req = req.json(&body);
+                req.send()
+            } else {
+                // Standard JSON body for all other engines
+                let mut body = serde_json::Map::new();
+                if !self.config.text_field.is_empty() {
                     body.insert(
-                        model_param.clone(),
-                        serde_json::Value::String(model.clone()),
+                        self.config.text_field.clone(),
+                        serde_json::Value::String(text.clone()),
                     );
                 }
-            }
-            // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal)
-            // onto the deterministic voice_settings.speed API parameter
-            // (valid range 0.7–1.2; clamped). Only sent for explicit
-            // non-default rates. pitch/volume have no API equivalent
-            // (v3 models: use audio tags). Inserted before extra_body so
-            // a config-supplied voice_settings object (stability,
-            // similarity, …) takes precedence over the derived one.
-            if self.config.provider_id == "elevenlabs"
-                && rate > 0.0
-                && (rate - 1.0).abs() > f32::EPSILON
-                && !self.config.extra_body.contains_key("voice_settings")
-            {
-                let speed = rate.clamp(0.7, 1.2);
-                body.insert(
-                    "voice_settings".to_string(),
-                    serde_json::json!({ "speed": speed }),
-                );
-            }
-            for (k, v) in &self.config.extra_body {
-                body.insert(k.clone(), v.clone());
-            }
-            req = req.json(&serde_json::Value::Object(body));
-            req.send()
+                if !self.config.voice_param.is_empty() && !voice_to_use.is_empty() {
+                    body.insert(
+                        self.config.voice_param.clone(),
+                        serde_json::Value::String(voice_to_use.clone()),
+                    );
+                }
+                if let Some(ref model_param) = self.config.model_param {
+                    if let Some(ref model) = self.config.model_default {
+                        body.insert(
+                            model_param.clone(),
+                            serde_json::Value::String(model.clone()),
+                        );
+                    }
+                }
+                // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal)
+                // onto the deterministic voice_settings.speed API parameter
+                // (valid range 0.7–1.2; clamped). Only sent for explicit
+                // non-default rates. pitch/volume have no API equivalent
+                // (v3 models: use audio tags). Inserted before extra_body so
+                // a config-supplied voice_settings object (stability,
+                // similarity, …) takes precedence over the derived one.
+                if self.config.provider_id == "elevenlabs"
+                    && rate > 0.0
+                    && (rate - 1.0).abs() > f32::EPSILON
+                    && !self.config.extra_body.contains_key("voice_settings")
+                {
+                    let speed = rate.clamp(0.7, 1.2);
+                    body.insert(
+                        "voice_settings".to_string(),
+                        serde_json::json!({ "speed": speed }),
+                    );
+                }
+                for (k, v) in &self.config.extra_body {
+                    body.insert(k.clone(), v.clone());
+                }
+                req = req.json(&serde_json::Value::Object(body));
+                req.send()
+            };
+            resp.map_err(|e| TtsError(format!("HTTP error: {e}")))
         };
 
-        let resp = resp.map_err(|e| TtsError(format!("HTTP error: {e}")))?;
+        let mut synth_url = self.config.synth_url.clone();
+        if wants_timestamps {
+            synth_url.push_str("/with-timestamps");
+        }
+        let mut resp = send_synthesis(&synth_url)?;
+
+        let mut timestamps_degraded = false;
+        if wants_timestamps && !resp.status().is_success() {
+            // The /with-timestamps variant was rejected (likely a model
+            // that doesn't support it). Drop the suffix and fall back to
+            // streamed audio + estimated boundaries. A genuine failure
+            // (auth, quota, bad voice) fails the retry too and surfaces
+            // its error there.
+            timestamps_degraded = true;
+            resp = send_synthesis(&self.config.synth_url)?;
+        }
 
         if !resp.status().is_success() {
             let status = resp.status();
@@ -2427,7 +2453,11 @@ impl TtsEngine for CloudEngine {
         // silent success.
         let mut audio_total = 0usize;
 
-        if self.config.provider_id == "elevenlabs" && on_boundary.is_some() {
+        // The with-timestamps response is one JSON document (base64 audio
+        // + character alignment). When the variant was rejected above, the
+        // retry returned a plain streamed body — handled by the streaming
+        // branch below with estimated boundaries.
+        if wants_timestamps && !timestamps_degraded {
             let resp_text = resp
                 .text()
                 .map_err(|e| TtsError(format!("Read error: {e}")))?;
@@ -2534,20 +2564,30 @@ impl TtsEngine for CloudEngine {
                     }
                 }
             }
-        } else if let Some(cb) = on_audio.as_mut() {
+        } else if on_audio.is_some() || (timestamps_degraded && on_boundary.is_some()) {
             // Most providers respond with an MP3 body (OpenAI, ElevenLabs,
             // Deepgram, Watson, …); a few return raw PCM natively (Azure via
             // X-Microsoft-OutputFormat, Cartesia). Stream the body as it
             // downloads, decoding MP3 → PCM16 mono incrementally so audio
             // reaches on_audio before the response completes.
             //
             // Estimated word boundaries fire progressively, anchored to
-            // delivered audio, instead of all-at-once afterwards.
-            let plan = on_boundary.is_some().then(|| EstimatePlan::build(&text));
+            // delivered audio, instead of all-at-once afterwards. The plan
+            // is built from the caller-facing text so formatter-injected
+            // tags are not estimated as words (user-authored markup in
+            // the SpeechMarkdown source still is — the estimator has no
+            // markup filter). Also entered for a boundaries-only request
+            // when the timestamps variant degraded — otherwise those
+            // callers would get audio but no boundaries at all.
+            let plan = on_boundary
+                .is_some()
+                .then(|| EstimatePlan::build(boundary_search_text));
             let mut on_event = |ev: StreamEvt<'_>| match ev {
                 StreamEvt::Audio(bytes) => {
                     audio_total += bytes.len();
-                    cb(bytes);
+                    if let Some(cb) = on_audio.as_mut() {
+                        cb(bytes);
+                    }
                 }
                 StreamEvt::Boundary(word, start, end, offset, len) => {
                     if let Some(bcb) = on_boundary.as_mut() {

tests/elevenlabs_timestamps_fallback.rs

@@ -0,0 +1,123 @@
+//! Offline test for the ElevenLabs `/with-timestamps` degrade path: when
+//! the endpoint variant is rejected (a model that doesn't support it),
+//! speak() must retry the plain synthesis endpoint and deliver estimated
+//! boundaries instead of failing the call.
+//!
+//! Uses a `std::net::TcpListener` mock so no network access or API key is
+//! needed. The MP3 fixture is 0.4s of silence, regenerated with:
+//!   ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 0.4 -q:a 9 silence.mp3
+
+use std::io::{Read, Write};
+use std::net::{TcpListener, TcpStream};
+
+use rust_tts_wrapper::engine::TtsEngine;
+use rust_tts_wrapper::factory::create_engine;
+
+const SILENCE_MP3: &[u8] = include_bytes!("fixtures/silence.mp3");
+
+fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    haystack
+        .windows(needle.len())
+        .position(|window| window == needle)
+}
+
+/// Minimal HTTP/1.1 responder: drains exactly one request (headers plus
+/// Content-Length body) and writes `status` with `body`. Returns the
+/// request line so tests can assert on the path.
+fn respond(stream: &mut TcpStream, status: &str, content_type: &str, body: &[u8]) -> String {
+    let mut buf = vec![0u8; 16_384];
+    let mut read = 0usize;
+    let header_end = loop {
+        let n = stream.read(&mut buf[read..]).expect("read request");
+        assert!(n > 0, "client closed connection before sending a request");
+        read += n;
+        if let Some(pos) = find_subsequence(&buf[..read], b"\r\n\r\n") {
+            break pos;
+        }
+    };
+    let headers = String::from_utf8_lossy(&buf[..header_end]).to_string();
+    let content_length: usize = headers
+        .lines()
+        .find(|l| l.to_ascii_lowercase().starts_with("content-length"))
+        .and_then(|l| l.split(':').nth(1))
+        .map(|v| v.trim().parse().expect("content-length"))
+        .unwrap_or(0);
+    let mut received = read - header_end - 4;
+    while received < content_length {
+        let n = stream.read(&mut buf[read..]).expect("read body");
+        read += n;
+        received += n;
+    }
+
+    let response = format!(
+        "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+        body.len()
+    );
+    stream
+        .write_all(response.as_bytes())
+        .expect("write headers");
+    stream.write_all(body).expect("write body");
+    stream.flush().expect("flush");
+    headers.lines().next().unwrap_or_default().to_string()
+}
+
+#[test]
+fn elevenlabs_timestamps_rejection_degrades_to_estimated_boundaries() {
+    let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
+    let addr = listener.local_addr().expect("local addr");
+    let server = std::thread::spawn(move || {
+        let mut paths = Vec::new();
+        // First attempt: the /with-timestamps variant is rejected.
+        let (mut stream, _) = listener.accept().expect("accept 1");
+        paths.push(respond(
+            &mut stream,
+            "404 Not Found",
+            "application/json",
+            b"{\"detail\":{\"message\":\"not found\"}}",
+        ));
+        // Retry: plain synthesis endpoint with an MP3 body.
+        let (mut stream, _) = listener.accept().expect("accept 2");
+        paths.push(respond(&mut stream, "200 OK", "audio/mpeg", SILENCE_MP3));
+        paths
+    });
+
+    let creds = format!(r#"{{"apiKey":"test-key","synthUrl":"http://{addr}"}}"#);
+    let engine = create_engine("elevenlabs", &creds).expect("elevenlabs engine");
+
+    let mut words: Vec<String> = Vec::new();
+    let mut on_boundary =
+        |word: &str, _start: f32, _end: f32, offset: i32, _len: i32, _final: bool| {
+            assert!(
+                offset >= 0,
+                "estimated word {word:?} must resolve in caller text"
+            );
+            words.push(word.to_string());
+        };
+    // Boundaries requested, no on_audio: exercises the degraded streaming
+    // entry that previously would have delivered no boundaries at all.
+    engine
+        .speak(
+            "Hello boundary fallback",
+            None,
+            1.0,
+            1.0,
+            1.0,
+            None,
+            Some(&mut on_boundary),
+            None,
+        )
+        .expect("degraded speak must succeed");
+
+    let paths = server.join().expect("server thread");
+    assert!(
+        paths[0].contains("/with-timestamps"),
+        "first attempt must use the variant: {}",
+        paths[0]
+    );
+    assert!(
+        !paths[1].contains("/with-timestamps"),
+        "retry must drop the variant: {}",
+        paths[1]
+    );
+    assert_eq!(words, ["Hello", "boundary", "fallback"]);
+}