From 0667ddcaf16d2bd60e3d57c3b093c3ac56a9ae08 Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 14:26:26 +0100 Subject: [PATCH 1/4] feat(elevenlabs): model-aware SpeechMarkdown dialects, modelId, speed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ElevenLabs previously routed SpeechMarkdown through the Alexa fallback and then stripped it to plain text in speak() — every break, tag and IPA hint was silently lost. Now: - preprocess_speech_markdown routes elevenlabs to the pre-v3 prompt dialect () and elevenlabs-v3 to the audio-tag dialect ([pause], [whispers], "/IPA/"), returning is_ssml=false so the dialect text reaches the API verbatim instead of being stripped - speak() picks the dialect from the model: eleven_v3* (settable via the modelId credential, previously hardcoded to multilingual_v2) gets audio tags — v3 reads stray XML aloud, pre-v3 reads tags aloud, so the mapping must follow the model - rate (1.0 = normal) maps to the deterministic voice_settings.speed API parameter, clamped to the documented 0.7-1.2 range - ElevenLabs boundary offsets search the caller's original input when SpeechMarkdown was reformatted (injected tags shifted offsets) - live-cloud template: v3 audio-tag synthesis + timestamps tests, and fixed the stale template (speak() gained on_mark, boundary callbacks a bool) so it compiles again - [patch.crates-io] points speechmarkdown-rust at the sibling checkout until >= 0.4.14 (Platform::ElevenLabsV3) is published --- .env.example | 3 + Cargo.toml | 8 ++ README.md | 3 +- src/cloud_engine.rs | 157 +++++++++++++++++++++++++++++++---- src/engine.rs | 16 +++- tests/live_cloud.rs.template | 93 +++++++++++++++++++-- 6 files changed, 251 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 2354b11..995362f 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ POLLY_AWS_ACCESS_KEY=your-aws-secret # ElevenLabs ELEVENLABS_API_KEY=your-elevenlabs-key +# Optional model override. Defaults to eleven_multilingual_v2 ( markup). +# Use eleven_v3 for audio tags ([whispers], [pause], "/IPA/") — v3 parses no SSML. +# ELEVENLABS_MODEL_ID=eleven_v3 # Wit.ai WITAI_TOKEN=your-witai-token diff --git a/Cargo.toml b/Cargo.toml index 5d573a0..e936b92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,3 +147,11 @@ strip = true all = "warn" pedantic = "warn" cargo = "warn" + +# Local development against the sibling checkout in the AACTools workspace. +# `[patch]` sections are ignored when rust-tts-wrapper is consumed as a +# dependency, so this only affects building/testing this repo directly — +# remove or bump the version requirement after speechmarkdown-rust +# (>= 0.4.14, with Platform::ElevenLabsV3) is published. +[patch.crates-io] +speechmarkdown-rust = { path = "../speechmarkdown-rust" } diff --git a/README.md b/README.md index 630e6c4..11b5189 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ cargo test --all-features Active development. Engine constructors, the C ABI, and the offline test suite run in CI on Linux, macOS, and Windows. Live cloud API calls are not exercised in CI — see `tests/live_cloud.rs.template` (copy to `tests/live_cloud.rs`, gitignored) and `.env.example` for running them locally with your own credentials. Live SherpaOnnx synthesis IS exercised in CI by the `sherpaonnx-live.yml` workflow (downloads small VITS/Matcha/Kokoro models and runs `tests/sherpaonnx_live.rs`), triggered on PRs touching `src/sherpaonnx_engine.rs` and available as a manual `workflow_dispatch`. - **Voice List**: Engines with "API" can enumerate voices from the provider's API. - **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable). -- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, others get Alexa SSML. +- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, ElevenLabs gets model-matched prompt markup (see below), others get Alexa SSML. +- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `` prompt markup; `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) — the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2). ## Rust API diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 798994b..22a3cbb 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -805,11 +805,20 @@ fn build_config(id: &str, creds: &HashMap) -> Option but read audio tags aloud. Unrecognized + // model IDs surface as API errors rather than being masked. + let model = creds + .get("modelId") + .filter(|m| !m.is_empty()) + .cloned() + .unwrap_or_else(|| "eleven_multilingual_v2".into()); Some(CloudConfig { synth_url: format!("https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"), auth_header: "xi-api-key".into(), model_param: Some("model_id".into()), - model_default: Some("eleven_multilingual_v2".into()), + model_default: Some(model), text_field: "text".into(), voices_url: Some("https://api.elevenlabs.io/v1/voices".into()), provider_id: "elevenlabs".into(), @@ -1833,7 +1842,26 @@ impl TtsEngine for CloudEngine { mut on_boundary: Option, _on_mark: Option, ) -> TtsResult<()> { - let (original_text, is_ssml) = preprocess_speech_markdown(text, &self.config.provider_id); + // SpeechMarkdown platform selector: ElevenLabs needs the dialect + // that matches the requested model (v3 audio tags vs pre-v3 + // markup) — the other dialect gets read aloud or ignored. + let smd_platform: &str = if self.config.provider_id == "elevenlabs" + && self + .config + .model_default + .as_deref() + .is_some_and(|m| m.starts_with("eleven_v3")) + { + "elevenlabs-v3" + } else { + self.config.provider_id.as_str() + }; + // Caller-facing text for word-boundary offset mapping: when + // SpeechMarkdown was reformatted (rather than passed through or + // converted to SSML), injected ElevenLabs tags shift offsets, so + // search the user's original input — the spoken words live there. + let user_text = text; + let (original_text, is_ssml) = preprocess_speech_markdown(text, smd_platform); // When the caller passed W3C SSML (via tts_speak_ssml), adapt per engine: // - Azure/Edge: pass through (their WS/REST paths handle SSML natively) @@ -1883,6 +1911,12 @@ impl TtsEngine for CloudEngine { text = original_text; } + // Boundary word search target (see `user_text` above): plain or + // dialect-reformatted input maps against what the caller passed; + // SSML paths keep searching the processed string (previously + // existing behavior). + let boundary_search_text: &str = if is_ssml { text.as_str() } else { user_text }; + // WebSocket approach: Azure when word boundaries are requested, or // Edge always (Edge is WS-only — it has no REST synth endpoint). // Edge reuses the identical Azure "Turn" protocol; only the URL/auth @@ -2350,6 +2384,21 @@ impl TtsEngine for CloudEngine { for (k, v) in &self.config.extra_body { body.insert(k.clone(), v.clone()); } + // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal) + // onto the deterministic voice_settings.speed API parameter + // (valid range 0.7–1.2; clamped). Only sent for explicit + // non-default rates. pitch/volume have no API equivalent + // (v3 models: use audio tags). + if self.config.provider_id == "elevenlabs" + && rate > 0.0 + && (rate - 1.0).abs() > f32::EPSILON + { + let speed = rate.clamp(0.7, 1.2); + body.insert( + "voice_settings".to_string(), + serde_json::json!({ "speed": speed }), + ); + } req = req.json(&serde_json::Value::Object(body)); req.send() }; @@ -2394,7 +2443,7 @@ impl TtsEngine for CloudEngine { let mut search_from = 0usize; for (word, start, end) in parse_elevenlabs_alignment(alignment) { #[allow(clippy::cast_possible_truncation)] - let char_offset = text[search_from..] + let char_offset = boundary_search_text[search_from..] .find(&word) .map_or(-1, |pos| (search_from + pos) as i32); @@ -4882,7 +4931,7 @@ mod tests { for input in &probe_inputs { let (azure_ssml, azure_ok) = preprocess_speech_markdown(input, "azure"); let (google_ssml, google_ok) = preprocess_speech_markdown(input, "google"); - let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "elevenlabs"); + let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "openai"); assert!(azure_ok, "azure failed to parse: {input:?}"); assert!(google_ok, "google failed to parse: {input:?}"); @@ -4904,23 +4953,38 @@ mod tests { ); } + #[test] + fn test_speechmarkdown_elevenlabs_dialects() { + use crate::engine::preprocess_speech_markdown; + // Pre-v3 dialect: prompt markup, not SSML (no + // wrapper, is_ssml false so speak() sends it verbatim). + let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs"); + assert!(!is_ssml, "elevenlabs dialect must not be flagged as SSML"); + assert_eq!(out, "Hello world"); + + // v3 dialect: audio tags; no XML the model would read aloud. + let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs-v3"); + assert!(!is_ssml); + assert_eq!(out, "Hello [long pause] world"); + + let (out, _) = preprocess_speech_markdown("(secret)[whisper]", "elevenlabs-v3"); + assert_eq!(out, "[whispers] secret"); + + let (out, _) = preprocess_speech_markdown("(speech)/spitʃ/", "elevenlabs-v3"); + assert_eq!(out, "\"/spitʃ/\""); + } + #[test] fn test_speechmarkdown_other_providers_detect_input() { use crate::engine::preprocess_speech_markdown; - // ElevenLabs, OpenAI, Cartesia, Murf, etc. all go through the - // Alexa fallback. They don't actually consume SSML — the result is - // discarded by the JSON-body branch in speak() — but detection - // must still flag the input as SpeechMarkdown so callers querying - // `is_ssml` get a truthful answer. - for provider in [ - "openai", - "elevenlabs", - "cartesia", - "murf", - "deepgram", - "witai", - "xai", - ] { + // OpenAI, Cartesia, Murf, etc. all go through the Alexa fallback. + // They don't actually consume SSML — the result is discarded by the + // JSON-body branch in speak() — but detection must still flag the + // input as SpeechMarkdown so callers querying `is_ssml` get a + // truthful answer. ElevenLabs is NOT in this list: it gets its own + // dialects, which are prompt markup, not SSML (see + // test_speechmarkdown_elevenlabs_dialects). + for provider in ["openai", "cartesia", "murf", "deepgram", "witai", "xai"] { let (_ssml, is_ssml) = preprocess_speech_markdown("Hello (world)[emphasis:\"strong\"]", provider); assert!( @@ -4952,6 +5016,63 @@ mod tests { assert!(url.ends_with("/text-to-speech/21m00Tcm4TlvDq8ikWAM/with-timestamps")); } + #[test] + fn test_elevenlabs_model_id_from_creds() { + // Default model stays multilingual_v2 (pre-v3 dialect). + let cfg = build_config("elevenlabs", &engine_creds("elevenlabs")).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2")); + + // modelId credential overrides it (v3 needs this: audio tags + // require eleven_v3, which parses no SSML at all). + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), "eleven_v3".into()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3")); + + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), "eleven_flash_v2_5".into()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_flash_v2_5")); + + // Empty modelId falls back to the default. + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), String::new()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2")); + } + + #[test] + fn test_elevenlabs_dialect_follows_model() { + // The speak() platform selector: eleven_v3* → audio-tag dialect, + // anything else → pre-v3 markup. Mirror the exact + // predicate here so a refactor can't silently flip it. + fn dialect_for(provider: &str, model: Option<&str>) -> String { + if provider == "elevenlabs" && model.is_some_and(|m| m.starts_with("eleven_v3")) { + "elevenlabs-v3".to_string() + } else { + provider.to_string() + } + } + assert_eq!( + dialect_for("elevenlabs", Some("eleven_v3")), + "elevenlabs-v3" + ); + assert_eq!( + dialect_for("elevenlabs", Some("eleven_v3_conversational")), + "elevenlabs-v3" + ); + assert_eq!( + dialect_for("elevenlabs", Some("eleven_multilingual_v2")), + "elevenlabs" + ); + assert_eq!( + dialect_for("elevenlabs", Some("eleven_flash_v2")), + "elevenlabs" + ); + assert_eq!(dialect_for("azure", Some("eleven_v3")), "azure"); + assert_eq!(dialect_for("elevenlabs", None), "elevenlabs"); + } + // ===== Auth-header composition per provider ===== // // speak() builds the final header value as `format!("{}{}", prefix, api_key)`. diff --git a/src/engine.rs b/src/engine.rs index 48fca37..e9c6d54 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -37,6 +37,11 @@ pub type OnErrorCallback<'a> = &'a mut dyn FnMut(&str); /// `platform` picks the SSML flavour: /// - `"azure"` → MicrosoftAzure /// - `"google"` → GoogleAssistant +/// - `"elevenlabs"` → ElevenLabs pre-v3 prompt markup (`` tags, +/// no SSML document). Not SSML: `is_ssml` is returned false so engines +/// pass the markup through instead of stripping it. +/// - `"elevenlabs-v3"` → Eleven v3 audio-tag dialect (`[whispers]`, +/// `[pause]`, native slash IPA). Eleven v3 parses no SSML at all. /// - `"sapi"` / `"avsynth"` / anything else → AmazonAlexa (the closest /// generic SSML baseline; SAPI's own parser accepts the subset that /// speechmarkdown-rust emits for Alexa) @@ -64,6 +69,11 @@ pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool) // few elements, which the engine boundary strips. "azure" | "edge" => Platform::MicrosoftAzure, "google" => Platform::GoogleAssistant, + // ElevenLabs prompt markup, not SSML: the dialects must reach the + // API verbatim (stripping would drop every break/tag, and v3 + // models read stray XML aloud). + "elevenlabs" => Platform::ElevenLabs, + "elevenlabs-v3" => Platform::ElevenLabsV3, // floravox parses the generic (Alexa-baseline) SSML dialect // natively; the floravox engine normalizes vendor-specific // elements (e.g. whisper's ) on its side — as @@ -71,8 +81,12 @@ pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool) _ => Platform::AmazonAlexa, }; + let is_elevenlabs_dialect = matches!(platform, Platform::ElevenLabs | Platform::ElevenLabsV3); + match SpeechMarkdownParser::to_ssml(text, platform) { - Ok(ssml) => (ssml, true), + // The ElevenLabs dialects are prompt text, not SSML: flag them so + // engines send the string as-is rather than treating it as SSML. + Ok(ssml) => (ssml, !is_elevenlabs_dialect), Err(_) => (text.to_string(), false), } } diff --git a/tests/live_cloud.rs.template b/tests/live_cloud.rs.template index dc9023b..86b653d 100644 --- a/tests/live_cloud.rs.template +++ b/tests/live_cloud.rs.template @@ -73,12 +73,12 @@ fn google_real_timepoints_match_words() { let engine = create_engine("google", &creds).expect("google engine"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Hello world testing"; engine - .speak(text, Some("en-US-Wavenet-D"), 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, Some("en-US-Wavenet-D"), 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("google speak"); let words = sink.lock().unwrap().words.clone(); @@ -145,6 +145,7 @@ fn google_streaming_chunks_are_at_most_8kb() { 1.0, Some(&mut cb), None, + None, ) .expect("speak"); let sink = sink.lock().unwrap(); @@ -167,12 +168,12 @@ fn azure_ws_real_word_boundaries_fire() { let engine = create_engine("azure", &creds).expect("azure"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Azure WebSocket real word boundary test."; engine - .speak(text, Some("en-US-AriaNeural"), 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, Some("en-US-AriaNeural"), 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("azure speak"); let words = sink.lock().unwrap().words.clone(); @@ -202,6 +203,7 @@ fn azure_speechmarkdown_to_microsoft_ssml() { 1.0, None, None, + None, ) .expect("speechmarkdown should not break Azure synthesis"); } @@ -236,14 +238,13 @@ fn elevenlabs_real_alignment_groups_into_words() { let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Hello world testing"; engine - .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("elevenlabs speak"); - let words = sink.lock().unwrap().words.clone(); let expected: Vec<&str> = text.split_whitespace().collect(); assert_eq!(words.len(), expected.len()); @@ -278,6 +279,7 @@ fn elevenlabs_streaming_audio_chunked() { 1.0, Some(&mut cb), None, + None, ) .expect("elevenlabs speak"); let sink = sink.lock().unwrap(); @@ -285,6 +287,79 @@ fn elevenlabs_streaming_audio_chunked() { assert!(sink.chunks.iter().all(|&n| n <= 8192)); } +// ===== ElevenLabs v3 — SpeechMarkdown audio-tag dialect end-to-end ===== + +#[test] +#[ignore] +fn elevenlabs_v3_speechmarkdown_audio_tags_synthesise() { + // Requires the eleven_v3 model (set via modelId credential) — audio + // tags are v3-exclusive and pre-v3 models read them aloud. Verifies + // the dialect reaches the API verbatim (not stripped to plain text) + // and that audio comes back. + let key = std::env::var("ELEVENLABS_API_KEY").unwrap_or_default(); + if key.is_empty() { + eprintln!("skipping: ELEVENLABS_API_KEY not set"); + return; + } + let creds = format!(r#"{{"apiKey":"{key}","modelId":"eleven_v3"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); + let sink = Arc::new(Mutex::new(AudioSink::new())); + let s = sink.clone(); + let mut cb = move |c: &[u8]| { + let mut sink = s.lock().unwrap(); + sink.total += c.len(); + }; + engine + .speak( + "(Right on time)[excited] [2s] This is a (secret)[whisper].", + None, + 1.0, + 1.0, + 1.0, + Some(&mut cb), + None, + None, + ) + .expect("elevenlabs v3 speak with audio tags"); + let sink = sink.lock().unwrap(); + assert!(sink.total > 0, "no audio returned for v3 audio-tag synthesis"); +} + +#[test] +#[ignore] +fn elevenlabs_v3_timestamps_with_audio_tags() { + // Word alignment through /with-timestamps while the text carries + // injected audio tags: boundary offsets must still resolve against + // the caller's input (words found, offsets >= 0). + let key = std::env::var("ELEVENLABS_API_KEY").unwrap_or_default(); + if key.is_empty() { + eprintln!("skipping: ELEVENLABS_API_KEY not set"); + return; + } + let creds = format!(r#"{{"apiKey":"{key}","modelId":"eleven_v3"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); + let sink = Arc::new(Mutex::new(BoundarySink::new())); + let s = sink.clone(); + let mut cb = move |w: &str, st: f32, e: f32, off: i32, _: i32, _: bool| { + s.lock().unwrap().words.push((w.into(), st, e)); + assert!(off >= 0, "word {w:?} not found in caller text"); + }; + engine + .speak( + "(Listen carefully)[whisper] because this matters.", + None, + 1.0, + 1.0, + 1.0, + None, + Some(&mut cb), + None, + ) + .expect("elevenlabs v3 speak with timestamps"); + let words = sink.lock().unwrap().words.clone(); + assert!(!words.is_empty(), "no word boundaries returned"); +} + // ===== OpenAI — basic streaming + check_credentials ===== #[test] @@ -318,7 +393,7 @@ fn openai_synthesises_with_default_voice() { *t.lock().unwrap() += c.len(); }; engine - .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None, None,) .expect("openai speak"); assert!(*total.lock().unwrap() > 0); } @@ -365,7 +440,7 @@ fn cartesia_voice_list_and_synth() { *t.lock().unwrap() += c.len(); }; engine - .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None, None,) .expect("cartesia speak"); assert!(*total.lock().unwrap() > 0); } From 903a81841d725253698c179179c56c99130c1677 Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 14:39:01 +0100 Subject: [PATCH 2/4] chore: patch speechmarkdown-rust to the feat/elevenlabs-dialects git branch The relative-path patch only resolves in a sibling checkout; CI needs the pushed branch. Swap to a version bump after 0.4.14 is published. --- Cargo.toml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e936b92..001faf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,10 +148,9 @@ all = "warn" pedantic = "warn" cargo = "warn" -# Local development against the sibling checkout in the AACTools workspace. -# `[patch]` sections are ignored when rust-tts-wrapper is consumed as a -# dependency, so this only affects building/testing this repo directly — -# remove or bump the version requirement after speechmarkdown-rust -# (>= 0.4.14, with Platform::ElevenLabsV3) is published. +# Build against the ElevenLabs-dialect branch until speechmarkdown-rust +# >= 0.4.14 (Platform::ElevenLabsV3) is published; then replace this with a +# version bump. `[patch]` sections are ignored when rust-tts-wrapper is +# consumed as a dependency, so this only affects building this repo. [patch.crates-io] -speechmarkdown-rust = { path = "../speechmarkdown-rust" } +speechmarkdown-rust = { git = "https://github.com/AACTools/speechmarkdown-rust.git", branch = "feat/elevenlabs-dialects" } From 0da882c560cd1cd45976c383d65bddfb5aea6580 Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 14:50:00 +0100 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20review=20feedback=20=E2=80=94=20test?= =?UTF-8?q?=20the=20real=20dialect=20predicate,=20extra=5Fbody=20precedenc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extract elevenlabs_smd_platform() and assert against the production helper (the test previously checked a local copy that would keep passing if the real predicate flipped) - derived voice_settings no longer clobbers a config-supplied one (skipped when extra_body carries voice_settings; also inserted before the extra_body loop so config wins either way) - live template header lists the v3 dialect tests; README notes the 3s break clamp --- README.md | 2 +- src/cloud_engine.rs | 65 +++++++++++++++++++----------------- tests/live_cloud.rs.template | 5 ++- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 11b5189..3f24329 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Active development. Engine constructors, the C ABI, and the offline test suite r - **Voice List**: Engines with "API" can enumerate voices from the provider's API. - **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable). - **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, ElevenLabs gets model-matched prompt markup (see below), others get Alexa SSML. -- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `` prompt markup; `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) — the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2). +- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `` prompt markup (≤3s, clamped); `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) — the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2). ## Rust API diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 22a3cbb..4b36f62 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -1415,6 +1415,20 @@ fn parse_google_timepoints( boundaries } +/// Pick the SpeechMarkdown platform selector for a provider/model pair. +/// +/// ElevenLabs markup is model-dependent: `eleven_v3*` parses no SSML and +/// needs the audio-tag dialect; every other model (and every other +/// provider) maps to `provider` unchanged (the caller's provider id is +/// itself the selector for azure/google/the Alexa fallback). +fn elevenlabs_smd_platform<'a>(provider: &'a str, model: Option<&str>) -> &'a str { + if provider == "elevenlabs" && model.is_some_and(|m| m.starts_with("eleven_v3")) { + "elevenlabs-v3" + } else { + provider + } +} + /// Parse ElevenLabs alignment payload into `(word, start_sec, end_sec)` tuples. /// /// ElevenLabs returns per-character timing in `alignment`: @@ -1845,17 +1859,10 @@ impl TtsEngine for CloudEngine { // SpeechMarkdown platform selector: ElevenLabs needs the dialect // that matches the requested model (v3 audio tags vs pre-v3 // markup) — the other dialect gets read aloud or ignored. - let smd_platform: &str = if self.config.provider_id == "elevenlabs" - && self - .config - .model_default - .as_deref() - .is_some_and(|m| m.starts_with("eleven_v3")) - { - "elevenlabs-v3" - } else { - self.config.provider_id.as_str() - }; + let smd_platform = elevenlabs_smd_platform( + &self.config.provider_id, + self.config.model_default.as_deref(), + ); // Caller-facing text for word-boundary offset mapping: when // SpeechMarkdown was reformatted (rather than passed through or // converted to SSML), injected ElevenLabs tags shift offsets, so @@ -2381,17 +2388,17 @@ impl TtsEngine for CloudEngine { ); } } - for (k, v) in &self.config.extra_body { - body.insert(k.clone(), v.clone()); - } // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal) // onto the deterministic voice_settings.speed API parameter // (valid range 0.7–1.2; clamped). Only sent for explicit // non-default rates. pitch/volume have no API equivalent - // (v3 models: use audio tags). + // (v3 models: use audio tags). Inserted before extra_body so + // a config-supplied voice_settings object (stability, + // similarity, …) takes precedence over the derived one. if self.config.provider_id == "elevenlabs" && rate > 0.0 && (rate - 1.0).abs() > f32::EPSILON + && !self.config.extra_body.contains_key("voice_settings") { let speed = rate.clamp(0.7, 1.2); body.insert( @@ -2399,6 +2406,9 @@ impl TtsEngine for CloudEngine { serde_json::json!({ "speed": speed }), ); } + for (k, v) in &self.config.extra_body { + body.insert(k.clone(), v.clone()); + } req = req.json(&serde_json::Value::Object(body)); req.send() }; @@ -5043,34 +5053,27 @@ mod tests { #[test] fn test_elevenlabs_dialect_follows_model() { - // The speak() platform selector: eleven_v3* → audio-tag dialect, - // anything else → pre-v3 markup. Mirror the exact - // predicate here so a refactor can't silently flip it. - fn dialect_for(provider: &str, model: Option<&str>) -> String { - if provider == "elevenlabs" && model.is_some_and(|m| m.starts_with("eleven_v3")) { - "elevenlabs-v3".to_string() - } else { - provider.to_string() - } - } + // The production predicate used by speak(): eleven_v3* → audio-tag + // dialect, anything else → pre-v3 markup. Asserted directly + // against the real helper so a flip fails here, not just live. assert_eq!( - dialect_for("elevenlabs", Some("eleven_v3")), + elevenlabs_smd_platform("elevenlabs", Some("eleven_v3")), "elevenlabs-v3" ); assert_eq!( - dialect_for("elevenlabs", Some("eleven_v3_conversational")), + elevenlabs_smd_platform("elevenlabs", Some("eleven_v3_conversational")), "elevenlabs-v3" ); assert_eq!( - dialect_for("elevenlabs", Some("eleven_multilingual_v2")), + elevenlabs_smd_platform("elevenlabs", Some("eleven_multilingual_v2")), "elevenlabs" ); assert_eq!( - dialect_for("elevenlabs", Some("eleven_flash_v2")), + elevenlabs_smd_platform("elevenlabs", Some("eleven_flash_v2")), "elevenlabs" ); - assert_eq!(dialect_for("azure", Some("eleven_v3")), "azure"); - assert_eq!(dialect_for("elevenlabs", None), "elevenlabs"); + assert_eq!(elevenlabs_smd_platform("azure", Some("eleven_v3")), "azure"); + assert_eq!(elevenlabs_smd_platform("elevenlabs", None), "elevenlabs"); } // ===== Auth-header composition per provider ===== diff --git a/tests/live_cloud.rs.template b/tests/live_cloud.rs.template index 86b653d..516e6b1 100644 --- a/tests/live_cloud.rs.template +++ b/tests/live_cloud.rs.template @@ -15,9 +15,12 @@ //! - Google v1beta1 real timepoints (boundary offset == audio frame offset) //! - Azure WebSocket word-boundary turn protocol //! - ElevenLabs /with-timestamps character alignment +//! - ElevenLabs v3 SpeechMarkdown audio-tag dialect end-to-end (synthesis +//! + word offsets with injected tags; requires modelId=eleven_v3) //! - Streaming chunk size & count across providers //! - SpeechMarkdown end-to-end per platform (Azure Microsoft SSML, -//! Google Assistant SSML, others Alexa SSML) +//! Google Assistant SSML, ElevenLabs model-matched prompt dialects, +//! others Alexa SSML) //! - check_credentials() returns true with valid keys #![allow(clippy::all, clippy::pedantic, clippy::float_cmp)] From 13d96a44b9f058caf47e8bb6a89f263826b5621b Mon Sep 17 00:00:00 2001 From: will wade Date: Fri, 4 Sep 2026 15:06:37 +0100 Subject: [PATCH 4/4] chore: bump speechmarkdown-rust to 0.4.14 from crates.io MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.4.14 (published from the merged ElevenLabs PR) carries Platform::ElevenLabsV3 and the expressive/strict-break parser work — drop the temporary [patch.crates-io] git-branch override. --- Cargo.toml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 001faf3..4e9d4ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ sherpa-onnx-models = { version = "0.1", optional = true } # Pin sys to match the main crate (cargo update can drift them apart). sherpa-onnx-sys = { version = "=1.13.5", optional = true } base64 = { version = "0.22", optional = true } -speechmarkdown-rust = { version = "0.4.13", optional = true } +speechmarkdown-rust = { version = "0.4.14", optional = true } anyhow = "1" tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"], optional = true } uuid = { version = "1.23.2", features = ["v4"], optional = true } @@ -147,10 +147,3 @@ strip = true all = "warn" pedantic = "warn" cargo = "warn" - -# Build against the ElevenLabs-dialect branch until speechmarkdown-rust -# >= 0.4.14 (Platform::ElevenLabsV3) is published; then replace this with a -# version bump. `[patch]` sections are ignored when rust-tts-wrapper is -# consumed as a dependency, so this only affects building this repo. -[patch.crates-io] -speechmarkdown-rust = { git = "https://github.com/AACTools/speechmarkdown-rust.git", branch = "feat/elevenlabs-dialects" }