diff --git a/.env.example b/.env.example index 2354b11..995362f 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ POLLY_AWS_ACCESS_KEY=your-aws-secret # ElevenLabs ELEVENLABS_API_KEY=your-elevenlabs-key +# Optional model override. Defaults to eleven_multilingual_v2 ( markup). +# Use eleven_v3 for audio tags ([whispers], [pause], "/IPA/") — v3 parses no SSML. +# ELEVENLABS_MODEL_ID=eleven_v3 # Wit.ai WITAI_TOKEN=your-witai-token diff --git a/Cargo.toml b/Cargo.toml index 5d573a0..4e9d4ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ sherpa-onnx-models = { version = "0.1", optional = true } # Pin sys to match the main crate (cargo update can drift them apart). sherpa-onnx-sys = { version = "=1.13.5", optional = true } base64 = { version = "0.22", optional = true } -speechmarkdown-rust = { version = "0.4.13", optional = true } +speechmarkdown-rust = { version = "0.4.14", optional = true } anyhow = "1" tungstenite = { version = "0.29.0", features = ["rustls-tls-webpki-roots"], optional = true } uuid = { version = "1.23.2", features = ["v4"], optional = true } diff --git a/README.md b/README.md index 630e6c4..3f24329 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ cargo test --all-features Active development. Engine constructors, the C ABI, and the offline test suite run in CI on Linux, macOS, and Windows. Live cloud API calls are not exercised in CI — see `tests/live_cloud.rs.template` (copy to `tests/live_cloud.rs`, gitignored) and `.env.example` for running them locally with your own credentials. Live SherpaOnnx synthesis IS exercised in CI by the `sherpaonnx-live.yml` workflow (downloads small VITS/Matcha/Kokoro models and runs `tests/sherpaonnx_live.rs`), triggered on PRs touching `src/sherpaonnx_engine.rs` and available as a manual `workflow_dispatch`. - **Voice List**: Engines with "API" can enumerate voices from the provider's API. - **Word Boundaries**: Google returns real timing via v1beta1 timepoints with SSML marks. All others use word-length-adjusted estimation (150 WPM baseline, configurable). -- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, others get Alexa SSML. +- **Speech Markdown**: Auto-detected and converted to platform-specific SSML via [speechmarkdown-rust](https://github.com/AACTools/speechmarkdown-rust). Azure gets Microsoft SSML, Google gets Assistant SSML, ElevenLabs gets model-matched prompt markup (see below), others get Alexa SSML. +- **ElevenLabs markup**: ElevenLabs parses no SSML documents. Pre-v3 models (`eleven_multilingual_v2`, `flash_v2_5`, `flash_v2`) get `` prompt markup (≤3s, clamped); `eleven_v3*` (set via the `modelId` credential) gets audio tags (`[whispers]`, `[pause]`, `[long pause]`, native `"/IPA/"`) — the dialect is chosen from the model because v3 reads stray XML aloud and pre-v3 models read audio tags aloud. The `rate` parameter maps to the deterministic `voice_settings.speed` API setting (0.7–1.2). ## Rust API diff --git a/src/cloud_engine.rs b/src/cloud_engine.rs index 798994b..4b36f62 100644 --- a/src/cloud_engine.rs +++ b/src/cloud_engine.rs @@ -805,11 +805,20 @@ fn build_config(id: &str, creds: &HashMap) -> Option but read audio tags aloud. Unrecognized + // model IDs surface as API errors rather than being masked. + let model = creds + .get("modelId") + .filter(|m| !m.is_empty()) + .cloned() + .unwrap_or_else(|| "eleven_multilingual_v2".into()); Some(CloudConfig { synth_url: format!("https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"), auth_header: "xi-api-key".into(), model_param: Some("model_id".into()), - model_default: Some("eleven_multilingual_v2".into()), + model_default: Some(model), text_field: "text".into(), voices_url: Some("https://api.elevenlabs.io/v1/voices".into()), provider_id: "elevenlabs".into(), @@ -1406,6 +1415,20 @@ fn parse_google_timepoints( boundaries } +/// Pick the SpeechMarkdown platform selector for a provider/model pair. +/// +/// ElevenLabs markup is model-dependent: `eleven_v3*` parses no SSML and +/// needs the audio-tag dialect; every other model (and every other +/// provider) maps to `provider` unchanged (the caller's provider id is +/// itself the selector for azure/google/the Alexa fallback). +fn elevenlabs_smd_platform<'a>(provider: &'a str, model: Option<&str>) -> &'a str { + if provider == "elevenlabs" && model.is_some_and(|m| m.starts_with("eleven_v3")) { + "elevenlabs-v3" + } else { + provider + } +} + /// Parse ElevenLabs alignment payload into `(word, start_sec, end_sec)` tuples. /// /// ElevenLabs returns per-character timing in `alignment`: @@ -1833,7 +1856,19 @@ impl TtsEngine for CloudEngine { mut on_boundary: Option, _on_mark: Option, ) -> TtsResult<()> { - let (original_text, is_ssml) = preprocess_speech_markdown(text, &self.config.provider_id); + // SpeechMarkdown platform selector: ElevenLabs needs the dialect + // that matches the requested model (v3 audio tags vs pre-v3 + // markup) — the other dialect gets read aloud or ignored. + let smd_platform = elevenlabs_smd_platform( + &self.config.provider_id, + self.config.model_default.as_deref(), + ); + // Caller-facing text for word-boundary offset mapping: when + // SpeechMarkdown was reformatted (rather than passed through or + // converted to SSML), injected ElevenLabs tags shift offsets, so + // search the user's original input — the spoken words live there. + let user_text = text; + let (original_text, is_ssml) = preprocess_speech_markdown(text, smd_platform); // When the caller passed W3C SSML (via tts_speak_ssml), adapt per engine: // - Azure/Edge: pass through (their WS/REST paths handle SSML natively) @@ -1883,6 +1918,12 @@ impl TtsEngine for CloudEngine { text = original_text; } + // Boundary word search target (see `user_text` above): plain or + // dialect-reformatted input maps against what the caller passed; + // SSML paths keep searching the processed string (previously + // existing behavior). + let boundary_search_text: &str = if is_ssml { text.as_str() } else { user_text }; + // WebSocket approach: Azure when word boundaries are requested, or // Edge always (Edge is WS-only — it has no REST synth endpoint). // Edge reuses the identical Azure "Turn" protocol; only the URL/auth @@ -2347,6 +2388,24 @@ impl TtsEngine for CloudEngine { ); } } + // ElevenLabs: map the wrapper's rate multiplier (1.0 = normal) + // onto the deterministic voice_settings.speed API parameter + // (valid range 0.7–1.2; clamped). Only sent for explicit + // non-default rates. pitch/volume have no API equivalent + // (v3 models: use audio tags). Inserted before extra_body so + // a config-supplied voice_settings object (stability, + // similarity, …) takes precedence over the derived one. + if self.config.provider_id == "elevenlabs" + && rate > 0.0 + && (rate - 1.0).abs() > f32::EPSILON + && !self.config.extra_body.contains_key("voice_settings") + { + let speed = rate.clamp(0.7, 1.2); + body.insert( + "voice_settings".to_string(), + serde_json::json!({ "speed": speed }), + ); + } for (k, v) in &self.config.extra_body { body.insert(k.clone(), v.clone()); } @@ -2394,7 +2453,7 @@ impl TtsEngine for CloudEngine { let mut search_from = 0usize; for (word, start, end) in parse_elevenlabs_alignment(alignment) { #[allow(clippy::cast_possible_truncation)] - let char_offset = text[search_from..] + let char_offset = boundary_search_text[search_from..] .find(&word) .map_or(-1, |pos| (search_from + pos) as i32); @@ -4882,7 +4941,7 @@ mod tests { for input in &probe_inputs { let (azure_ssml, azure_ok) = preprocess_speech_markdown(input, "azure"); let (google_ssml, google_ok) = preprocess_speech_markdown(input, "google"); - let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "elevenlabs"); + let (alexa_ssml, alexa_ok) = preprocess_speech_markdown(input, "openai"); assert!(azure_ok, "azure failed to parse: {input:?}"); assert!(google_ok, "google failed to parse: {input:?}"); @@ -4904,23 +4963,38 @@ mod tests { ); } + #[test] + fn test_speechmarkdown_elevenlabs_dialects() { + use crate::engine::preprocess_speech_markdown; + // Pre-v3 dialect: prompt markup, not SSML (no + // wrapper, is_ssml false so speak() sends it verbatim). + let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs"); + assert!(!is_ssml, "elevenlabs dialect must not be flagged as SSML"); + assert_eq!(out, "Hello world"); + + // v3 dialect: audio tags; no XML the model would read aloud. + let (out, is_ssml) = preprocess_speech_markdown("Hello [2s] world", "elevenlabs-v3"); + assert!(!is_ssml); + assert_eq!(out, "Hello [long pause] world"); + + let (out, _) = preprocess_speech_markdown("(secret)[whisper]", "elevenlabs-v3"); + assert_eq!(out, "[whispers] secret"); + + let (out, _) = preprocess_speech_markdown("(speech)/spitʃ/", "elevenlabs-v3"); + assert_eq!(out, "\"/spitʃ/\""); + } + #[test] fn test_speechmarkdown_other_providers_detect_input() { use crate::engine::preprocess_speech_markdown; - // ElevenLabs, OpenAI, Cartesia, Murf, etc. all go through the - // Alexa fallback. They don't actually consume SSML — the result is - // discarded by the JSON-body branch in speak() — but detection - // must still flag the input as SpeechMarkdown so callers querying - // `is_ssml` get a truthful answer. - for provider in [ - "openai", - "elevenlabs", - "cartesia", - "murf", - "deepgram", - "witai", - "xai", - ] { + // OpenAI, Cartesia, Murf, etc. all go through the Alexa fallback. + // They don't actually consume SSML — the result is discarded by the + // JSON-body branch in speak() — but detection must still flag the + // input as SpeechMarkdown so callers querying `is_ssml` get a + // truthful answer. ElevenLabs is NOT in this list: it gets its own + // dialects, which are prompt markup, not SSML (see + // test_speechmarkdown_elevenlabs_dialects). + for provider in ["openai", "cartesia", "murf", "deepgram", "witai", "xai"] { let (_ssml, is_ssml) = preprocess_speech_markdown("Hello (world)[emphasis:\"strong\"]", provider); assert!( @@ -4952,6 +5026,56 @@ mod tests { assert!(url.ends_with("/text-to-speech/21m00Tcm4TlvDq8ikWAM/with-timestamps")); } + #[test] + fn test_elevenlabs_model_id_from_creds() { + // Default model stays multilingual_v2 (pre-v3 dialect). + let cfg = build_config("elevenlabs", &engine_creds("elevenlabs")).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2")); + + // modelId credential overrides it (v3 needs this: audio tags + // require eleven_v3, which parses no SSML at all). + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), "eleven_v3".into()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_v3")); + + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), "eleven_flash_v2_5".into()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_flash_v2_5")); + + // Empty modelId falls back to the default. + let mut c = engine_creds("elevenlabs"); + c.insert("modelId".into(), String::new()); + let cfg = build_config("elevenlabs", &c).unwrap(); + assert_eq!(cfg.model_default.as_deref(), Some("eleven_multilingual_v2")); + } + + #[test] + fn test_elevenlabs_dialect_follows_model() { + // The production predicate used by speak(): eleven_v3* → audio-tag + // dialect, anything else → pre-v3 markup. Asserted directly + // against the real helper so a flip fails here, not just live. + assert_eq!( + elevenlabs_smd_platform("elevenlabs", Some("eleven_v3")), + "elevenlabs-v3" + ); + assert_eq!( + elevenlabs_smd_platform("elevenlabs", Some("eleven_v3_conversational")), + "elevenlabs-v3" + ); + assert_eq!( + elevenlabs_smd_platform("elevenlabs", Some("eleven_multilingual_v2")), + "elevenlabs" + ); + assert_eq!( + elevenlabs_smd_platform("elevenlabs", Some("eleven_flash_v2")), + "elevenlabs" + ); + assert_eq!(elevenlabs_smd_platform("azure", Some("eleven_v3")), "azure"); + assert_eq!(elevenlabs_smd_platform("elevenlabs", None), "elevenlabs"); + } + // ===== Auth-header composition per provider ===== // // speak() builds the final header value as `format!("{}{}", prefix, api_key)`. diff --git a/src/engine.rs b/src/engine.rs index 48fca37..e9c6d54 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -37,6 +37,11 @@ pub type OnErrorCallback<'a> = &'a mut dyn FnMut(&str); /// `platform` picks the SSML flavour: /// - `"azure"` → MicrosoftAzure /// - `"google"` → GoogleAssistant +/// - `"elevenlabs"` → ElevenLabs pre-v3 prompt markup (`` tags, +/// no SSML document). Not SSML: `is_ssml` is returned false so engines +/// pass the markup through instead of stripping it. +/// - `"elevenlabs-v3"` → Eleven v3 audio-tag dialect (`[whispers]`, +/// `[pause]`, native slash IPA). Eleven v3 parses no SSML at all. /// - `"sapi"` / `"avsynth"` / anything else → AmazonAlexa (the closest /// generic SSML baseline; SAPI's own parser accepts the subset that /// speechmarkdown-rust emits for Alexa) @@ -64,6 +69,11 @@ pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool) // few elements, which the engine boundary strips. "azure" | "edge" => Platform::MicrosoftAzure, "google" => Platform::GoogleAssistant, + // ElevenLabs prompt markup, not SSML: the dialects must reach the + // API verbatim (stripping would drop every break/tag, and v3 + // models read stray XML aloud). + "elevenlabs" => Platform::ElevenLabs, + "elevenlabs-v3" => Platform::ElevenLabsV3, // floravox parses the generic (Alexa-baseline) SSML dialect // natively; the floravox engine normalizes vendor-specific // elements (e.g. whisper's ) on its side — as @@ -71,8 +81,12 @@ pub fn preprocess_speech_markdown(text: &str, platform: &str) -> (String, bool) _ => Platform::AmazonAlexa, }; + let is_elevenlabs_dialect = matches!(platform, Platform::ElevenLabs | Platform::ElevenLabsV3); + match SpeechMarkdownParser::to_ssml(text, platform) { - Ok(ssml) => (ssml, true), + // The ElevenLabs dialects are prompt text, not SSML: flag them so + // engines send the string as-is rather than treating it as SSML. + Ok(ssml) => (ssml, !is_elevenlabs_dialect), Err(_) => (text.to_string(), false), } } diff --git a/tests/live_cloud.rs.template b/tests/live_cloud.rs.template index dc9023b..516e6b1 100644 --- a/tests/live_cloud.rs.template +++ b/tests/live_cloud.rs.template @@ -15,9 +15,12 @@ //! - Google v1beta1 real timepoints (boundary offset == audio frame offset) //! - Azure WebSocket word-boundary turn protocol //! - ElevenLabs /with-timestamps character alignment +//! - ElevenLabs v3 SpeechMarkdown audio-tag dialect end-to-end (synthesis +//! + word offsets with injected tags; requires modelId=eleven_v3) //! - Streaming chunk size & count across providers //! - SpeechMarkdown end-to-end per platform (Azure Microsoft SSML, -//! Google Assistant SSML, others Alexa SSML) +//! Google Assistant SSML, ElevenLabs model-matched prompt dialects, +//! others Alexa SSML) //! - check_credentials() returns true with valid keys #![allow(clippy::all, clippy::pedantic, clippy::float_cmp)] @@ -73,12 +76,12 @@ fn google_real_timepoints_match_words() { let engine = create_engine("google", &creds).expect("google engine"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Hello world testing"; engine - .speak(text, Some("en-US-Wavenet-D"), 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, Some("en-US-Wavenet-D"), 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("google speak"); let words = sink.lock().unwrap().words.clone(); @@ -145,6 +148,7 @@ fn google_streaming_chunks_are_at_most_8kb() { 1.0, Some(&mut cb), None, + None, ) .expect("speak"); let sink = sink.lock().unwrap(); @@ -167,12 +171,12 @@ fn azure_ws_real_word_boundaries_fire() { let engine = create_engine("azure", &creds).expect("azure"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Azure WebSocket real word boundary test."; engine - .speak(text, Some("en-US-AriaNeural"), 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, Some("en-US-AriaNeural"), 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("azure speak"); let words = sink.lock().unwrap().words.clone(); @@ -202,6 +206,7 @@ fn azure_speechmarkdown_to_microsoft_ssml() { 1.0, None, None, + None, ) .expect("speechmarkdown should not break Azure synthesis"); } @@ -236,14 +241,13 @@ fn elevenlabs_real_alignment_groups_into_words() { let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); let sink = Arc::new(Mutex::new(BoundarySink::new())); let s = sink.clone(); - let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32| { + let mut cb = move |w: &str, st: f32, e: f32, _: i32, _: i32, _: bool| { s.lock().unwrap().words.push((w.into(), st, e)); }; let text = "Hello world testing"; engine - .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut cb)) + .speak(text, None, 1.0, 1.0, 1.0, None, Some(&mut cb), None,) .expect("elevenlabs speak"); - let words = sink.lock().unwrap().words.clone(); let expected: Vec<&str> = text.split_whitespace().collect(); assert_eq!(words.len(), expected.len()); @@ -278,6 +282,7 @@ fn elevenlabs_streaming_audio_chunked() { 1.0, Some(&mut cb), None, + None, ) .expect("elevenlabs speak"); let sink = sink.lock().unwrap(); @@ -285,6 +290,79 @@ fn elevenlabs_streaming_audio_chunked() { assert!(sink.chunks.iter().all(|&n| n <= 8192)); } +// ===== ElevenLabs v3 — SpeechMarkdown audio-tag dialect end-to-end ===== + +#[test] +#[ignore] +fn elevenlabs_v3_speechmarkdown_audio_tags_synthesise() { + // Requires the eleven_v3 model (set via modelId credential) — audio + // tags are v3-exclusive and pre-v3 models read them aloud. Verifies + // the dialect reaches the API verbatim (not stripped to plain text) + // and that audio comes back. + let key = std::env::var("ELEVENLABS_API_KEY").unwrap_or_default(); + if key.is_empty() { + eprintln!("skipping: ELEVENLABS_API_KEY not set"); + return; + } + let creds = format!(r#"{{"apiKey":"{key}","modelId":"eleven_v3"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); + let sink = Arc::new(Mutex::new(AudioSink::new())); + let s = sink.clone(); + let mut cb = move |c: &[u8]| { + let mut sink = s.lock().unwrap(); + sink.total += c.len(); + }; + engine + .speak( + "(Right on time)[excited] [2s] This is a (secret)[whisper].", + None, + 1.0, + 1.0, + 1.0, + Some(&mut cb), + None, + None, + ) + .expect("elevenlabs v3 speak with audio tags"); + let sink = sink.lock().unwrap(); + assert!(sink.total > 0, "no audio returned for v3 audio-tag synthesis"); +} + +#[test] +#[ignore] +fn elevenlabs_v3_timestamps_with_audio_tags() { + // Word alignment through /with-timestamps while the text carries + // injected audio tags: boundary offsets must still resolve against + // the caller's input (words found, offsets >= 0). + let key = std::env::var("ELEVENLABS_API_KEY").unwrap_or_default(); + if key.is_empty() { + eprintln!("skipping: ELEVENLABS_API_KEY not set"); + return; + } + let creds = format!(r#"{{"apiKey":"{key}","modelId":"eleven_v3"}}"#); + let engine = create_engine("elevenlabs", &creds).expect("elevenlabs"); + let sink = Arc::new(Mutex::new(BoundarySink::new())); + let s = sink.clone(); + let mut cb = move |w: &str, st: f32, e: f32, off: i32, _: i32, _: bool| { + s.lock().unwrap().words.push((w.into(), st, e)); + assert!(off >= 0, "word {w:?} not found in caller text"); + }; + engine + .speak( + "(Listen carefully)[whisper] because this matters.", + None, + 1.0, + 1.0, + 1.0, + None, + Some(&mut cb), + None, + ) + .expect("elevenlabs v3 speak with timestamps"); + let words = sink.lock().unwrap().words.clone(); + assert!(!words.is_empty(), "no word boundaries returned"); +} + // ===== OpenAI — basic streaming + check_credentials ===== #[test] @@ -318,7 +396,7 @@ fn openai_synthesises_with_default_voice() { *t.lock().unwrap() += c.len(); }; engine - .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None, None,) .expect("openai speak"); assert!(*total.lock().unwrap() > 0); } @@ -365,7 +443,7 @@ fn cartesia_voice_list_and_synth() { *t.lock().unwrap() += c.len(); }; engine - .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None) + .speak("Hello world", None, 1.0, 1.0, 1.0, Some(&mut cb), None, None,) .expect("cartesia speak"); assert!(*total.lock().unwrap() > 0); }