RustTts floravox catch-up: ABI canary, pinned DLL, floravox routing, sidecar G2P (issue #15)

.github/workflows/msbuild.yml

@@ -223,14 +223,18 @@ jobs:
           Where-Object { $_.Name -like 'Microsoft.CognitiveServices.Speech.*.dll' -or $_.Name -in 'SpeechSDKShim.dll','SpeechSDKPatcher.exe' } |
           Remove-Item -Force
 
-        # Rust TTS wrapper native DLL (from NuGet package)
+        # Rust TTS wrapper native DLL (from NuGet package).
+        # Pinned to the exact RustTtsWrapper.Bindings version referenced by
+        # the csproj — grabbing "whatever is in the cache" once shipped a DLL
+        # with the wrong callback ABI (issue #15). Hard-fail if the pinned
+        # package has no native DLL: the adapter cannot run without it.
         dotnet restore VoiceGarden.UI\VoiceGarden.UI.csproj
-        $rustDll = Get-ChildItem "$env:USERPROFILE\.nuget\packages\rustttswrapper.bindings" -Recurse -Filter "rust_tts_wrapper.dll" | Select-Object -First 1
-        if ($rustDll) {
-          Copy-Item $rustDll.FullName payload\x64\ -Force
-          Write-Host "Copied rust_tts_wrapper.dll to payload\x64\"
-        } else {
-          Write-Host "WARNING: rust_tts_wrapper.dll not found in NuGet cache"
+        foreach ($rid in @('win-x64', 'win-x86')) {
+          $rustDll = & .\scripts\Get-RustTtsWrapperDll.ps1 -Rid $rid
+          if (-not $rustDll) { throw "rust_tts_wrapper.dll missing for $rid (see warnings above)" }
+          $arch = $rid -replace 'win-', ''
+          Copy-Item $rustDll "payload\$arch\" -Force
+          Write-Host "Copied rust_tts_wrapper.dll ($rid, csproj-pinned) to payload\$arch\"
         }
 
     - name: Build MSI

README.md

@@ -2,20 +2,22 @@
 
 > Forked from [NaturalVoiceSAPIAdapter](https://github.com/gexgd0419/NaturalVoiceSAPIAdapter). Developed at [AACTools/VoiceGarden-SAPI](https://github.com/AACTools/VoiceGarden-SAPI).
 
-A [SAPI 5 text-to-speech engine][1] that connects **21+ TTS engines** to any Windows application that supports SAPI voices — including Grid 3, Mind Express, Balabolka, Clicker, and any software using `System.Speech`.
+A [SAPI 5 text-to-speech engine][1] that connects **23 TTS engines** to any Windows application that supports SAPI voices — including Grid 3, Mind Express, Balabolka, Clicker, and any software using `System.Speech`.
 
 Powered by [rust-tts-wrapper](https://github.com/AACTools/rust-tts-wrapper) for all synthesis, voice listing, and word boundary events.
 
 ## What voices are supported?
 
 | Category | Engines | Cloud? |
 |----------|---------|--------|
-| **Offline neural** | SherpaOnnx (Kokoro, Piper, MMS, VITS, Matcha, Kitten) | No — fully local |
+| **Offline neural** | floravox (piper/MMS VITS, Matcha, Kokoro — measured word timings, SSML bookmarks, lexicon + Phonetisaurus + ByT5 G2P) and SherpaOnnx (Kokoro, Piper, MMS, VITS, Matcha, Kitten) as fallback | No — fully local* |
 | **Microsoft** | Azure Cognitive Services, Edge browser voices (credential-free) | Yes |
 | **Cloud TTS** | OpenAI, Google Cloud, AWS Polly, ElevenLabs, Cartesia, Deepgram | Yes |
 | **More cloud** | Watson, PlayHT, Wit.ai, Gemini, Hume AI, xAI Grok, Fish Audio, Mistral, Murf, Unreal Speech, Resemble, Uplift AI, Models Lab | Yes |
 
-All engines support **word boundary events** for word highlighting in AAC software.
+All engines support **word boundary events** for word highlighting in AAC software. Offline voices route to **floravox** wherever it's supported (piper/MMS VITS, Matcha, Kokoro — self-served from sherpa layout; duration-tensor timings on patched voices, SSML `<mark>` → `SPEI_TTSBOOKMARK`, published per-language G2P bundles cached locally, synthesis still works offline). **SherpaOnnx** is the automatic fallback (32-bit hosts, engine failure) and the only engine for the flow/diffusion families (zipvoice, supertonic, pocket, kitten).
+
+\* floravox on x64 needs Windows 10 1903+ (the static onnxruntime DirectML floor); older systems fall back to SherpaOnnx.
 
 ## Quick Start
 
@@ -56,19 +58,21 @@ VoiceGarden.UI.exe validate --engine azure --voice en-US-JennyNeural --key KEY -
 │  │  VoiceGardenSAPIAdapter.dll (C++ COM DLL)          │  │
 │  │  • BuildSSML (SAPI fragments → SSML)               │  │
 │  │  • Word boundary offset mapping                     │  │
+│  │  • SSML marks → SPEI_TTSBOOKMARK (floravox)         │  │
 │  │  • Audio streaming + silence compensation           │  │
 │  │         │ loads via LoadLibrary                     │  │
-│  │  ┌────────▼─────────────────────────────────────┐   │  │
-│  │  │  rust_tts_wrapper.dll (Rust, 22MB)           │   │  │
-│  │  │  • 21 engines (SherpaOnnx, Azure, Edge,      │   │  │
-│  │  │    OpenAI, Google, ElevenLabs, Polly, ...)   │   │  │
-│  │  │  • Word boundary events (Azure/Google: real,  │   │  │
-│  │  │    others: estimated)                         │   │  │
-│  │  │  • Viseme events (Azure/Edge)                 │   │  │
-│  │  │  • Connection pooling (Azure/Edge WS)         │   │  │
-│  │  │  • Sec-MS-GEC token (Edge voices)             │   │  │
-│  │  │  • SherpaOnnx model auto-detection            │   │  │
-│  │  └──────────────────────────────────────────────┘   │  │
+│  │  ┌────────▼─────────────────────────────────┐   │  │
+│  │  │  rust_tts_wrapper.dll (Rust)              │   │  │
+│  │  │  • 23 engines (floravox, SherpaOnnx,      │   │  │
+│  │  │    Azure, Edge, OpenAI, Google, ...)      │   │  │
+│  │  │  • Word boundaries (Azure/Google: real,   │   │  │
+│  │  │    floravox: measured, others: estimated) │   │  │
+│  │  │  • Viseme events (Azure/Edge)             │   │  │
+│  │  │  • Connection pooling (Azure/Edge WS)     │   │  │
+│  │  │  • Sec-MS-GEC token (Edge voices)         │   │  │
+│  │  │  • SherpaOnnx model auto-detection        │   │  │
+│  │  │  • ABI canary: refuses pre-0.5 DLLs       │   │  │
+│  │  └───────────────────────────────────────────┘   │  │
 │  └────────────────────────────────────────────────────┘  │
 │                                                           │
 │  ┌─────────────────────────────────────────────────────┐ │
@@ -87,8 +91,10 @@ VoiceGarden.UI.exe validate --engine azure --voice en-US-JennyNeural --key KEY -
 | Component | Description |
 |-----------|-------------|
 | `VoiceGardenSAPIAdapter/` | C++ SAPI COM DLL — SSML building, offset mapping, audio streaming. Loads `rust_tts_wrapper.dll` for all synthesis |
-| `VoiceGardenSAPIAdapter/RustTts/` | C++ wrapper for the Rust DLL (dynamic loading, callback marshalling) |
+| `VoiceGardenSAPIAdapter/RustTts/` | C++ wrapper for the Rust DLL (dynamic loading, ABI canary, callback marshalling) |
 | `VoiceGarden.UI/` | Avalonia UI app — configuration, model management, voice preview, analytics |
+| `VoiceGarden.UI/Services/PiperSidecarGenerator.cs` | Generates piper `*.onnx.json` sidecars for sherpa-layout voices so they route through floravox |
+| `VoiceGarden.UI.Tests/` | xunit tests: floravox integration (boundaries, marks, G2P) + sidecar generator units |
 | `SherpaOnnx/` | Model discovery (voice enumerator scans for installed models) |
 | `Setup/` + `SetupLauncher/` | WiX MSI package + setup.exe bootstrapper |
 
@@ -128,6 +134,13 @@ Each cloud engine needs its API key set in the Engine Config tab. Search voices
 
 ## Testing
 
+### Unit + integration (.NET)
+```powershell
+dotnet test VoiceGarden.UI.Tests\VoiceGarden.UI.Tests.csproj   # floravox integration (skips w/o local models) + sidecar generator units
+.\scripts\test-rust-dll-pin.ps1                                 # CI ships the csproj-pinned rust DLL
+.\scripts\test-rust-abi.ps1                                     # export/ABI canary check incl. negative control
+```
+
 ### Boundary crash test (Grid3 pattern)
 ```powershell
 .\scripts\test-boundary-crash.ps1    # PromptBuilder with rate changes — reproduces Grid3 crash

VoiceGarden.UI.Tests/FloravoxEngineTests.cs

@@ -0,0 +1,172 @@
+// Integration tests for the floravox engine through the shipped NuGet DLL
+// (issue #15 stage 4). These exercise the exact native binary the SAPI
+// adapter loads: RustTtsWrapper.Bindings 0.5.3 with sherpaonnx +
+// floravox-lexicons on win-x64.
+//
+// Boundary/mark timings require a real piper voice. They are skipped when
+// no voice is found under %LOCALAPPDATA%\VoiceGardenSAPIAdapter\models so CI
+// without models still runs the offline checks.
+
+using RustTtsWrapper;
+using Xunit;
+
+namespace VoiceGarden.UI.Tests;
+
+public sealed class FloravoxEngineTests : IDisposable
+{
+    private readonly string? _voiceDir;
+    private readonly string? _kokoroDir;
+
+    public FloravoxEngineTests()
+    {
+        var modelsRoot = Path.Combine(
+            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+            "VoiceGardenSAPIAdapter", "models");
+        if (!Directory.Exists(modelsRoot))
+        {
+            return;
+        }
+        // First piper voice dir with a *.onnx.json sidecar (routing key used
+        // by the C++ adapter — see ModelSupportsFloravox in TTSEngine.cpp).
+        _voiceDir = (from d in Directory.EnumerateDirectories(modelsRoot, "*", SearchOption.AllDirectories)
+                     where Directory.EnumerateFiles(d, "*.onnx.json").Any()
+                     let onnx = Directory.EnumerateFiles(d, "*.onnx").FirstOrDefault()
+                     where onnx != null
+                     select d).FirstOrDefault();
+        // Kokoro voice in pure sherpa layout (model.onnx + tokens.txt +
+        // voices.bin, NO sidecar) — floravox self-serves these.
+        _kokoroDir = (from d in Directory.EnumerateDirectories(modelsRoot, "*", SearchOption.TopDirectoryOnly)
+                      where Path.GetFileName(d).StartsWith("kokoro-en-", StringComparison.OrdinalIgnoreCase)
+                      where Directory.EnumerateFiles(d, "voices.bin", SearchOption.AllDirectories).Any()
+                      where Directory.EnumerateFiles(d, "tokens.txt", SearchOption.AllDirectories).Any()
+                      where Directory.EnumerateFiles(d, "*.onnx", SearchOption.AllDirectories)
+                          .Any(f => !Path.GetFileName(f).Contains("vocoder"))
+                      select Directory.EnumerateDirectories(d).FirstOrDefault(
+                          inner => Directory.EnumerateFiles(inner, "model.onnx").Any()))
+                     .FirstOrDefault();
+    }
+
+    public void Dispose() { }
+
+    private string? ModelsRoot => _voiceDir is null ? null
+        : Path.GetFullPath(Path.Combine(_voiceDir, "..", ".."));
+
+    private Dictionary<string, string> Creds(string modelId, string modelsRoot, bool withLang) =>
+        withLang
+            ? new() { ["modelId"] = modelId, ["modelsDir"] = modelsRoot, ["lang"] = "en" }
+            : new() { ["modelId"] = modelId, ["modelsDir"] = modelsRoot };
+
+    [Fact]
+    public void KokoroSpeaksThroughFloravoxFromSherpaLayout()
+    {
+        // Kokoro ships as model.onnx + tokens.txt + voices.bin with NO
+        // piper sidecar — floravox's KokoroBackend self-serves the layout.
+        // This is the path the C++ adapter now routes kokoro voices through.
+        if (_kokoroDir is null)
+        {
+            return; // no local kokoro voice — skip
+        }
+
+        var modelsRoot = ModelsRoot!.Replace('\\', '/');
+        var modelId = _kokoroDir!.Replace('\\', '/')[(modelsRoot.Length + 1)..];
+        var creds = new Dictionary<string, string>
+        {
+            ["modelId"] = modelId,
+            ["modelsDir"] = modelsRoot,
+            ["lang"] = "en",
+            ["misaki"] = "us",
+        };
+
+        using var client = new TtsClient("floravox", creds);
+        var audioBytes = 0L;
+        var boundaries = new List<string>();
+        client.SetOnAudio(data => audioBytes += data.Length);
+        client.SetOnBoundary((w, _, _, s, e, _) => boundaries.Add(w));
+
+        client.SpeakSync("<speak>Kokoro speaks through floravox with the number 42.</speak>");
+
+        Assert.True(audioBytes > 0, $"no audio produced ({audioBytes} bytes)");
+        Assert.NotEmpty(boundaries);
+    }
+
+    [Fact]
+    public void MissingVoiceSurfacesAsSpeakError()
+    {
+        // Engine construction is lazy (the model is resolved on first
+        // synthesis), so a nonexistent modelId must surface as a speak-time
+        // exception — never a silent success or a crash. The C++ adapter's
+        // sherpaonnx fallback keys off construction failures (engine absent
+        // from the DLL), which this complements.
+        using var c = new TtsClient("floravox", Creds("no-such-voice", ".", withLang: false));
+        Assert.ThrowsAny<Exception>(() => c.SpeakSync("hello"));
+    }
+
+    [Fact]
+    public void FloravoxSpeaksSsmlWithBoundariesAndMarks()
+    {
+        if (_voiceDir is null)
+        {
+            return; // no local voice — skip
+        }
+
+        var modelsRoot = ModelsRoot!.Replace('\\', '/');
+        var modelId = _voiceDir!.Replace('\\', '/')[(modelsRoot.Length + 1)..];
+
+        using var client = new TtsClient("floravox", Creds(modelId, modelsRoot, withLang: true));
+
+        var audioBytes = 0L;
+        var boundaries = new List<(string word, float start, float end, bool estimated)>();
+        var marks = new List<(string name, float start)>();
+
+        client.SetOnAudio(data => audioBytes += data.Length);
+        client.SetOnBoundary((word, _, _, start, end, estimated) =>
+            boundaries.Add((word, start, end, estimated)));
+        client.SetOnMark((name, _, start, _) => marks.Add((name, start)));
+
+        client.SpeakSync("<speak>Hello <mark name='vg1'/>world, floravox measures this.</speak>");
+
+        Assert.True(audioBytes > 0, $"no audio produced ({audioBytes} bytes)");
+        Assert.NotEmpty(boundaries);
+
+        // Timings are monotonic and land inside the audio (they are scaled to
+        // the synthesized length even for unpatched voices; voices patched
+        // with floravox's duration-graph surgery additionally report
+        // estimated == false).
+        var starts = boundaries.Select(b => b.start).ToList();
+        Assert.Equal(starts.OrderBy(s => s), starts);
+        Assert.All(boundaries, b => Assert.True(b.start >= 0 && b.end > b.start));
+
+        // The bookmark must fire as a mark event (mapped to SPEI_TTSBOOKMARK
+        // by the C++ adapter).
+        Assert.Contains(marks, m => m.name == "vg1");
+    }
+
+    [Fact]
+    public void LexiconG2pHandlesOovWord()
+    {
+        // "floravox" is not in a gruut lexicon; the OOV chain (lexicon →
+        // Phonetisaurus → ByT5 → spell) must still produce audio without
+        // failing. Needs the lang bundle, which may require network on first
+        // run — treated as skip if synthesis fails offline.
+        if (_voiceDir is null)
+        {
+            return;
+        }
+
+        var modelsRoot = ModelsRoot!.Replace('\\', '/');
+        var modelId = _voiceDir!.Replace('\\', '/')[(modelsRoot.Length + 1)..];
+
+        using var client = new TtsClient("floravox", Creds(modelId, modelsRoot, withLang: true));
+        var audioBytes = 0L;
+        client.SetOnAudio(data => audioBytes += data.Length);
+        try
+        {
+            client.SpeakSync("<speak>The floravox vocalizes.</speak>");
+            Assert.True(audioBytes > 0, $"no audio for OOV sentence ({audioBytes} bytes)");
+        }
+        catch (Exception ex) when (ex.Message.Contains("lexicon", StringComparison.OrdinalIgnoreCase))
+        {
+            // First-run bundle fetch offline — acceptable skip.
+        }
+    }
+}