Add voice input (Whisper STT) and read-aloud (browser TTS)

Mic button records and transcribes speech via OpenAI's Whisper API into
the composer; assistant replies can be read aloud per-message or
automatically, using the OS's own TTS voices (no API key needed for
read-aloud). Both are interruptible mid-action.

Author: voidstackloop

README.md

@@ -56,6 +56,12 @@ Beyond chat, Modelforge includes an **agentic mode** — the model can read/writ
 - Data export/import, and one-click "copy diagnostic info" for bug reports.
 - **Updates** — packaged builds check GitHub Releases for new versions automatically on launch, plus a manual "Check for updates" button in Settings (also available from the app menu).
 
+**Voice**
+- **Voice input** — record a question with the mic button; it's transcribed via OpenAI's Whisper API and dropped into the composer (requires an OpenAI API key in Settings, even when chatting with a local Ollama model).
+- **Read aloud** — any assistant reply can be played back through your OS's own text-to-speech voices, with a per-message speaker button, an optional "auto-read every response" toggle, and a voice picker with a test button in Settings → Voice. Works fully offline, no API key needed.
+- Both are start/stop/cancel controllable mid-action — stop a reply from being read, or cancel a recording before it's sent for transcription.
+- Not included: fully real-time, bidirectional voice conversation (speaking over the model and having it react instantly, à la OpenAI's Realtime API). That's a different streaming architecture and hasn't been built.
+
 ## Screenshots
 
 <details>

app/src/main.ts

@@ -275,6 +275,27 @@ function registerIpcHandlers(): void {
         secretsStore.setSecret(requireString(key, "secret key"), value ?? "")
     );
 
+    ipcMain.handle(
+        "audio:transcribe",
+        async (_event: IpcMainInvokeEvent, { audioBase64, mimeType }: { audioBase64: string; mimeType: string }) => {
+            requireString(audioBase64, "audio data");
+            const apiKey = secretsStore.getSecret("openai_api_key");
+            if (!apiKey) {
+                return { error: "Voice input needs an OpenAI API key — add one in Settings to use it." };
+            }
+            try {
+                const buffer = Buffer.from(audioBase64, "base64");
+                const ext = mimeType.includes("webm") ? "webm" : mimeType.includes("ogg") ? "ogg" : "wav";
+                const text = await openaiProvider.transcribeAudio(apiKey, buffer, `audio.${ext}`);
+                return { text };
+            } catch (err) {
+                const error = err as Error;
+                logger.error(`Audio transcription failed: ${error.message}`);
+                return { error: error.message };
+            }
+        }
+    );
+
     ipcMain.handle("app:setBusy", (_event: IpcMainInvokeEvent, busy: boolean) => {
         isBusy = busy;
     });

app/src/preload.ts

@@ -85,6 +85,11 @@ contextBridge.exposeInMainWorld("api", {
         set: (key: string, value: string) => ipcRenderer.invoke("secrets:set", { key, value }),
     },
 
+    audio: {
+        transcribe: (audioBase64: string, mimeType: string): Promise<{ text?: string; error?: string }> =>
+            ipcRenderer.invoke("audio:transcribe", { audioBase64, mimeType }),
+    },
+
     app: {
         setBusy: (busy: boolean) => ipcRenderer.invoke("app:setBusy", busy),
         getVersion: () => ipcRenderer.invoke("app:getVersion"),

app/src/providers/openai.ts

@@ -2,6 +2,33 @@ import { streamSSE } from "./sse";
 import { describeHttpError, describeNetworkError } from "./errors";
 import type { ChatFn, ToolDefinition } from "./types";
 
+// Speech-to-text for voice input. Whisper's API is a separate REST endpoint
+// from chat completions (multipart file upload, not JSON), so it doesn't fit
+// the streaming ChatFn shape above.
+export async function transcribeAudio(apiKey: string, audioBuffer: Buffer, filename: string): Promise<string> {
+    const form = new FormData();
+    form.append("file", new Blob([new Uint8Array(audioBuffer)]), filename);
+    form.append("model", "whisper-1");
+
+    let res: Response;
+    try {
+        res = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+            method: "POST",
+            headers: { Authorization: `Bearer ${apiKey}` },
+            body: form,
+        });
+    } catch (err) {
+        throw describeNetworkError("OpenAI", err);
+    }
+
+    if (!res.ok) {
+        throw new Error(await describeHttpError(res, "OpenAI"));
+    }
+
+    const data = (await res.json()) as { text?: string };
+    return data.text ?? "";
+}
+
 function toOpenAiTools(tools: ToolDefinition[]): unknown[] {
     return tools.map((t) => ({
         type: "function",

app/src/settings-store.ts

@@ -38,6 +38,11 @@ export interface AppSettings {
     promptPresets: PromptPreset[];
     theme: "light" | "dark" | "system";
     language: "en" | "tr";
+    // Text-to-speech: which browser/OS voice to use (voiceURI from
+    // speechSynthesis.getVoices(), chosen client-side) and whether assistant
+    // responses should be read aloud automatically as they finish.
+    ttsVoiceURI?: string;
+    ttsAutoRead?: boolean;
 }
 
 const DEFAULTS: AppSettings = {

frontend/src/lib/translations.ts

@@ -80,6 +80,17 @@ export interface Dictionary {
     noPreviousVersions: string;
     savePreset: string;
     promptLibraryVariablesHint: string;
+    startRecording: string;
+    stopRecording: string;
+    cancelRecording: string;
+    transcribing: string;
+    ttsSection: string;
+    ttsAutoRead: string;
+    ttsVoice: string;
+    ttsVoiceDefault: string;
+    ttsVoiceTest: string;
+    enabled: string;
+    disabled: string;
     resetToDefault: string;
     usingCustomPrompt: string;
     dataManagement: string;
@@ -198,6 +209,17 @@ export const en: Dictionary = {
     noPreviousVersions: "No previous versions yet.",
     savePreset: "Save",
     promptLibraryVariablesHint: "Add {{variables}} to a prompt (e.g. {{topic}}) and you'll be asked to fill them in each time you apply it. Edits keep version history so you can undo a change.",
+    startRecording: "Start voice input",
+    stopRecording: "Stop and transcribe",
+    cancelRecording: "Cancel recording",
+    transcribing: "Transcribing...",
+    ttsSection: "Voice",
+    ttsAutoRead: "Automatically read responses aloud",
+    ttsVoice: "Voice",
+    ttsVoiceDefault: "System default",
+    ttsVoiceTest: "Test",
+    enabled: "Enabled",
+    disabled: "Disabled",
     resetToDefault: "Reset to default",
     usingCustomPrompt: "Custom prompt for this chat",
     dataManagement: "Data management",
@@ -324,6 +346,17 @@ export const tr: Dictionary = {
     noPreviousVersions: "Henüz önceki bir sürüm yok.",
     savePreset: "Kaydet",
     promptLibraryVariablesHint: "Bir isteme {{değişkenler}} ekleyin (ör. {{konu}}) — her uyguladığınızda bunları doldurmanız istenir. Düzenlemeler sürüm geçmişini korur, böylece bir değişikliği geri alabilirsiniz.",
+    startRecording: "Sesli girişi başlat",
+    stopRecording: "Durdur ve yazıya dök",
+    cancelRecording: "Kaydı iptal et",
+    transcribing: "Yazıya dökülüyor...",
+    ttsSection: "Ses",
+    ttsAutoRead: "Yanıtları otomatik olarak sesli oku",
+    ttsVoice: "Ses",
+    ttsVoiceDefault: "Sistem varsayılanı",
+    ttsVoiceTest: "Test et",
+    enabled: "Etkin",
+    disabled: "Kapalı",
     resetToDefault: "Varsayılana dön",
     usingCustomPrompt: "Bu sohbet için özel istem",
     dataManagement: "Veri yönetimi",

frontend/src/lib/tts.ts

@@ -0,0 +1,20 @@
+// Text-to-speech via the browser's native SpeechSynthesis API — works
+// offline using whatever voices the OS provides (no API key, no network
+// call), consistent with this app's local-first approach. Electron's
+// Chromium supports this directly since it doesn't depend on the Google
+// cloud speech services that plain SpeechRecognition (STT) would need.
+export function speakText(text: string, voiceURI: string | undefined, onEnd: () => void): void {
+    window.speechSynthesis.cancel();
+    const utterance = new SpeechSynthesisUtterance(text);
+    if (voiceURI) {
+        const voice = window.speechSynthesis.getVoices().find((v) => v.voiceURI === voiceURI);
+        if (voice) utterance.voice = voice;
+    }
+    utterance.onend = onEnd;
+    utterance.onerror = onEnd;
+    window.speechSynthesis.speak(utterance);
+}
+
+export function stopSpeaking(): void {
+    window.speechSynthesis.cancel();
+}