add local Nemotron speech recognition

ts/packages/chat-ui/src/chatPanel.ts

@@ -1417,7 +1417,10 @@ export class ChatPanel {
             this.speechState === "always-on" ||
             this.speechState === "wake-word";
         this.micButton.classList.toggle("listening", listening);
-        this.micButton.disabled = this.speechState === "disabled";
+        this.micButton.disabled =
+            this.speechState === "disabled" ||
+            this.speechState === "preparing" ||
+            this.speechState === "finalizing";
         // Filled mic glyph; the `listening` class drives the pulse styling.
         this.micButton.innerHTML = `<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 24 24"><path d="M12 14a3 3 0 0 0 3-3V5a3 3 0 1 0-6 0v6a3 3 0 0 0 3 3m5-3a5 5 0 0 1-10 0H5a7 7 0 0 0 6 6.92V21h2v-3.08A7 7 0 0 0 19 11z"/></svg>`;
     }
@@ -1428,7 +1431,16 @@ export class ChatPanel {
             this.speechState === "listening" ||
             this.speechState === "always-on" ||
             this.speechState === "wake-word";
-        if (listening) {
+        if (
+            this.speechState === "preparing" ||
+            this.speechState === "finalizing"
+        ) {
+            this.voiceBanner.textContent =
+                this.speechState === "preparing"
+                    ? "Preparing speech recognition..."
+                    : "Finishing transcription...";
+            this.voiceBanner.style.display = "";
+        } else if (listening) {
             this.voiceBanner.textContent =
                 this.speechState === "wake-word"
                     ? "Waiting for wake word…"

ts/packages/chat-ui/src/providers.ts

@@ -17,6 +17,8 @@ export type SpeechState =
     | "idle"
     | "listening"
     | "disabled"
+    | "preparing"
+    | "finalizing"
     | "always-on"
     | "wake-word";
 

ts/packages/chat-ui/test/speechInput.spec.ts

@@ -0,0 +1,61 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { afterEach, expect, it, jest } from "@jest/globals";
+import { ChatPanel } from "../src/chatPanel.js";
+import type { SpeechInputProvider, SpeechState } from "../src/providers.js";
+
+let panel: ChatPanel | undefined;
+afterEach(() => {
+    panel?.dispose();
+    document.body.innerHTML = "";
+});
+
+it("shows actual capture/finalizing state and replaces streaming textbox hypotheses without submitting", () => {
+    let stateChanged: (state: SpeechState) => void = () => {};
+    let result: (text: string, final: boolean) => void = () => {};
+    const provider: SpeechInputProvider = {
+        getState: () => "idle",
+        start: () => {},
+        stop: () => {},
+        setContinuous: () => {},
+        onStateChange: (callback) => {
+            stateChanged = callback;
+        },
+        onResult: (callback) => {
+            result = callback;
+        },
+    };
+    const root = document.createElement("div");
+    document.body.appendChild(root);
+    const onSend = jest.fn();
+    panel = new ChatPanel(root, {
+        platformAdapter: { handleLinkClick() {} },
+        speechProvider: provider,
+        onSend,
+    });
+    const mic = root.querySelector<HTMLButtonElement>(".chat-mic-button")!;
+    const banner = root.querySelector<HTMLElement>(".chat-voice-banner")!;
+    const input = root.querySelector<HTMLElement>("#phraseDiv")!;
+
+    stateChanged("preparing");
+    expect(mic.disabled).toBe(true);
+    expect(mic.classList.contains("listening")).toBe(false);
+    expect(banner.textContent).toBe("Preparing speech recognition...");
+    stateChanged("listening");
+    expect(mic.disabled).toBe(false);
+    expect(mic.classList.contains("listening")).toBe(true);
+    result("turn", false);
+    expect(input.textContent).toBe("turn");
+    result("turn on the lights", false);
+    expect(input.textContent).toBe("turn on the lights");
+    stateChanged("finalizing");
+    expect(mic.classList.contains("listening")).toBe(false);
+    expect(banner.textContent).toBe("Finishing transcription...");
+    result("Turn on the lights.", true);
+    stateChanged("idle");
+    expect(input.textContent).toBe("Turn on the lights.");
+    expect(mic.disabled).toBe(false);
+    expect(banner.style.display).toBe("none");
+    expect(onSend).not.toHaveBeenCalled();
+});

ts/packages/shell/README.md

@@ -105,7 +105,7 @@ To read the chat view from another device, such as a phone on a trusted home net
 
 ### Azure Speech to Text service (Optional)
 
-Currently, TypeAgent Shell optionally supports voice input via Azure Speech Services or [Local Whisper Service](../../../python/stt/whisperService/) in addition to keyboard input.
+Voice input supports Azure (default), Python [Whisper](../../../python/stt/whisperService/), and Windows x64 Nemotron. Select with `@shell speechProvider azure|whisper|nemotron`; Nemotron downloads on first selection.
 
 To set up Azure [Speech to Text service](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/index-speech-to-text), the following variables in `config.local.yaml` (under `speech`) or the legacy `.env` are needed:
 

ts/packages/shell/electron-builder.config.js

@@ -95,6 +95,7 @@ export default {
 
     // Don't need to install
     npmRebuild: false,
+    asarUnpack: ["node_modules/**/foundry-local-sdk/**/*"],
     artifactName: name + "-${version}-${platform}-${arch}.${ext}",
     win: {
         signtoolOptions: {

ts/packages/shell/electron.vite.config.mts

@@ -35,6 +35,15 @@ export default defineConfig({
         ],
         build: {
             sourcemap: true,
+            rollupOptions: {
+                input: {
+                    index: resolve(__dirname, "src/main/index.ts"),
+                    localAsrWorker: resolve(
+                        __dirname,
+                        "src/main/localAsrWorker.ts",
+                    ),
+                },
+            },
         },
     },
     preload: {

ts/packages/shell/package.json

@@ -49,7 +49,8 @@
     "start:package:linux": "./dist/linux-unpacked/typeagentshell",
     "start:package:macos": "open ./dist/mac-arm64/TypeAgent\\ Shell.app",
     "start:package:win32": ".\\dist\\win-unpacked\\typeagentshell.exe",
-    "test:local": "pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"",
+    "test:asr": "tsx --test test/unit/localAsr.test.mts",
+    "test:local": "pnpm run test:asr && pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"",
     "tsc:model": "tsc -b src test",
     "typecheck": "concurrently npm:typecheck:node npm:typecheck:web",
     "typecheck:node": "tsc -p tsconfig.node.json",
@@ -82,6 +83,7 @@
     "dompurify": "^3.4.17",
     "dotenv": "^16.3.1",
     "electron-updater": "^6.8.9",
+    "foundry-local-sdk": "2.1.0",
     "jose": "^5.9.6",
     "js-yaml": "^4.3.2",
     "markdown-it": "^14.3.2",

ts/packages/shell/playwright.config.ts

@@ -17,7 +17,7 @@ import { defineConfig, devices } from "@playwright/test";
 export default defineConfig({
     testDir: "./test",
     /* Exclude Jest unit tests from Playwright test discovery */
-    testIgnore: /\/partialCompletion\//,
+    testIgnore: /\/(partialCompletion|unit)\//,
     /* Run tests sequentially otherwise the client will complain about locked session file */
     fullyParallel: false,
     /* Fail the build on CI if you accidentally left test.only in the source code. */

ts/packages/shell/src/main/agent.ts

@@ -23,6 +23,10 @@ import {
     displayWarn,
 } from "@typeagent/agent-sdk/helpers/display";
 import { getLocalWhisperCommandHandlers } from "./localWhisperCommandHandler.js";
+import {
+    getSpeechProviderCommandHandlers,
+    selectSpeechProvider,
+} from "./speechProvider.js";
 import { ShellWindow } from "./shellWindow.js";
 import {
     getObjectProperty,
@@ -388,7 +392,8 @@ const handlers: CommandHandlerTable = {
             },
         },
         topmost: new ShellSetTopMostCommandHandler(),
-        localWhisper: getLocalWhisperCommandHandlers(),
+        localWhisper: getLocalWhisperCommandHandlers(selectSpeechProvider),
+        speechProvider: getSpeechProviderCommandHandlers(),
         theme: getThemeCommandHandlers(),
         voice: getVoiceCommandHandlers(),
         update: updateHandlerTable,

ts/packages/shell/src/main/index.ts

@@ -19,6 +19,7 @@ import {
 } from "./shellSettings.js";
 import { readFileSync } from "node:fs";
 import { closeLocalWhisper } from "./localWhisperCommandHandler.js";
+import { localAsrHost } from "./localAsrHost.js";
 import { getInstanceDir } from "agent-dispatcher/helpers/data";
 import {
     initializeInstance,
@@ -465,6 +466,7 @@ function initializeQuit() {
         globalShortcut.unregisterAll();
 
         closeLocalWhisper();
+        await localAsrHost.dispose();
 
         debugShellCleanup("Closing instance");
 

ts/packages/shell/src/main/localAsrHost.ts

@@ -0,0 +1,200 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+    app,
+    dialog,
+    utilityProcess,
+    type UtilityProcess,
+    type WebContents,
+} from "electron";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import type {
+    LocalAsrCommand,
+    LocalAsrEvent,
+    LocalAsrWorkerEvent,
+} from "../preload/localAsrTypes.js";
+
+class LocalAsrHost {
+    private worker?: UtilityProcess;
+    private stoppingWorker?: UtilityProcess;
+    private requestId = 0;
+    private pending = new Map<
+        number,
+        { resolve: () => void; reject: (error: Error) => void }
+    >();
+    private owner?: { sender: WebContents; id: string; release: () => void };
+    private progress?: (message: string, percent?: number) => void;
+
+    private getWorker() {
+        if (this.worker) return this.worker;
+        const worker = utilityProcess.fork(
+            fileURLToPath(new URL("./localAsrWorker.js", import.meta.url)),
+            [],
+            { serviceName: "TypeAgent local speech", stdio: "pipe" },
+        );
+        worker.stderr?.on("data", (data: Buffer) =>
+            console.error(`[local-speech] ${data}`),
+        );
+        worker.on("message", (event: LocalAsrWorkerEvent) => {
+            if (event.type === "reply") {
+                const pending = this.pending.get(event.requestId);
+                this.pending.delete(event.requestId);
+                if (event.error) pending?.reject(new Error(event.error));
+                else pending?.resolve();
+            } else if (event.type === "progress") {
+                this.progress?.(event.message, event.percent);
+            } else if (
+                this.owner?.id === event.sessionId &&
+                !this.owner.sender.isDestroyed()
+            ) {
+                this.owner.sender.send("local-asr-event", event);
+            }
+        });
+        worker.on("exit", (code) => {
+            if (this.worker !== worker) return;
+            const stopped = this.stoppingWorker === worker;
+            this.worker = undefined;
+            this.stoppingWorker = undefined;
+            const error = new Error(
+                stopped
+                    ? "Local speech was stopped."
+                    : `Local speech worker exited (code ${code}). Select Nemotron again to restart it.`,
+            );
+            for (const pending of this.pending.values()) pending.reject(error);
+            this.pending.clear();
+            const owner = this.owner;
+            this.releaseOwner();
+            if (owner && !owner.sender.isDestroyed()) {
+                owner.sender.send("local-asr-event", {
+                    type: "error",
+                    sessionId: owner.id,
+                    message: error.message,
+                } satisfies LocalAsrEvent);
+            }
+            if (!stopped) console.error(error);
+        });
+        this.worker = worker;
+        return worker;
+    }
+
+    private request(command: LocalAsrCommand) {
+        const worker = this.getWorker();
+        const requestId = ++this.requestId;
+        return new Promise<void>((resolve, reject) => {
+            this.pending.set(requestId, { resolve, reject });
+            try {
+                worker.postMessage({ ...command, requestId });
+            } catch (error) {
+                this.pending.delete(requestId);
+                reject(error);
+            }
+        });
+    }
+
+    async prepare(
+        allowDownload: boolean,
+        progress?: (message: string, percent?: number) => void,
+    ) {
+        this.progress = progress;
+        try {
+            await this.request({
+                type: "prepare",
+                cacheDir: path.join(app.getPath("userData"), "local-speech"),
+                allowDownload,
+            });
+        } finally {
+            this.progress = undefined;
+        }
+    }
+
+    cancelSetup() {
+        return this.request({ type: "cancelSetup" });
+    }
+
+    async start(sender: WebContents) {
+        if (this.owner) throw new Error("Local speech is already recording.");
+        await this.prepare(false);
+        if (sender.isDestroyed())
+            throw new Error("The speech window was closed.");
+        if (this.owner) throw new Error("Local speech is already recording.");
+        const id = randomUUID();
+        const cancel = () => {
+            void this.end(sender, id, true).catch((error) =>
+                console.error("Failed to cancel local speech", error),
+            );
+        };
+        sender.once("destroyed", cancel);
+        sender.once("did-start-navigation", cancel);
+        this.owner = {
+            sender,
+            id,
+            release: () => {
+                sender.removeListener("destroyed", cancel);
+                sender.removeListener("did-start-navigation", cancel);
+            },
+        };
+        try {
+            await this.request({ type: "start", sessionId: id });
+            return id;
+        } catch (error) {
+            this.releaseOwner();
+            throw error;
+        }
+    }
+
+    async append(sender: WebContents, id: string, pcm: Uint8Array) {
+        this.checkOwner(sender, id);
+        await this.request({ type: "append", sessionId: id, pcm });
+    }
+
+    async end(sender: WebContents, id: string, cancel: boolean) {
+        this.checkOwner(sender, id);
+        try {
+            await this.request({
+                type: cancel ? "cancel" : "finish",
+                sessionId: id,
+            });
+        } finally {
+            if (this.owner?.id === id) this.releaseOwner();
+        }
+    }
+
+    private checkOwner(sender: WebContents, id: string) {
+        if (this.owner?.sender !== sender || this.owner.id !== id) {
+            throw new Error(
+                "The speech session does not belong to this window.",
+            );
+        }
+    }
+
+    private releaseOwner() {
+        this.owner?.release();
+        this.owner = undefined;
+    }
+
+    isRecording() {
+        return this.owner !== undefined;
+    }
+
+    async dispose() {
+        const worker = this.worker;
+        this.releaseOwner();
+        if (worker) {
+            this.stoppingWorker = worker;
+            await new Promise<void>((resolve) => {
+                worker.once("exit", () => resolve());
+                worker.kill();
+            });
+        }
+    }
+
+    reportError(message: string) {
+        console.error("[local-speech]", message);
+        dialog.showErrorBox("Speech recognition", message);
+    }
+}
+
+export const localAsrHost = new LocalAsrHost();