From 0cd041dc1a2866e714c9200bbb4ced88d79e145f Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:17:10 +0100 Subject: [PATCH 01/10] feat(speechify): add Speechify TTS plugin Add @livekit/agents-plugin-speechify: streaming text-to-speech with word-level timestamps via the official @speechify/api SDK. Chunks streamed input into sequential /audio/speech calls, emitting audio and timed transcripts per sentence (streaming + alignedTranscript). Maintained by Speechify (Speechify-AI/livekit-plugin-js). --- README.md | 1 + plugins/speechify/CHANGELOG.md | 1 + plugins/speechify/README.md | 21 ++ plugins/speechify/api-extractor.json | 20 ++ plugins/speechify/package.json | 52 +++++ plugins/speechify/src/index.ts | 19 ++ plugins/speechify/src/models.ts | 5 + plugins/speechify/src/tts.ts | 279 +++++++++++++++++++++++++++ plugins/speechify/tsconfig.json | 16 ++ plugins/speechify/tsup.config.ts | 7 + pnpm-lock.yaml | 48 ++++- turbo.json | 1 + 12 files changed, 463 insertions(+), 7 deletions(-) create mode 100644 plugins/speechify/CHANGELOG.md create mode 100644 plugins/speechify/README.md create mode 100644 plugins/speechify/api-extractor.json create mode 100644 plugins/speechify/package.json create mode 100644 plugins/speechify/src/index.ts create mode 100644 plugins/speechify/src/models.ts create mode 100644 plugins/speechify/src/tts.ts create mode 100644 plugins/speechify/tsconfig.json create mode 100644 plugins/speechify/tsup.config.ts diff --git a/README.md b/README.md index e708dcf6e..d3aa9b27f 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Currently, only the following plugins are supported: | [@livekit/agents-plugin-resemble](https://www.npmjs.com/package/@livekit/agents-plugin-resemble) | TTS | | [@livekit/agents-plugin-rime](https://www.npmjs.com/package/@livekit/agents-plugin-rime) | TTS | | [@livekit/agents-plugin-inworld](https://www.npmjs.com/package/@livekit/agents-plugin-inworld) | STT, TTS | +| [@livekit/agents-plugin-speechify](https://www.npmjs.com/package/@livekit/agents-plugin-speechify) | TTS | | [@livekit/agents-plugin-silero](https://www.npmjs.com/package/@livekit/agents-plugin-silero) | VAD | | [@livekit/agents-plugin-livekit](https://www.npmjs.com/package/@livekit/agents-plugin-livekit) | EOU | | [@livekit/agents-plugin-anam](https://www.npmjs.com/package/@livekit/agents-plugin-anam) | Avatar | diff --git a/plugins/speechify/CHANGELOG.md b/plugins/speechify/CHANGELOG.md new file mode 100644 index 000000000..62cf44533 --- /dev/null +++ b/plugins/speechify/CHANGELOG.md @@ -0,0 +1 @@ +# @livekit/agents-plugin-speechify diff --git a/plugins/speechify/README.md b/plugins/speechify/README.md new file mode 100644 index 000000000..4fe5f6dcf --- /dev/null +++ b/plugins/speechify/README.md @@ -0,0 +1,21 @@ + +# Speechify plugin for LiveKit Agents + +The Agents Framework is designed for building realtime, programmable +participants that run on servers. Use it to create conversational, multi-modal +voice agents that can see, hear, and understand. + +This package contains the Speechify plugin, which provides streaming text-to-speech +with word-level timestamps. Refer to the [documentation](https://docs.livekit.io/agents/overview/) +for information on how to use it, or browse the [API +reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_speechify.html). +See the [repository](https://github.com/livekit/agents-js) for more information +about the framework as a whole. + +This plugin is maintained by Speechify. Issues and contributions specific to the +plugin are welcome at +[Speechify-AI/livekit-plugin-js](https://github.com/Speechify-AI/livekit-plugin-js). diff --git a/plugins/speechify/api-extractor.json b/plugins/speechify/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/speechify/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/speechify/package.json b/plugins/speechify/package.json new file mode 100644 index 000000000..fbf45fcac --- /dev/null +++ b/plugins/speechify/package.json @@ -0,0 +1,52 @@ +{ + "name": "@livekit/agents-plugin-speechify", + "version": "1.5.0", + "description": "Speechify plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "Speechify", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-openai": "workspace:*", + "@livekit/agents-plugins-test": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "@speechify/api": "^2.0.0" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/speechify/src/index.ts b/plugins/speechify/src/index.ts new file mode 100644 index 000000000..252e901dd --- /dev/null +++ b/plugins/speechify/src/index.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export * from './models.js'; +export * from './tts.js'; + +class SpeechifyPlugin extends Plugin { + constructor() { + super({ + title: 'speechify', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new SpeechifyPlugin()); diff --git a/plugins/speechify/src/models.ts b/plugins/speechify/src/models.ts new file mode 100644 index 000000000..8c4b54a69 --- /dev/null +++ b/plugins/speechify/src/models.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: 2025 Speechify, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +export type TTSModels = 'simba-english' | 'simba-multilingual' | 'simba-3.0'; diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts new file mode 100644 index 000000000..032119e67 --- /dev/null +++ b/plugins/speechify/src/tts.ts @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: 2025 Speechify, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APIStatusError, + AudioByteStream, + USERDATA_TIMED_TRANSCRIPT, + createTimedString, + shortuuid, + tokenize, + tts, +} from '@livekit/agents'; +import type { Speechify } from '@speechify/api'; +import { SpeechifyClient, SpeechifyError } from '@speechify/api'; +import type { TTSModels } from './models.js'; + +const NUM_CHANNELS = 1; +const SAMPLE_RATE = 24000; +const AUDIO_FORMAT: Speechify.GetSpeechRequest.AudioFormat = 'pcm'; +const DEFAULT_VOICE_ID = 'jack'; + +export interface TTSOptions { + voiceId: string; + model?: TTSModels; + language?: string; + loudnessNormalization?: boolean; + textNormalization?: boolean; + apiKey?: string; + baseUrl?: string; + client?: SpeechifyClient; + tokenizer?: tokenize.SentenceTokenizer; +} + +const defaultOptions = (): Omit => ({ + voiceId: DEFAULT_VOICE_ID, +}); + +const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeechRequest => { + const request: Speechify.GetSpeechRequest = { + audio_format: AUDIO_FORMAT, + input: text, + voice_id: opts.voiceId, + }; + if (opts.model) request.model = opts.model; + if (opts.language) request.language = opts.language; + if (opts.loudnessNormalization !== undefined || opts.textNormalization !== undefined) { + request.options = { + loudness_normalization: opts.loudnessNormalization, + text_normalization: opts.textNormalization, + }; + } + return request; +}; + +const toError = (e: unknown): Error => { + if (e instanceof SpeechifyError) { + return new APIStatusError({ message: e.message, options: { statusCode: e.statusCode ?? -1 } }); + } + return new APIConnectionError({ message: e instanceof Error ? e.message : String(e) }); +}; + +export class TTS extends tts.TTS { + label = 'speechify.TTS'; + #opts: TTSOptions; + #client: SpeechifyClient; + #tokenizer: tokenize.SentenceTokenizer; + + /** + * Create a new instance of Speechify TTS. + * + * @remarks + * `apiKey` must be set, either via the constructor or the `SPEECHIFY_API_KEY` + * environment variable. Pass a preconfigured `client` to reuse an existing + * `SpeechifyClient` (in which case `apiKey`/`baseUrl` are ignored). + * + * Synthesis uses the Speechify `/audio/speech` endpoint, which returns raw PCM + * (24 kHz mono) plus word-level speech marks. `stream()` chunks input into + * sentences and issues one request per sentence, emitting audio and aligned + * word timestamps as each sentence completes. + */ + constructor(opts: Partial = {}) { + const merged = { ...defaultOptions(), ...opts }; + + super(SAMPLE_RATE, NUM_CHANNELS, { streaming: true, alignedTranscript: true }); + + this.#opts = merged; + this.#tokenizer = merged.tokenizer ?? new tokenize.basic.SentenceTokenizer(); + + if (merged.client) { + this.#client = merged.client; + } else { + const apiKey = merged.apiKey ?? process.env.SPEECHIFY_API_KEY; + if (!apiKey) { + throw new Error( + 'Speechify API key is required, whether as an argument or as $SPEECHIFY_API_KEY', + ); + } + this.#client = new SpeechifyClient({ apiKey, baseUrl: merged.baseUrl }); + } + } + + get model(): string { + return this.#opts.model ?? 'unknown'; + } + + get provider(): string { + return 'Speechify'; + } + + get client(): SpeechifyClient { + return this.#client; + } + + get options(): TTSOptions { + return this.#opts; + } + + get tokenizer(): tokenize.SentenceTokenizer { + return this.#tokenizer; + } + + updateOptions(opts: Partial>) { + this.#opts = { ...this.#opts, ...opts }; + } + + synthesize(text: string, connOptions?: APIConnectOptions): tts.ChunkedStream { + return new ChunkedStream(this, text, this.#opts, connOptions); + } + + stream(options?: { connOptions?: APIConnectOptions }): tts.SynthesizeStream { + return new SynthesizeStream(this, this.#opts, options?.connOptions); + } +} + +const timedStringsFromMarks = ( + marks: Speechify.SpeechMarks | undefined, + offsetSeconds: number, +): ReturnType[] => { + if (!marks?.chunks) return []; + const out: ReturnType[] = []; + for (const chunk of marks.chunks) { + if (!chunk.value || chunk.start_time === undefined) continue; + out.push( + createTimedString({ + text: chunk.value, + startTime: chunk.start_time / 1000 + offsetSeconds, + endTime: chunk.end_time !== undefined ? chunk.end_time / 1000 + offsetSeconds : undefined, + }), + ); + } + return out; +}; + +export class ChunkedStream extends tts.ChunkedStream { + label = 'speechify.ChunkedStream'; + #opts: TTSOptions; + #tts: TTS; + #timeoutInSeconds?: number; + + constructor(ttsInstance: TTS, text: string, opts: TTSOptions, connOptions?: APIConnectOptions) { + super(text, ttsInstance, connOptions); + this.#tts = ttsInstance; + this.#opts = opts; + this.#timeoutInSeconds = + connOptions?.timeoutMs !== undefined ? connOptions.timeoutMs / 1000 : undefined; + } + + protected async run() { + const requestId = shortuuid(); + const bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS); + + try { + const response = await this.#tts.client.audio.speech( + buildSpeechRequest(this.inputText, this.#opts), + { abortSignal: this.abortSignal, timeoutInSeconds: this.#timeoutInSeconds }, + ); + + const audio = Buffer.from(response.audio_data, 'base64'); + const timed = timedStringsFromMarks(response.speech_marks, 0); + let attached = false; + + const putFrames = (frames: ReturnType) => { + for (const frame of frames) { + if (!attached && timed.length > 0) { + frame.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; + attached = true; + } + this.queue.put({ requestId, frame, final: false, segmentId: requestId }); + } + }; + + putFrames(bstream.write(audio)); + putFrames(bstream.flush()); + this.queue.close(); + } catch (e) { + if (!this.queue.closed) this.queue.close(); + if (this.abortSignal.aborted) return; + throw toError(e); + } + } +} + +export class SynthesizeStream extends tts.SynthesizeStream { + label = 'speechify.SynthesizeStream'; + #opts: TTSOptions; + #tts: TTS; + + constructor(ttsInstance: TTS, opts: TTSOptions, connOptions?: APIConnectOptions) { + super(ttsInstance, connOptions); + this.#tts = ttsInstance; + this.#opts = opts; + } + + protected async run() { + const sentenceStream = this.#tts.tokenizer.stream(); + let cumulativeDuration = 0; + + const forwardInput = async () => { + for await (const input of this.input) { + if (this.abortController.signal.aborted) break; + if (input === SynthesizeStream.FLUSH_SENTINEL) { + sentenceStream.flush(); + } else { + sentenceStream.pushText(input); + } + } + sentenceStream.endInput(); + sentenceStream.close(); + }; + + const synthesizeSentence = async (text: string) => { + const requestId = shortuuid(); + const bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS); + + const response = await this.#tts.client.audio.speech(buildSpeechRequest(text, this.#opts), { + abortSignal: this.abortSignal, + timeoutInSeconds: this.connOptions.timeoutMs / 1000, + }); + + const audio = Buffer.from(response.audio_data, 'base64'); + const timed = timedStringsFromMarks(response.speech_marks, cumulativeDuration); + let attached = false; + + const putFrames = (frames: ReturnType) => { + for (const frame of frames) { + if (!attached && timed.length > 0) { + frame.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; + attached = true; + } + cumulativeDuration += frame.samplesPerChannel / frame.sampleRate; + this.queue.put({ requestId, frame, final: false, segmentId: requestId }); + } + }; + + putFrames(bstream.write(audio)); + putFrames(bstream.flush()); + }; + + const consume = async () => { + for await (const ev of sentenceStream) { + if (this.abortController.signal.aborted) break; + const text = ev.token.trim(); + if (!text) continue; + await synthesizeSentence(text); + } + this.queue.put(SynthesizeStream.END_OF_STREAM); + }; + + try { + await Promise.all([forwardInput(), consume()]); + } catch (e) { + if (!this.queue.closed) this.queue.close(); + if (this.abortSignal.aborted) return; + throw toError(e); + } + } +} diff --git a/plugins/speechify/tsconfig.json b/plugins/speechify/tsconfig.json new file mode 100644 index 000000000..eed8cb31e --- /dev/null +++ b/plugins/speechify/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-speechify", + "entryPointStrategy": "resolve", + "readme": "none", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/speechify/tsup.config.ts b/plugins/speechify/tsup.config.ts new file mode 100644 index 000000000..8ca20961f --- /dev/null +++ b/plugins/speechify/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f01e7345..10262223b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1327,6 +1327,34 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/speechify: + dependencies: + '@speechify/api': + specifier: ^2.0.0 + version: 2.0.0 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugin-openai': + specifier: workspace:* + version: link:../openai + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.30 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/tavus: dependencies: livekit-server-sdk: @@ -2713,6 +2741,10 @@ packages: '@rushstack/ts-command-line@4.21.0': resolution: {integrity: sha512-z38FLUCn8M9FQf19gJ9eltdwkvc47PxvJmVZS6aKwbBAa3Pis3r3A+ZcBCVPNb9h/Tbga+i0tHdzoSGUoji9GQ==} + '@speechify/api@2.0.0': + resolution: {integrity: sha512-GYIYO/B0bRArg9e4ljLxHL/ZG27p7X68UGu6bvEqBk0I8CSpqPKHO+rsGGJR+oIRzXZpR5LVNcS6RPN0ljnvNw==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -5678,7 +5710,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.4.1 + debug: 4.4.3 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -5758,7 +5790,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.4.1 + debug: 4.4.3 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -6521,6 +6553,8 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@speechify/api@2.0.0': {} + '@standard-schema/spec@1.1.0': {} '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': @@ -6647,7 +6681,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.9.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.9.3) - debug: 4.4.1 + debug: 4.4.3 eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.9.3) optionalDependencies: @@ -6661,7 +6695,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.4.1 + debug: 4.4.3 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -7360,7 +7394,7 @@ snapshots: eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: - debug: 4.4.1 + debug: 4.4.3 enhanced-resolve: 5.16.1 eslint: 8.57.0 eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) @@ -9077,7 +9111,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.1 + debug: 4.4.3 esbuild: 0.25.2 joycon: 3.1.1 picocolors: 1.1.1 @@ -9105,7 +9139,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.1 + debug: 4.4.3 esbuild: 0.25.2 joycon: 3.1.1 picocolors: 1.1.1 diff --git a/turbo.json b/turbo.json index b3a1c44c5..b68291059 100644 --- a/turbo.json +++ b/turbo.json @@ -87,6 +87,7 @@ "RUNWAY_AVATAR_PRESET_ID", "SARVAM_API_KEY", "SONIOX_API_KEY", + "SPEECHIFY_API_KEY", "SIP_PARTICIPANT_IDENTITY", "SIP_PHONE_NUMBER", "LK_OPENAI_DEBUG", From d39adea4ac34ba30d21c1883c78ba6ea4bfc61dc Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:36:15 +0100 Subject: [PATCH 02/10] chore: add changeset for speechify plugin --- .changeset/add-speechify-plugin.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/add-speechify-plugin.md diff --git a/.changeset/add-speechify-plugin.md b/.changeset/add-speechify-plugin.md new file mode 100644 index 000000000..72ff73c75 --- /dev/null +++ b/.changeset/add-speechify-plugin.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents-plugin-speechify": patch +--- + +feat(speechify): add Speechify TTS plugin with streaming and word-level timestamps From 1c6a58139231b0d2337c98807e61670e4c301cae Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:58:03 +0100 Subject: [PATCH 03/10] chore: use LiveKit copyright headers to match repo convention --- plugins/speechify/README.md | 2 +- plugins/speechify/src/models.ts | 2 +- plugins/speechify/src/tts.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/speechify/README.md b/plugins/speechify/README.md index 4fe5f6dcf..22827844c 100644 --- a/plugins/speechify/README.md +++ b/plugins/speechify/README.md @@ -1,5 +1,5 @@ diff --git a/plugins/speechify/src/models.ts b/plugins/speechify/src/models.ts index 8c4b54a69..f5b83dfc6 100644 --- a/plugins/speechify/src/models.ts +++ b/plugins/speechify/src/models.ts @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2025 Speechify, Inc. +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index 032119e67..72abca5ec 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2025 Speechify, Inc. +// SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 import { From 26fede213d102a7aac5007c46fe32c4852dc5ded Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:18:05 +0100 Subject: [PATCH 04/10] fix(speechify): flag final frame per segment so metrics are emitted --- plugins/speechify/src/tts.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index 72abca5ec..eefbc7130 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -12,6 +12,7 @@ import { tokenize, tts, } from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; import type { Speechify } from '@speechify/api'; import { SpeechifyClient, SpeechifyError } from '@speechify/api'; import type { TTSModels } from './models.js'; @@ -243,19 +244,26 @@ export class SynthesizeStream extends tts.SynthesizeStream { const timed = timedStringsFromMarks(response.speech_marks, cumulativeDuration); let attached = false; - const putFrames = (frames: ReturnType) => { - for (const frame of frames) { - if (!attached && timed.length > 0) { - frame.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; - attached = true; - } - cumulativeDuration += frame.samplesPerChannel / frame.sampleRate; - this.queue.put({ requestId, frame, final: false, segmentId: requestId }); + // Defer emitting each frame by one so the last frame of this segment can + // be flagged final: true, which the base class needs to emit per-segment + // metrics (matches the elevenlabs/cartesia plugins). + let lastFrame: AudioFrame | undefined; + const sendFrame = (final: boolean) => { + if (!lastFrame) return; + if (!attached && timed.length > 0) { + lastFrame.userdata[USERDATA_TIMED_TRANSCRIPT] = timed; + attached = true; } + cumulativeDuration += lastFrame.samplesPerChannel / lastFrame.sampleRate; + this.queue.put({ requestId, frame: lastFrame, final, segmentId: requestId }); + lastFrame = undefined; }; - putFrames(bstream.write(audio)); - putFrames(bstream.flush()); + for (const frame of [...bstream.write(audio), ...bstream.flush()]) { + sendFrame(false); + lastFrame = frame; + } + sendFrame(true); }; const consume = async () => { From d1236468df0f2710caa884931f16c545e29023d0 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:43:36 +0100 Subject: [PATCH 05/10] feat(speechify): add simba-3.2 and default to dominic_32 voice --- plugins/speechify/src/models.ts | 2 +- plugins/speechify/src/tts.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/plugins/speechify/src/models.ts b/plugins/speechify/src/models.ts index f5b83dfc6..69ef2f9ea 100644 --- a/plugins/speechify/src/models.ts +++ b/plugins/speechify/src/models.ts @@ -2,4 +2,4 @@ // // SPDX-License-Identifier: Apache-2.0 -export type TTSModels = 'simba-english' | 'simba-multilingual' | 'simba-3.0'; +export type TTSModels = 'simba-english' | 'simba-multilingual' | 'simba-3.0' | 'simba-3.2'; diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index eefbc7130..83511839b 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -20,7 +20,8 @@ import type { TTSModels } from './models.js'; const NUM_CHANNELS = 1; const SAMPLE_RATE = 24000; const AUDIO_FORMAT: Speechify.GetSpeechRequest.AudioFormat = 'pcm'; -const DEFAULT_VOICE_ID = 'jack'; +const DEFAULT_VOICE_ID = 'dominic_32'; +const DEFAULT_MODEL: TTSModels = 'simba-3.2'; export interface TTSOptions { voiceId: string; @@ -36,6 +37,7 @@ export interface TTSOptions { const defaultOptions = (): Omit => ({ voiceId: DEFAULT_VOICE_ID, + model: DEFAULT_MODEL, }); const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeechRequest => { @@ -44,7 +46,9 @@ const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeech input: text, voice_id: opts.voiceId, }; - if (opts.model) request.model = opts.model; + // The SDK's Model union can lag newly released models; TTSModels is the + // plugin's source of truth and the API accepts the value. + if (opts.model) request.model = opts.model as Speechify.GetSpeechRequest.Model; if (opts.language) request.language = opts.language; if (opts.loudnessNormalization !== undefined || opts.textNormalization !== undefined) { request.options = { @@ -80,6 +84,9 @@ export class TTS extends tts.TTS { * (24 kHz mono) plus word-level speech marks. `stream()` chunks input into * sentences and issues one request per sentence, emitting audio and aligned * word timestamps as each sentence completes. + * + * Defaults to the `dominic_32` voice and the `simba-3.2` model. The voice must + * support the chosen model; see the `/v1/voices` endpoint. */ constructor(opts: Partial = {}) { const merged = { ...defaultOptions(), ...opts }; From 79a06ad9e9fed0f42a8e36c964002ef352426ff5 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:13:57 +0100 Subject: [PATCH 06/10] fix(speechify): pass through APIError; use 2026 license headers --- plugins/speechify/README.md | 2 +- plugins/speechify/src/index.ts | 2 +- plugins/speechify/src/models.ts | 2 +- plugins/speechify/src/tts.ts | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/speechify/README.md b/plugins/speechify/README.md index 22827844c..8468d7ceb 100644 --- a/plugins/speechify/README.md +++ b/plugins/speechify/README.md @@ -1,5 +1,5 @@ diff --git a/plugins/speechify/src/index.ts b/plugins/speechify/src/index.ts index 252e901dd..cd06b682b 100644 --- a/plugins/speechify/src/index.ts +++ b/plugins/speechify/src/index.ts @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 import { Plugin } from '@livekit/agents'; diff --git a/plugins/speechify/src/models.ts b/plugins/speechify/src/models.ts index 69ef2f9ea..6d54a43e1 100644 --- a/plugins/speechify/src/models.ts +++ b/plugins/speechify/src/models.ts @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index 83511839b..1a66f4e8d 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -1,9 +1,10 @@ -// SPDX-FileCopyrightText: 2024 LiveKit, Inc. +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 import { type APIConnectOptions, APIConnectionError, + APIError, APIStatusError, AudioByteStream, USERDATA_TIMED_TRANSCRIPT, @@ -60,6 +61,9 @@ const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeech }; const toError = (e: unknown): Error => { + if (e instanceof APIError) { + return e; + } if (e instanceof SpeechifyError) { return new APIStatusError({ message: e.message, options: { statusCode: e.statusCode ?? -1 } }); } From aa53d11bef7c2210799632b8b3f0146611e32af7 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:09:41 +0100 Subject: [PATCH 07/10] fix(speechify): align plugin with @speechify/api v3.0.1 (token= auth, drop model cast) The SDK v3.0.0 renamed the auth constructor option from apiKey to token, and v3.0.1's GetSpeechRequest.Model union now includes simba-3.0 and simba-3.2 natively so the plugin no longer needs to widen it via a cast. - plugins/speechify/package.json: @speechify/api ^2.0.0 -> ^3.0.1 - plugins/speechify/src/tts.ts: - rename public TTSOptions.apiKey -> token; pass through to new SpeechifyClient({ token, baseUrl }) - update JSDoc + local variable + Omit-union for updateOptions - drop the `opts.model as Speechify.GetSpeechRequest.Model` cast + comment; v3.0.1's Model union already covers simba-english / simba-multilingual / simba-3.0 / simba-3.2 - pnpm-lock.yaml: refreshed --- plugins/speechify/package.json | 2 +- plugins/speechify/src/tts.ts | 18 ++++++++---------- pnpm-lock.yaml | 10 +++++----- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/plugins/speechify/package.json b/plugins/speechify/package.json index fbf45fcac..fe7bd5514 100644 --- a/plugins/speechify/package.json +++ b/plugins/speechify/package.json @@ -43,7 +43,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@speechify/api": "^2.0.0" + "@speechify/api": "^3.0.1" }, "peerDependencies": { "@livekit/agents": "workspace:*", diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index 1a66f4e8d..c0e16a235 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -30,7 +30,7 @@ export interface TTSOptions { language?: string; loudnessNormalization?: boolean; textNormalization?: boolean; - apiKey?: string; + token?: string; baseUrl?: string; client?: SpeechifyClient; tokenizer?: tokenize.SentenceTokenizer; @@ -47,9 +47,7 @@ const buildSpeechRequest = (text: string, opts: TTSOptions): Speechify.GetSpeech input: text, voice_id: opts.voiceId, }; - // The SDK's Model union can lag newly released models; TTSModels is the - // plugin's source of truth and the API accepts the value. - if (opts.model) request.model = opts.model as Speechify.GetSpeechRequest.Model; + if (opts.model) request.model = opts.model; if (opts.language) request.language = opts.language; if (opts.loudnessNormalization !== undefined || opts.textNormalization !== undefined) { request.options = { @@ -80,9 +78,9 @@ export class TTS extends tts.TTS { * Create a new instance of Speechify TTS. * * @remarks - * `apiKey` must be set, either via the constructor or the `SPEECHIFY_API_KEY` + * `token` must be set, either via the constructor or the `SPEECHIFY_API_KEY` * environment variable. Pass a preconfigured `client` to reuse an existing - * `SpeechifyClient` (in which case `apiKey`/`baseUrl` are ignored). + * `SpeechifyClient` (in which case `token`/`baseUrl` are ignored). * * Synthesis uses the Speechify `/audio/speech` endpoint, which returns raw PCM * (24 kHz mono) plus word-level speech marks. `stream()` chunks input into @@ -103,13 +101,13 @@ export class TTS extends tts.TTS { if (merged.client) { this.#client = merged.client; } else { - const apiKey = merged.apiKey ?? process.env.SPEECHIFY_API_KEY; - if (!apiKey) { + const token = merged.token ?? process.env.SPEECHIFY_API_KEY; + if (!token) { throw new Error( 'Speechify API key is required, whether as an argument or as $SPEECHIFY_API_KEY', ); } - this.#client = new SpeechifyClient({ apiKey, baseUrl: merged.baseUrl }); + this.#client = new SpeechifyClient({ token, baseUrl: merged.baseUrl }); } } @@ -133,7 +131,7 @@ export class TTS extends tts.TTS { return this.#tokenizer; } - updateOptions(opts: Partial>) { + updateOptions(opts: Partial>) { this.#opts = { ...this.#opts, ...opts }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10262223b..9b5078b9f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1330,8 +1330,8 @@ importers: plugins/speechify: dependencies: '@speechify/api': - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^3.0.1 + version: 3.0.1 devDependencies: '@livekit/agents': specifier: workspace:* @@ -2741,8 +2741,8 @@ packages: '@rushstack/ts-command-line@4.21.0': resolution: {integrity: sha512-z38FLUCn8M9FQf19gJ9eltdwkvc47PxvJmVZS6aKwbBAa3Pis3r3A+ZcBCVPNb9h/Tbga+i0tHdzoSGUoji9GQ==} - '@speechify/api@2.0.0': - resolution: {integrity: sha512-GYIYO/B0bRArg9e4ljLxHL/ZG27p7X68UGu6bvEqBk0I8CSpqPKHO+rsGGJR+oIRzXZpR5LVNcS6RPN0ljnvNw==} + '@speechify/api@3.0.1': + resolution: {integrity: sha512-bEqbyvwGW0A/cgq+ZsQJVP3Hm4/0Bm1sniVtPLBh7vDNG2f/YY8y8sweWojoUnhIeIAL6bHBde43+VZLA2MYpA==} engines: {node: '>=18.0.0'} '@standard-schema/spec@1.1.0': @@ -6553,7 +6553,7 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@speechify/api@2.0.0': {} + '@speechify/api@3.0.1': {} '@standard-schema/spec@1.1.0': {} From 9951b068075cd20a9692af6da1c22fc4904bc06f Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:30:56 +0100 Subject: [PATCH 08/10] fix(speechify): call markStarted before each synthesis request Addresses Devin review feedback on livekit/agents-js#1968. The base class SynthesizeStream.monitorMetrics gates all TTS metric emission on this.#startedHrTime being set, which only happens via markStarted(). Without it, TTFB, duration, and character-count metrics were silently dropped for all Speechify streaming synthesis, and USERDATA_TTS_STARTED_TIME never got stamped on frames so downstream transcription timing was degraded. Matches the elevenlabs / cartesia / resemble / deepgram plugin pattern. --- plugins/speechify/src/tts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index c0e16a235..a7f742f0c 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -244,6 +244,7 @@ export class SynthesizeStream extends tts.SynthesizeStream { const requestId = shortuuid(); const bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS); + this.markStarted(); const response = await this.#tts.client.audio.speech(buildSpeechRequest(text, this.#opts), { abortSignal: this.abortSignal, timeoutInSeconds: this.connOptions.timeoutMs / 1000, From 8e0669e6175d32bf1482e969667c9e499720a656 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:39:09 +0100 Subject: [PATCH 09/10] fix(speechify): stop closing output queue in run() error paths Addresses Devin review findings on livekit/agents-js#1968 that both Chunked and Synthesize streams close `this.queue` in their run() catch blocks before re-throwing. The base class already closes the queue in its own `finally` after run() returns, and it retries run() on retryable APIErrors - so closing on error made the next retry's `queue.put(...)` throw 'Queue is closed', turning transient network errors into permanent failures. Matches the guard the minimax plugin documents on its own catch. --- plugins/speechify/src/tts.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index a7f742f0c..c9d056cf4 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -205,7 +205,6 @@ export class ChunkedStream extends tts.ChunkedStream { putFrames(bstream.flush()); this.queue.close(); } catch (e) { - if (!this.queue.closed) this.queue.close(); if (this.abortSignal.aborted) return; throw toError(e); } @@ -289,7 +288,6 @@ export class SynthesizeStream extends tts.SynthesizeStream { try { await Promise.all([forwardInput(), consume()]); } catch (e) { - if (!this.queue.closed) this.queue.close(); if (this.abortSignal.aborted) return; throw toError(e); } From d86166598b7a10b4849a1bafdc594310b9bb91e6 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:48:42 +0100 Subject: [PATCH 10/10] fix(speechify): keep sentence tokenizer alive after endInput Addresses Devin review finding on livekit/agents-js#1968: calling `sentenceStream.close()` immediately after `endInput()` closes the tokenizer's output queue before its internal async loop has a chance to flush the last buffered sentence, so the final `queue.put(token)` throws 'Queue is closed' and consume() never sees the tail. Result: the last sentence of a TTS response could be silently dropped. Matches the neuphonic plugin pattern: only call endInput(), let the tokenizer close its own output queue when its input drains. --- plugins/speechify/src/tts.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/speechify/src/tts.ts b/plugins/speechify/src/tts.ts index c9d056cf4..2d5fd83de 100644 --- a/plugins/speechify/src/tts.ts +++ b/plugins/speechify/src/tts.ts @@ -236,7 +236,6 @@ export class SynthesizeStream extends tts.SynthesizeStream { } } sentenceStream.endInput(); - sentenceStream.close(); }; const synthesizeSentence = async (text: string) => {